Restructure solution layout by module
This commit is contained in:
23
src/Excititor/StellaOps.Excititor.Worker/AGENTS.md
Normal file
23
src/Excititor/StellaOps.Excititor.Worker/AGENTS.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Background processing host coordinating scheduled pulls, retries, reconciliation, verification, and cache maintenance for Excititor.
|
||||
## Scope
|
||||
- Hosted service (Worker Service) wiring timers/queues for provider pulls and reconciliation cycles.
|
||||
- Resume token management, retry policies, and failure quarantines for connectors.
|
||||
- Re-verification of stored attestations and cache garbage collection routines.
|
||||
- Operational metrics and structured logging for offline-friendly monitoring.
|
||||
## Participants
|
||||
- Triggered by WebService job requests or internal schedules to run connector pulls.
|
||||
- Collaborates with Storage.Mongo repositories and Attestation verification utilities.
|
||||
- Emits telemetry consumed by observability stack and CLI status queries.
|
||||
## Interfaces & contracts
|
||||
- Scheduler abstractions, provider run controllers, retry/backoff strategies, and queue processors.
|
||||
- Hooks for policy revision changes and cache GC thresholds.
|
||||
## In/Out of scope
|
||||
In: background orchestration, job lifecycle management, observability for worker operations.
|
||||
Out: HTTP endpoint definitions, domain modeling, connector-specific parsing logic.
|
||||
## Observability & security expectations
|
||||
- Publish metrics for pull latency, failure counts, retry depth, cache size, and verification outcomes.
|
||||
- Log correlation IDs & provider IDs; avoid leaking secret config values.
|
||||
## Tests
|
||||
- Worker orchestration tests, timer controls, and retry behavior will live in `../StellaOps.Excititor.Worker.Tests`.
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Excititor.Worker.Scheduling;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Options;
|
||||
|
||||
public sealed class VexWorkerOptions
|
||||
{
|
||||
public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6);
|
||||
|
||||
public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public bool OfflineMode { get; set; }
|
||||
|
||||
public IList<VexWorkerProviderOptions> Providers { get; } = new List<VexWorkerProviderOptions>();
|
||||
|
||||
public VexWorkerRetryOptions Retry { get; } = new();
|
||||
|
||||
public VexWorkerRefreshOptions Refresh { get; } = new();
|
||||
|
||||
internal IReadOnlyList<VexWorkerSchedule> ResolveSchedules()
|
||||
{
|
||||
var schedules = new List<VexWorkerSchedule>();
|
||||
foreach (var provider in Providers)
|
||||
{
|
||||
if (!provider.Enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var providerId = provider.ProviderId?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(providerId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval);
|
||||
if (interval <= TimeSpan.Zero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var initialDelay = provider.InitialDelay ?? DefaultInitialDelay;
|
||||
if (initialDelay < TimeSpan.Zero)
|
||||
{
|
||||
initialDelay = TimeSpan.Zero;
|
||||
}
|
||||
|
||||
var connectorSettings = provider.Settings.Count == 0
|
||||
? VexConnectorSettings.Empty
|
||||
: new VexConnectorSettings(provider.Settings.ToImmutableDictionary(StringComparer.Ordinal));
|
||||
|
||||
schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay, connectorSettings));
|
||||
}
|
||||
|
||||
return schedules;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class VexWorkerProviderOptions
|
||||
{
|
||||
public string ProviderId { get; set; } = string.Empty;
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public TimeSpan? Interval { get; set; }
|
||||
|
||||
public TimeSpan? InitialDelay { get; set; }
|
||||
|
||||
public IDictionary<string, string> Settings { get; } = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Options;
|
||||
|
||||
internal sealed class VexWorkerOptionsValidator : IValidateOptions<VexWorkerOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, VexWorkerOptions options)
|
||||
{
|
||||
var failures = new List<string>();
|
||||
|
||||
if (options.DefaultInterval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.DefaultInterval must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.OfflineInterval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.OfflineInterval must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.DefaultInitialDelay < TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.DefaultInitialDelay cannot be negative.");
|
||||
}
|
||||
|
||||
if (options.Retry.BaseDelay <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.BaseDelay must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Retry.MaxDelay < options.Retry.BaseDelay)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.MaxDelay must be greater than or equal to BaseDelay.");
|
||||
}
|
||||
|
||||
if (options.Retry.QuarantineDuration <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.QuarantineDuration must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Retry.FailureThreshold < 1)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.FailureThreshold must be at least 1.");
|
||||
}
|
||||
|
||||
if (options.Retry.JitterRatio < 0 || options.Retry.JitterRatio > 1)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.JitterRatio must be between 0 and 1.");
|
||||
}
|
||||
|
||||
if (options.Retry.RetryCap < options.Retry.BaseDelay)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to BaseDelay.");
|
||||
}
|
||||
|
||||
if (options.Retry.RetryCap < options.Retry.MaxDelay)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to MaxDelay.");
|
||||
}
|
||||
|
||||
if (options.Refresh.ScanInterval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.ScanInterval must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Refresh.ConsensusTtl <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.ConsensusTtl must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Refresh.ScanBatchSize <= 0)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.ScanBatchSize must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Refresh.Damper.Minimum < TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.Damper.Minimum cannot be negative.");
|
||||
}
|
||||
|
||||
if (options.Refresh.Damper.Maximum <= options.Refresh.Damper.Minimum)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.Damper.Maximum must be greater than Minimum.");
|
||||
}
|
||||
|
||||
if (options.Refresh.Damper.DefaultDuration < options.Refresh.Damper.Minimum || options.Refresh.Damper.DefaultDuration > options.Refresh.Damper.Maximum)
|
||||
{
|
||||
failures.Add("Excititor.Worker.Refresh.Damper.DefaultDuration must be within [Minimum, Maximum].");
|
||||
}
|
||||
|
||||
for (var i = 0; i < options.Refresh.Damper.Rules.Count; i++)
|
||||
{
|
||||
var rule = options.Refresh.Damper.Rules[i];
|
||||
if (rule.MinWeight < 0)
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].MinWeight must be non-negative.");
|
||||
}
|
||||
|
||||
if (rule.Duration <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be greater than zero.");
|
||||
}
|
||||
|
||||
if (rule.Duration < options.Refresh.Damper.Minimum || rule.Duration > options.Refresh.Damper.Maximum)
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be within [Minimum, Maximum].");
|
||||
}
|
||||
}
|
||||
|
||||
for (var i = 0; i < options.Providers.Count; i++)
|
||||
{
|
||||
var provider = options.Providers[i];
|
||||
if (string.IsNullOrWhiteSpace(provider.ProviderId))
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Providers[{i}].ProviderId must be set.");
|
||||
}
|
||||
|
||||
if (provider.Interval is { } interval && interval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Providers[{i}].Interval must be greater than zero when specified.");
|
||||
}
|
||||
|
||||
if (provider.InitialDelay is { } delay && delay < TimeSpan.Zero)
|
||||
{
|
||||
failures.Add($"Excititor.Worker.Providers[{i}].InitialDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
|
||||
return failures.Count > 0
|
||||
? ValidateOptionsResult.Fail(failures)
|
||||
: ValidateOptionsResult.Success;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Options;
|
||||
|
||||
public sealed class VexWorkerPluginOptions
|
||||
{
|
||||
public string? Directory { get; set; }
|
||||
|
||||
public string? SearchPattern { get; set; }
|
||||
|
||||
internal string ResolveDirectory()
|
||||
=> string.IsNullOrWhiteSpace(Directory)
|
||||
? Path.Combine(AppContext.BaseDirectory, "plugins")
|
||||
: Path.GetFullPath(Directory);
|
||||
|
||||
internal string ResolveSearchPattern()
|
||||
=> string.IsNullOrWhiteSpace(SearchPattern)
|
||||
? "StellaOps.Excititor.Connectors.*.dll"
|
||||
: SearchPattern!;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Options;
|
||||
|
||||
public sealed class VexWorkerRefreshOptions
|
||||
{
|
||||
private static readonly TimeSpan DefaultScanInterval = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan DefaultConsensusTtl = TimeSpan.FromHours(2);
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public TimeSpan ScanInterval { get; set; } = DefaultScanInterval;
|
||||
|
||||
public TimeSpan ConsensusTtl { get; set; } = DefaultConsensusTtl;
|
||||
|
||||
public int ScanBatchSize { get; set; } = 250;
|
||||
|
||||
public VexStabilityDamperOptions Damper { get; } = new();
|
||||
}
|
||||
|
||||
public sealed class VexStabilityDamperOptions
|
||||
{
|
||||
private static readonly TimeSpan DefaultMinimum = TimeSpan.FromHours(24);
|
||||
private static readonly TimeSpan DefaultMaximum = TimeSpan.FromHours(48);
|
||||
private static readonly TimeSpan DefaultDurationBaseline = TimeSpan.FromHours(36);
|
||||
|
||||
public TimeSpan Minimum { get; set; } = DefaultMinimum;
|
||||
|
||||
public TimeSpan Maximum { get; set; } = DefaultMaximum;
|
||||
|
||||
public TimeSpan DefaultDuration { get; set; } = DefaultDurationBaseline;
|
||||
|
||||
public IList<VexStabilityDamperRule> Rules { get; } = new List<VexStabilityDamperRule>
|
||||
{
|
||||
new() { MinWeight = 0.9, Duration = TimeSpan.FromHours(24) },
|
||||
new() { MinWeight = 0.75, Duration = TimeSpan.FromHours(30) },
|
||||
new() { MinWeight = 0.5, Duration = TimeSpan.FromHours(36) },
|
||||
};
|
||||
|
||||
internal TimeSpan ClampDuration(TimeSpan duration)
|
||||
{
|
||||
if (duration < Minimum)
|
||||
{
|
||||
return Minimum;
|
||||
}
|
||||
|
||||
if (duration > Maximum)
|
||||
{
|
||||
return Maximum;
|
||||
}
|
||||
|
||||
return duration;
|
||||
}
|
||||
|
||||
public TimeSpan ResolveDuration(double weight)
|
||||
{
|
||||
if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0)
|
||||
{
|
||||
return ClampDuration(DefaultDuration);
|
||||
}
|
||||
|
||||
if (Rules.Count == 0)
|
||||
{
|
||||
return ClampDuration(DefaultDuration);
|
||||
}
|
||||
|
||||
// Evaluate highest weight threshold first.
|
||||
TimeSpan? selected = null;
|
||||
foreach (var rule in Rules.OrderByDescending(static r => r.MinWeight))
|
||||
{
|
||||
if (weight >= rule.MinWeight)
|
||||
{
|
||||
selected = rule.Duration;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return ClampDuration(selected ?? DefaultDuration);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class VexStabilityDamperRule
|
||||
{
|
||||
public double MinWeight { get; set; }
|
||||
= 1.0;
|
||||
|
||||
public TimeSpan Duration { get; set; }
|
||||
= TimeSpan.FromHours(24);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Options;
|
||||
|
||||
public sealed class VexWorkerRetryOptions
|
||||
{
|
||||
[Range(1, int.MaxValue)]
|
||||
public int FailureThreshold { get; set; } = 3;
|
||||
|
||||
[Range(typeof(double), "0.0", "1.0")]
|
||||
public double JitterRatio { get; set; } = 0.2;
|
||||
|
||||
public TimeSpan BaseDelay { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromHours(6);
|
||||
|
||||
public TimeSpan QuarantineDuration { get; set; } = TimeSpan.FromHours(12);
|
||||
|
||||
public TimeSpan RetryCap { get; set; } = TimeSpan.FromHours(24);
|
||||
}
|
||||
93
src/Excititor/StellaOps.Excititor.Worker/Program.cs
Normal file
93
src/Excititor/StellaOps.Excititor.Worker/Program.cs
Normal file
@@ -0,0 +1,93 @@
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Plugin;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.Formats.CSAF;
|
||||
using StellaOps.Excititor.Formats.CycloneDX;
|
||||
using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Scheduling;
|
||||
using StellaOps.Excititor.Worker.Signature;
|
||||
using StellaOps.Excititor.Attestation.Extensions;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
var services = builder.Services;
|
||||
var configuration = builder.Configuration;
|
||||
services.AddOptions<VexWorkerOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Worker"))
|
||||
.ValidateOnStart();
|
||||
|
||||
services.Configure<VexWorkerPluginOptions>(configuration.GetSection("Excititor:Worker:Plugins"));
|
||||
services.AddRedHatCsafConnector();
|
||||
|
||||
services.AddOptions<VexMongoStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddExcititorMongoStorage();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
services.AddOpenVexNormalizer();
|
||||
services.AddSingleton<IVexSignatureVerifier, WorkerSignatureVerifier>();
|
||||
services.AddVexAttestation();
|
||||
services.Configure<VexAttestationVerificationOptions>(configuration.GetSection("Excititor:Attestation:Verification"));
|
||||
services.PostConfigure<VexAttestationVerificationOptions>(options =>
|
||||
{
|
||||
// Workers operate in offline-first environments; allow verification to succeed without Rekor.
|
||||
options.AllowOfflineTransparency = true;
|
||||
if (!configuration.GetSection("Excititor:Attestation:Verification").Exists())
|
||||
{
|
||||
options.RequireTransparencyLog = false;
|
||||
}
|
||||
});
|
||||
services.AddExcititorAocGuards();
|
||||
|
||||
services.AddSingleton<IValidateOptions<VexWorkerOptions>, VexWorkerOptionsValidator>();
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.PostConfigure<VexWorkerOptions>(options =>
|
||||
{
|
||||
if (!options.Providers.Any(provider => string.Equals(provider.ProviderId, "excititor:redhat", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
options.Providers.Add(new VexWorkerProviderOptions
|
||||
{
|
||||
ProviderId = "excititor:redhat",
|
||||
});
|
||||
}
|
||||
});
|
||||
services.AddSingleton<PluginCatalog>(provider =>
|
||||
{
|
||||
var pluginOptions = provider.GetRequiredService<IOptions<VexWorkerPluginOptions>>().Value;
|
||||
var catalog = new PluginCatalog();
|
||||
|
||||
var directory = pluginOptions.ResolveDirectory();
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
catalog.AddFromDirectory(directory, pluginOptions.ResolveSearchPattern());
|
||||
}
|
||||
else
|
||||
{
|
||||
var logger = provider.GetRequiredService<ILogger<Program>>();
|
||||
logger.LogWarning("Excititor worker plugin directory '{Directory}' does not exist; proceeding without external connectors.", directory);
|
||||
}
|
||||
|
||||
return catalog;
|
||||
});
|
||||
|
||||
services.AddSingleton<IVexProviderRunner, DefaultVexProviderRunner>();
|
||||
services.AddSingleton<VexConsensusRefreshService>();
|
||||
services.AddSingleton<IVexConsensusRefreshScheduler>(static provider => provider.GetRequiredService<VexConsensusRefreshService>());
|
||||
services.AddHostedService<VexWorkerHostedService>();
|
||||
services.AddHostedService(static provider => provider.GetRequiredService<VexConsensusRefreshService>());
|
||||
|
||||
var host = builder.Build();
|
||||
await host.RunAsync();
|
||||
|
||||
public partial class Program;
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.Worker.Tests")]
|
||||
@@ -0,0 +1,271 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Plugin;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Signature;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
internal sealed class DefaultVexProviderRunner : IVexProviderRunner
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly PluginCatalog _pluginCatalog;
|
||||
private readonly ILogger<DefaultVexProviderRunner> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly VexWorkerRetryOptions _retryOptions;
|
||||
|
||||
public DefaultVexProviderRunner(
|
||||
IServiceProvider serviceProvider,
|
||||
PluginCatalog pluginCatalog,
|
||||
ILogger<DefaultVexProviderRunner> logger,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<VexWorkerOptions> workerOptions)
|
||||
{
|
||||
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
|
||||
_pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
if (workerOptions is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(workerOptions));
|
||||
}
|
||||
|
||||
_retryOptions = workerOptions.Value?.Retry ?? throw new InvalidOperationException("VexWorkerOptions.Retry must be configured.");
|
||||
}
|
||||
|
||||
public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(schedule);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId);
|
||||
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
|
||||
var matched = availablePlugins.FirstOrDefault(plugin =>
|
||||
string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (matched is not null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.",
|
||||
matched.Name,
|
||||
schedule.ProviderId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId);
|
||||
}
|
||||
|
||||
var connectors = scope.ServiceProvider.GetServices<IVexConnector>();
|
||||
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (connector is null)
|
||||
{
|
||||
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId);
|
||||
return;
|
||||
}
|
||||
|
||||
await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
var effectiveSettings = settings ?? VexConnectorSettings.Empty;
|
||||
var rawStore = scopeProvider.GetRequiredService<IVexRawStore>();
|
||||
var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>();
|
||||
var stateRepository = scopeProvider.GetRequiredService<IVexConnectorStateRepository>();
|
||||
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
|
||||
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
|
||||
var sessionProvider = scopeProvider.GetService<IVexMongoSessionProvider>();
|
||||
IClientSessionHandle? session = null;
|
||||
if (sessionProvider is not null)
|
||||
{
|
||||
session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var descriptor = connector switch
|
||||
{
|
||||
VexConnectorBase baseConnector => baseConnector.Descriptor,
|
||||
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
|
||||
};
|
||||
|
||||
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false)
|
||||
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
|
||||
|
||||
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
|
||||
|
||||
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Connector {ConnectorId} is in backoff until {NextEligible:O}; skipping run.",
|
||||
connector.Id,
|
||||
nextEligible);
|
||||
return;
|
||||
}
|
||||
|
||||
await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var verifyingSink = new VerifyingVexRawDocumentSink(rawStore, signatureVerifier);
|
||||
|
||||
var context = new VexConnectorContext(
|
||||
Since: stateBeforeRun?.LastUpdated,
|
||||
Settings: effectiveSettings,
|
||||
RawSink: verifyingSink,
|
||||
SignatureVerifier: signatureVerifier,
|
||||
Normalizers: normalizerRouter,
|
||||
Services: scopeProvider,
|
||||
ResumeTokens: stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var documentCount = 0;
|
||||
|
||||
try
|
||||
{
|
||||
await foreach (var document in connector.FetchAsync(context, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
documentCount++;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Connector {ConnectorId} persisted {DocumentCount} raw document(s) this run.",
|
||||
connector.Id,
|
||||
documentCount);
|
||||
|
||||
await UpdateSuccessStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await UpdateFailureStateAsync(stateRepository, descriptor.Id, _timeProvider.GetUtcNow(), ex, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateSuccessStateAsync(
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
string connectorId,
|
||||
DateTimeOffset completedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
|
||||
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
|
||||
|
||||
var updated = current with
|
||||
{
|
||||
LastSuccessAt = completedAt,
|
||||
FailureCount = 0,
|
||||
NextEligibleRun = null,
|
||||
LastFailureReason = null
|
||||
};
|
||||
|
||||
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task UpdateFailureStateAsync(
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
string connectorId,
|
||||
DateTimeOffset failureTime,
|
||||
Exception exception,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await stateRepository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false)
|
||||
?? new VexConnectorState(connectorId, null, ImmutableArray<string>.Empty);
|
||||
|
||||
var failureCount = current.FailureCount + 1;
|
||||
var delay = CalculateDelayWithJitter(failureCount);
|
||||
var nextEligible = failureTime + delay;
|
||||
|
||||
if (failureCount >= _retryOptions.FailureThreshold)
|
||||
{
|
||||
var quarantineUntil = failureTime + _retryOptions.QuarantineDuration;
|
||||
if (quarantineUntil > nextEligible)
|
||||
{
|
||||
nextEligible = quarantineUntil;
|
||||
}
|
||||
}
|
||||
|
||||
var retryCap = failureTime + _retryOptions.RetryCap;
|
||||
if (nextEligible > retryCap)
|
||||
{
|
||||
nextEligible = retryCap;
|
||||
}
|
||||
|
||||
if (nextEligible < failureTime)
|
||||
{
|
||||
nextEligible = failureTime;
|
||||
}
|
||||
|
||||
var updated = current with
|
||||
{
|
||||
FailureCount = failureCount,
|
||||
NextEligibleRun = nextEligible,
|
||||
LastFailureReason = Truncate(exception.Message, 512)
|
||||
};
|
||||
|
||||
await stateRepository.SaveAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Connector {ConnectorId} failed (attempt {Attempt}). Next eligible run at {NextEligible:O}.",
|
||||
connectorId,
|
||||
failureCount,
|
||||
nextEligible);
|
||||
}
|
||||
|
||||
private TimeSpan CalculateDelayWithJitter(int failureCount)
|
||||
{
|
||||
var exponent = Math.Max(0, failureCount - 1);
|
||||
var factor = Math.Pow(2, exponent);
|
||||
var baselineTicks = (long)Math.Min(_retryOptions.BaseDelay.Ticks * factor, _retryOptions.MaxDelay.Ticks);
|
||||
|
||||
if (_retryOptions.JitterRatio <= 0)
|
||||
{
|
||||
return TimeSpan.FromTicks(baselineTicks);
|
||||
}
|
||||
|
||||
var minFactor = 1.0 - _retryOptions.JitterRatio;
|
||||
var maxFactor = 1.0 + _retryOptions.JitterRatio;
|
||||
Span<byte> buffer = stackalloc byte[8];
|
||||
RandomNumberGenerator.Fill(buffer);
|
||||
var sample = BitConverter.ToUInt64(buffer) / (double)ulong.MaxValue;
|
||||
var jitterFactor = minFactor + (maxFactor - minFactor) * sample;
|
||||
var jitteredTicks = (long)Math.Round(baselineTicks * jitterFactor);
|
||||
|
||||
if (jitteredTicks < _retryOptions.BaseDelay.Ticks)
|
||||
{
|
||||
jitteredTicks = _retryOptions.BaseDelay.Ticks;
|
||||
}
|
||||
|
||||
if (jitteredTicks > _retryOptions.MaxDelay.Ticks)
|
||||
{
|
||||
jitteredTicks = _retryOptions.MaxDelay.Ticks;
|
||||
}
|
||||
|
||||
return TimeSpan.FromTicks(jitteredTicks);
|
||||
}
|
||||
|
||||
private static string Truncate(string? value, int maxLength)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return value.Length <= maxLength
|
||||
? value
|
||||
: value[..maxLength];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
public interface IVexConsensusRefreshScheduler
|
||||
{
|
||||
void ScheduleRefresh(string vulnerabilityId, string productKey);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
internal interface IVexProviderRunner
|
||||
{
|
||||
ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,622 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsensusRefreshScheduler
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<VexConsensusRefreshService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly Channel<RefreshRequest> _refreshRequests;
|
||||
private readonly ConcurrentDictionary<string, byte> _scheduledKeys = new(StringComparer.Ordinal);
|
||||
private readonly IDisposable? _optionsSubscription;
|
||||
private RefreshState _refreshState;
|
||||
|
||||
public VexConsensusRefreshService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptionsMonitor<VexWorkerOptions> optionsMonitor,
|
||||
ILogger<VexConsensusRefreshService> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_refreshRequests = Channel.CreateUnbounded<RefreshRequest>(new UnboundedChannelOptions
|
||||
{
|
||||
AllowSynchronousContinuations = false,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
if (optionsMonitor is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(optionsMonitor));
|
||||
}
|
||||
|
||||
var options = optionsMonitor.CurrentValue;
|
||||
_refreshState = RefreshState.FromOptions(options.Refresh);
|
||||
_optionsSubscription = optionsMonitor.OnChange(o =>
|
||||
{
|
||||
var state = RefreshState.FromOptions((o?.Refresh) ?? new VexWorkerRefreshOptions());
|
||||
Volatile.Write(ref _refreshState, state);
|
||||
_logger.LogInformation(
|
||||
"Consensus refresh options updated: enabled={Enabled}, interval={Interval}, ttl={Ttl}, batch={Batch}",
|
||||
state.Enabled,
|
||||
state.ScanInterval,
|
||||
state.ConsensusTtl,
|
||||
state.ScanBatchSize);
|
||||
});
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
_optionsSubscription?.Dispose();
|
||||
base.Dispose();
|
||||
}
|
||||
|
||||
public void ScheduleRefresh(string vulnerabilityId, string productKey)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var key = BuildKey(vulnerabilityId, productKey);
|
||||
if (!_scheduledKeys.TryAdd(key, 0))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var request = new RefreshRequest(vulnerabilityId.Trim(), productKey.Trim());
|
||||
if (!_refreshRequests.Writer.TryWrite(request))
|
||||
{
|
||||
_scheduledKeys.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var queueTask = ProcessQueueAsync(stoppingToken);
|
||||
|
||||
try
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var options = CurrentOptions;
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessEligibleHoldsAsync(options, stoppingToken).ConfigureAwait(false);
|
||||
if (options.Enabled)
|
||||
{
|
||||
await ProcessTtlRefreshAsync(options, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Consensus refresh disabled; skipping TTL sweep.");
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Consensus refresh loop failed.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(options.ScanInterval, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_refreshRequests.Writer.TryComplete();
|
||||
try
|
||||
{
|
||||
await queueTask.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RefreshState CurrentOptions => Volatile.Read(ref _refreshState);
|
||||
|
||||
private async Task ProcessQueueAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (await _refreshRequests.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
while (_refreshRequests.Reader.TryRead(out var request))
|
||||
{
|
||||
var key = BuildKey(request.VulnerabilityId, request.ProductKey);
|
||||
try
|
||||
{
|
||||
await ProcessCandidateAsync(request.VulnerabilityId, request.ProductKey, existingConsensus: null, CurrentOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to refresh consensus for {VulnerabilityId}/{ProductKey} from queue.", request.VulnerabilityId, request.ProductKey);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_scheduledKeys.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessEligibleHoldsAsync(RefreshState options, CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var holdStore = scope.ServiceProvider.GetRequiredService<IVexConsensusHoldStore>();
|
||||
var consensusStore = scope.ServiceProvider.GetRequiredService<IVexConsensusStore>();
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
await foreach (var hold in holdStore.FindEligibleAsync(now, options.ScanBatchSize, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var key = BuildKey(hold.VulnerabilityId, hold.ProductKey);
|
||||
if (!_scheduledKeys.TryAdd(key, 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await consensusStore.SaveAsync(hold.Candidate with { }, cancellationToken).ConfigureAwait(false);
|
||||
await holdStore.RemoveAsync(hold.VulnerabilityId, hold.ProductKey, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"Promoted consensus hold for {VulnerabilityId}/{ProductKey}; status={Status}, reason={Reason}",
|
||||
hold.VulnerabilityId,
|
||||
hold.ProductKey,
|
||||
hold.Candidate.Status,
|
||||
hold.Reason);
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to promote consensus hold for {VulnerabilityId}/{ProductKey}.",
|
||||
hold.VulnerabilityId,
|
||||
hold.ProductKey);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_scheduledKeys.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessTtlRefreshAsync(RefreshState options, CancellationToken cancellationToken)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var cutoff = now - options.ConsensusTtl;
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var consensusStore = scope.ServiceProvider.GetRequiredService<IVexConsensusStore>();
|
||||
|
||||
await foreach (var consensus in consensusStore.FindCalculatedBeforeAsync(cutoff, options.ScanBatchSize, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var key = BuildKey(consensus.VulnerabilityId, consensus.Product.Key);
|
||||
if (!_scheduledKeys.TryAdd(key, 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessCandidateAsync(consensus.VulnerabilityId, consensus.Product.Key, consensus, options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to refresh consensus for {VulnerabilityId}/{ProductKey} during TTL sweep.",
|
||||
consensus.VulnerabilityId,
|
||||
consensus.Product.Key);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_scheduledKeys.TryRemove(key, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessCandidateAsync(
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
VexConsensus? existingConsensus,
|
||||
RefreshState options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var consensusStore = scope.ServiceProvider.GetRequiredService<IVexConsensusStore>();
|
||||
var holdStore = scope.ServiceProvider.GetRequiredService<IVexConsensusHoldStore>();
|
||||
var claimStore = scope.ServiceProvider.GetRequiredService<IVexClaimStore>();
|
||||
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
|
||||
var policyProvider = scope.ServiceProvider.GetRequiredService<IVexPolicyProvider>();
|
||||
|
||||
existingConsensus ??= await consensusStore.FindAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var claims = await claimStore.FindAsync(vulnerabilityId, productKey, since: null, cancellationToken).ConfigureAwait(false);
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No claims found for {VulnerabilityId}/{ProductKey}; skipping consensus refresh.", vulnerabilityId, productKey);
|
||||
await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var claimList = claims as IReadOnlyList<VexClaim> ?? claims.ToList();
|
||||
|
||||
var snapshot = policyProvider.GetSnapshot();
|
||||
var providerCache = new Dictionary<string, VexProvider>(StringComparer.Ordinal);
|
||||
var providers = await LoadProvidersAsync(claimList, providerStore, providerCache, cancellationToken).ConfigureAwait(false);
|
||||
var product = ResolveProduct(claimList, productKey);
|
||||
var calculatedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
var resolver = new VexConsensusResolver(snapshot.ConsensusPolicy);
|
||||
var request = new VexConsensusRequest(
|
||||
vulnerabilityId,
|
||||
product,
|
||||
claimList.ToArray(),
|
||||
providers,
|
||||
calculatedAt,
|
||||
snapshot.ConsensusOptions.WeightCeiling,
|
||||
AggregateSignals(claimList),
|
||||
snapshot.RevisionId,
|
||||
snapshot.Digest);
|
||||
|
||||
var resolution = resolver.Resolve(request);
|
||||
var candidate = NormalizePolicyMetadata(resolution.Consensus, snapshot);
|
||||
|
||||
await ApplyConsensusAsync(
|
||||
candidate,
|
||||
existingConsensus,
|
||||
holdStore,
|
||||
consensusStore,
|
||||
options.Damper,
|
||||
options,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task ApplyConsensusAsync(
|
||||
VexConsensus candidate,
|
||||
VexConsensus? existing,
|
||||
IVexConsensusHoldStore holdStore,
|
||||
IVexConsensusStore consensusStore,
|
||||
DamperState damper,
|
||||
RefreshState options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var vulnerabilityId = candidate.VulnerabilityId;
|
||||
var productKey = candidate.Product.Key;
|
||||
|
||||
var componentChanged = HasComponentChange(existing, candidate);
|
||||
var statusChanged = existing is not null && existing.Status != candidate.Status;
|
||||
|
||||
if (existing is null)
|
||||
{
|
||||
await consensusStore.SaveAsync(candidate, cancellationToken).ConfigureAwait(false);
|
||||
await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Stored initial consensus for {VulnerabilityId}/{ProductKey} with status {Status}.", vulnerabilityId, productKey, candidate.Status);
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan duration = TimeSpan.Zero;
|
||||
if (statusChanged)
|
||||
{
|
||||
if (componentChanged)
|
||||
{
|
||||
duration = TimeSpan.Zero;
|
||||
}
|
||||
else
|
||||
{
|
||||
var mappedStatus = MapConsensusStatus(candidate.Status);
|
||||
var supportingWeight = mappedStatus is null
|
||||
? 0d
|
||||
: candidate.Sources
|
||||
.Where(source => source.Status == mappedStatus.Value)
|
||||
.Sum(source => source.Weight);
|
||||
duration = damper.ResolveDuration(supportingWeight);
|
||||
}
|
||||
}
|
||||
|
||||
var requestedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
if (statusChanged && duration > TimeSpan.Zero)
|
||||
{
|
||||
var eligibleAt = requestedAt + duration;
|
||||
var reason = componentChanged ? "component_change" : "status_change";
|
||||
var newHold = new VexConsensusHold(vulnerabilityId, productKey, candidate, requestedAt, eligibleAt, reason);
|
||||
var existingHold = await holdStore.FindAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (existingHold is null || existingHold.Candidate != candidate || existingHold.EligibleAt != newHold.EligibleAt)
|
||||
{
|
||||
await holdStore.SaveAsync(newHold, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"Deferred consensus update for {VulnerabilityId}/{ProductKey} until {EligibleAt:O}; status {Status} pending (reason={Reason}).",
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
eligibleAt,
|
||||
candidate.Status,
|
||||
reason);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await consensusStore.SaveAsync(candidate, cancellationToken).ConfigureAwait(false);
|
||||
await holdStore.RemoveAsync(vulnerabilityId, productKey, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"Updated consensus for {VulnerabilityId}/{ProductKey}; status={Status}, componentChange={ComponentChanged}.",
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
candidate.Status,
|
||||
componentChanged);
|
||||
}
|
||||
|
||||
private static bool HasComponentChange(VexConsensus? existing, VexConsensus candidate)
|
||||
{
|
||||
if (existing is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var previous = existing.Product.ComponentIdentifiers;
|
||||
var current = candidate.Product.ComponentIdentifiers;
|
||||
|
||||
if (previous.IsDefaultOrEmpty && current.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (previous.Length != current.Length)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
for (var i = 0; i < previous.Length; i++)
|
||||
{
|
||||
if (!string.Equals(previous[i], current[i], StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static VexConsensus NormalizePolicyMetadata(VexConsensus consensus, VexPolicySnapshot snapshot)
|
||||
{
|
||||
if (string.Equals(consensus.PolicyVersion, snapshot.Version, StringComparison.Ordinal) &&
|
||||
string.Equals(consensus.PolicyRevisionId, snapshot.RevisionId, StringComparison.Ordinal) &&
|
||||
string.Equals(consensus.PolicyDigest, snapshot.Digest, StringComparison.Ordinal))
|
||||
{
|
||||
return consensus;
|
||||
}
|
||||
|
||||
return new VexConsensus(
|
||||
consensus.VulnerabilityId,
|
||||
consensus.Product,
|
||||
consensus.Status,
|
||||
consensus.CalculatedAt,
|
||||
consensus.Sources,
|
||||
consensus.Conflicts,
|
||||
consensus.Signals,
|
||||
snapshot.Version,
|
||||
consensus.Summary,
|
||||
snapshot.RevisionId,
|
||||
snapshot.Digest);
|
||||
}
|
||||
|
||||
private static VexClaimStatus? MapConsensusStatus(VexConsensusStatus status)
|
||||
=> status switch
|
||||
{
|
||||
VexConsensusStatus.Affected => VexClaimStatus.Affected,
|
||||
VexConsensusStatus.NotAffected => VexClaimStatus.NotAffected,
|
||||
VexConsensusStatus.Fixed => VexClaimStatus.Fixed,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static string BuildKey(string vulnerabilityId, string productKey)
|
||||
=> string.Create(
|
||||
vulnerabilityId.Length + productKey.Length + 1,
|
||||
(vulnerabilityId, productKey),
|
||||
static (span, tuple) =>
|
||||
{
|
||||
tuple.vulnerabilityId.AsSpan().CopyTo(span);
|
||||
span[tuple.vulnerabilityId.Length] = '|';
|
||||
tuple.productKey.AsSpan().CopyTo(span[(tuple.vulnerabilityId.Length + 1)..]);
|
||||
});
|
||||
|
||||
private static VexProduct ResolveProduct(IReadOnlyList<VexClaim> claims, string productKey)
|
||||
{
|
||||
if (claims.Count > 0)
|
||||
{
|
||||
return claims[0].Product;
|
||||
}
|
||||
|
||||
var inferredPurl = productKey.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase) ? productKey : null;
|
||||
return new VexProduct(productKey, name: null, version: null, purl: inferredPurl);
|
||||
}
|
||||
|
||||
private static VexSignalSnapshot? AggregateSignals(IReadOnlyList<VexClaim> claims)
|
||||
{
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
VexSeveritySignal? bestSeverity = null;
|
||||
double? bestScore = null;
|
||||
bool kevPresent = false;
|
||||
bool kevTrue = false;
|
||||
double? bestEpss = null;
|
||||
|
||||
foreach (var claim in claims)
|
||||
{
|
||||
if (claim.Signals is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var severity = claim.Signals.Severity;
|
||||
if (severity is not null)
|
||||
{
|
||||
var score = severity.Score;
|
||||
if (bestSeverity is null ||
|
||||
(score is not null && (bestScore is null || score.Value > bestScore.Value)) ||
|
||||
(score is null && bestScore is null && !string.IsNullOrWhiteSpace(severity.Label) && string.IsNullOrWhiteSpace(bestSeverity.Label)))
|
||||
{
|
||||
bestSeverity = severity;
|
||||
bestScore = severity.Score;
|
||||
}
|
||||
}
|
||||
|
||||
if (claim.Signals.Kev is { } kevValue)
|
||||
{
|
||||
kevPresent = true;
|
||||
if (kevValue)
|
||||
{
|
||||
kevTrue = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (claim.Signals.Epss is { } epss)
|
||||
{
|
||||
if (bestEpss is null || epss > bestEpss.Value)
|
||||
{
|
||||
bestEpss = epss;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bestSeverity is null && !kevPresent && bestEpss is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
bool? kev = kevTrue ? true : (kevPresent ? false : null);
|
||||
return new VexSignalSnapshot(bestSeverity, kev, bestEpss);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyDictionary<string, VexProvider>> LoadProvidersAsync(
|
||||
IReadOnlyList<VexClaim> claims,
|
||||
IVexProviderStore providerStore,
|
||||
IDictionary<string, VexProvider> cache,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (claims.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, VexProvider>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, VexProvider>(StringComparer.Ordinal);
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var providerId in claims.Select(claim => claim.ProviderId))
|
||||
{
|
||||
if (!seen.Add(providerId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cache.TryGetValue(providerId, out var cached))
|
||||
{
|
||||
builder[providerId] = cached;
|
||||
continue;
|
||||
}
|
||||
|
||||
var provider = await providerStore.FindAsync(providerId, cancellationToken).ConfigureAwait(false);
|
||||
if (provider is not null)
|
||||
{
|
||||
cache[providerId] = provider;
|
||||
builder[providerId] = provider;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private readonly record struct RefreshRequest(string VulnerabilityId, string ProductKey);
|
||||
|
||||
private sealed record RefreshState(
|
||||
bool Enabled,
|
||||
TimeSpan ScanInterval,
|
||||
TimeSpan ConsensusTtl,
|
||||
int ScanBatchSize,
|
||||
DamperState Damper)
|
||||
{
|
||||
public static RefreshState FromOptions(VexWorkerRefreshOptions options)
|
||||
{
|
||||
var interval = options.ScanInterval > TimeSpan.Zero ? options.ScanInterval : TimeSpan.FromMinutes(10);
|
||||
var ttl = options.ConsensusTtl > TimeSpan.Zero ? options.ConsensusTtl : TimeSpan.FromHours(2);
|
||||
var batchSize = options.ScanBatchSize > 0 ? options.ScanBatchSize : 250;
|
||||
var damper = DamperState.FromOptions(options.Damper);
|
||||
return new RefreshState(options.Enabled, interval, ttl, batchSize, damper);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record DamperState(TimeSpan Minimum, TimeSpan Maximum, TimeSpan DefaultDuration, ImmutableArray<DamperRuleState> Rules)
|
||||
{
|
||||
public static DamperState FromOptions(VexStabilityDamperOptions options)
|
||||
{
|
||||
var minimum = options.Minimum < TimeSpan.Zero ? TimeSpan.Zero : options.Minimum;
|
||||
var maximum = options.Maximum > minimum ? options.Maximum : minimum + TimeSpan.FromHours(1);
|
||||
var defaultDuration = options.ClampDuration(options.DefaultDuration);
|
||||
var rules = options.Rules
|
||||
.Select(rule => new DamperRuleState(Math.Max(0, rule.MinWeight), options.ClampDuration(rule.Duration)))
|
||||
.OrderByDescending(rule => rule.MinWeight)
|
||||
.ToImmutableArray();
|
||||
return new DamperState(minimum, maximum, defaultDuration, rules);
|
||||
}
|
||||
|
||||
public TimeSpan ResolveDuration(double weight)
|
||||
{
|
||||
if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0)
|
||||
{
|
||||
return DefaultDuration;
|
||||
}
|
||||
|
||||
foreach (var rule in Rules)
|
||||
{
|
||||
if (weight >= rule.MinWeight)
|
||||
{
|
||||
return rule.Duration;
|
||||
}
|
||||
}
|
||||
|
||||
return DefaultDuration;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record DamperRuleState(double MinWeight, TimeSpan Duration);
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
internal sealed class VexWorkerHostedService : BackgroundService
|
||||
{
|
||||
private readonly IOptions<VexWorkerOptions> _options;
|
||||
private readonly IVexProviderRunner _runner;
|
||||
private readonly ILogger<VexWorkerHostedService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public VexWorkerHostedService(
|
||||
IOptions<VexWorkerOptions> options,
|
||||
IVexProviderRunner runner,
|
||||
ILogger<VexWorkerHostedService> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var schedules = _options.Value.ResolveSchedules();
|
||||
if (schedules.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("Excititor worker has no configured provider schedules; the service will remain idle.");
|
||||
await Task.CompletedTask;
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Excititor worker starting with {ProviderCount} provider schedule(s).", schedules.Count);
|
||||
|
||||
var tasks = new List<Task>(schedules.Count);
|
||||
foreach (var schedule in schedules)
|
||||
{
|
||||
tasks.Add(RunScheduleAsync(schedule, stoppingToken));
|
||||
}
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (schedule.InitialDelay > TimeSpan.Zero)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Provider {ProviderId} initial delay of {InitialDelay} before first execution.",
|
||||
schedule.ProviderId,
|
||||
schedule.InitialDelay);
|
||||
|
||||
await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
using var timer = new PeriodicTimer(schedule.Interval);
|
||||
do
|
||||
{
|
||||
var startedAt = _timeProvider.GetUtcNow();
|
||||
_logger.LogInformation(
|
||||
"Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.",
|
||||
schedule.ProviderId,
|
||||
startedAt,
|
||||
schedule.Interval);
|
||||
|
||||
try
|
||||
{
|
||||
await _runner.RunAsync(schedule, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
var elapsed = completedAt - startedAt;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).",
|
||||
schedule.ProviderId,
|
||||
completedAt,
|
||||
elapsed);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId);
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Provider {ProviderId} run failed: {Message}",
|
||||
schedule.ProviderId,
|
||||
ex.Message);
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Scheduling;
|
||||
|
||||
internal sealed record VexWorkerSchedule(string ProviderId, TimeSpan Interval, TimeSpan InitialDelay, VexConnectorSettings Settings);
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Signature;
|
||||
|
||||
internal sealed class VerifyingVexRawDocumentSink : IVexRawDocumentSink
|
||||
{
|
||||
private readonly IVexRawStore _inner;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
|
||||
public VerifyingVexRawDocumentSink(IVexRawStore inner, IVexSignatureVerifier signatureVerifier)
|
||||
{
|
||||
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
}
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var signatureMetadata = await _signatureVerifier.VerifyAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
var enrichedDocument = signatureMetadata is null
|
||||
? document
|
||||
: document with { Metadata = EnrichMetadata(document.Metadata, signatureMetadata) };
|
||||
|
||||
await _inner.StoreAsync(enrichedDocument, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> EnrichMetadata(
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
VexSignatureMetadata signature)
|
||||
{
|
||||
var builder = metadata is null
|
||||
? ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal)
|
||||
: metadata.ToBuilder();
|
||||
|
||||
builder["signature.present"] = "true";
|
||||
builder["signature.verified"] = "true";
|
||||
builder["vex.signature.type"] = signature.Type;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Subject))
|
||||
{
|
||||
builder["vex.signature.subject"] = signature.Subject!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Issuer))
|
||||
{
|
||||
builder["vex.signature.issuer"] = signature.Issuer!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.KeyId))
|
||||
{
|
||||
builder["vex.signature.keyId"] = signature.KeyId!;
|
||||
}
|
||||
|
||||
if (signature.VerifiedAt is not null)
|
||||
{
|
||||
builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.TransparencyLogReference))
|
||||
{
|
||||
builder["vex.signature.transparencyLogReference"] = signature.TransparencyLogReference!;
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Aoc;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Models;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Signature;
|
||||
|
||||
/// <summary>
|
||||
/// Enforces checksum validation and records signature verification metadata.
|
||||
/// </summary>
|
||||
internal sealed class WorkerSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
private static readonly Meter Meter = new("StellaOps.Excititor.Worker", "1.0");
|
||||
private static readonly Counter<long> SignatureVerificationCounter = Meter.CreateCounter<long>(
|
||||
"ingestion_signature_verified_total",
|
||||
description: "Counts signature and checksum verification results for Excititor worker ingestion.");
|
||||
|
||||
private readonly ILogger<WorkerSignatureVerifier> _logger;
|
||||
private readonly IVexAttestationVerifier? _attestationVerifier;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions StatementSerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
|
||||
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) },
|
||||
};
|
||||
|
||||
public WorkerSignatureVerifier(
|
||||
ILogger<WorkerSignatureVerifier> logger,
|
||||
IVexAttestationVerifier? attestationVerifier = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_attestationVerifier = attestationVerifier;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
var expectedDigest = NormalizeDigest(document.Digest);
|
||||
var computedDigest = ComputeDigest(document.Content.Span);
|
||||
|
||||
if (!string.Equals(expectedDigest, computedDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
RecordVerification(document.ProviderId, metadata, "fail");
|
||||
_logger.LogError(
|
||||
"Checksum mismatch for provider {ProviderId} (expected={ExpectedDigest}, computed={ComputedDigest}, uri={SourceUri})",
|
||||
document.ProviderId,
|
||||
expectedDigest,
|
||||
computedDigest,
|
||||
document.SourceUri);
|
||||
|
||||
var violation = AocViolation.Create(
|
||||
AocViolationCode.SignatureInvalid,
|
||||
"/upstream/content_hash",
|
||||
$"Content hash mismatch. Expected {expectedDigest}, computed {computedDigest}.");
|
||||
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
|
||||
VexSignatureMetadata? signatureMetadata = null;
|
||||
if (document.Format == VexDocumentFormat.OciAttestation && _attestationVerifier is not null)
|
||||
{
|
||||
signatureMetadata = await VerifyAttestationAsync(document, metadata, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
signatureMetadata ??= ExtractSignatureMetadata(metadata);
|
||||
var resultLabel = signatureMetadata is null ? "skipped" : "ok";
|
||||
RecordVerification(document.ProviderId, metadata, resultLabel);
|
||||
|
||||
if (resultLabel == "skipped")
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Signature verification skipped for provider {ProviderId} (no signature metadata).",
|
||||
document.ProviderId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Signature metadata recorded for provider {ProviderId} (type={SignatureType}, subject={Subject}, issuer={Issuer}).",
|
||||
document.ProviderId,
|
||||
signatureMetadata!.Type,
|
||||
signatureMetadata.Subject ?? "<unknown>",
|
||||
signatureMetadata.Issuer ?? "<unknown>");
|
||||
}
|
||||
|
||||
return signatureMetadata;
|
||||
}
|
||||
|
||||
private async ValueTask<VexSignatureMetadata?> VerifyAttestationAsync(
|
||||
VexRawDocument document,
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var envelopeJson = Encoding.UTF8.GetString(document.Content.Span);
|
||||
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, EnvelopeSerializerOptions)
|
||||
?? throw new InvalidOperationException("DSSE envelope deserialized to null.");
|
||||
|
||||
var payloadBytes = Convert.FromBase64String(envelope.Payload);
|
||||
var statement = JsonSerializer.Deserialize<VexInTotoStatement>(payloadBytes, StatementSerializerOptions)
|
||||
?? throw new InvalidOperationException("DSSE statement deserialized to null.");
|
||||
|
||||
if (statement.Subject is null || statement.Subject.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("DSSE statement subject is missing.");
|
||||
}
|
||||
|
||||
var predicate = statement.Predicate ?? throw new InvalidOperationException("DSSE predicate is missing.");
|
||||
var request = BuildAttestationRequest(statement, predicate);
|
||||
var attestationMetadata = BuildAttestationMetadata(statement, envelope, metadata);
|
||||
|
||||
var verificationRequest = new VexAttestationVerificationRequest(
|
||||
request,
|
||||
attestationMetadata,
|
||||
envelopeJson);
|
||||
|
||||
var verification = await _attestationVerifier!
|
||||
.VerifyAsync(verificationRequest, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (!verification.IsValid)
|
||||
{
|
||||
var diagnostics = string.Join(", ", verification.Diagnostics.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
_logger.LogError(
|
||||
"Attestation verification failed for provider {ProviderId} (uri={SourceUri}) diagnostics={Diagnostics}",
|
||||
document.ProviderId,
|
||||
document.SourceUri,
|
||||
diagnostics);
|
||||
|
||||
var violation = AocViolation.Create(
|
||||
AocViolationCode.SignatureInvalid,
|
||||
"/upstream/signature",
|
||||
"Attestation verification failed.");
|
||||
|
||||
RecordVerification(document.ProviderId, metadata, "fail");
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Attestation verification succeeded for provider {ProviderId} (predicate={PredicateType}, subject={Subject}).",
|
||||
document.ProviderId,
|
||||
attestationMetadata.PredicateType,
|
||||
statement.Subject[0].Name ?? "<unknown>");
|
||||
|
||||
return BuildSignatureMetadata(statement, metadata, attestationMetadata, verification.Diagnostics);
|
||||
}
|
||||
catch (ExcititorAocGuardException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to verify attestation for provider {ProviderId} (uri={SourceUri})",
|
||||
document.ProviderId,
|
||||
document.SourceUri);
|
||||
|
||||
var violation = AocViolation.Create(
|
||||
AocViolationCode.SignatureInvalid,
|
||||
"/upstream/signature",
|
||||
$"Attestation verification encountered an error: {ex.Message}");
|
||||
|
||||
RecordVerification(document.ProviderId, metadata, "fail");
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
}
|
||||
|
||||
private VexAttestationRequest BuildAttestationRequest(VexInTotoStatement statement, VexAttestationPredicate predicate)
|
||||
{
|
||||
var subject = statement.Subject!.First();
|
||||
var exportId = predicate.ExportId ?? subject.Name ?? throw new InvalidOperationException("Attestation export ID missing.");
|
||||
var querySignature = new VexQuerySignature(predicate.QuerySignature ?? throw new InvalidOperationException("Attestation query signature missing."));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(predicate.ArtifactAlgorithm) || string.IsNullOrWhiteSpace(predicate.ArtifactDigest))
|
||||
{
|
||||
throw new InvalidOperationException("Attestation artifact metadata is incomplete.");
|
||||
}
|
||||
|
||||
var artifact = new VexContentAddress(predicate.ArtifactAlgorithm, predicate.ArtifactDigest);
|
||||
|
||||
var sourceProviders = predicate.SourceProviders?.ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var metadata = predicate.Metadata?.ToImmutableDictionary(StringComparer.Ordinal) ?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
return new VexAttestationRequest(
|
||||
exportId,
|
||||
querySignature,
|
||||
artifact,
|
||||
predicate.Format,
|
||||
predicate.CreatedAt,
|
||||
sourceProviders,
|
||||
metadata);
|
||||
}
|
||||
|
||||
private VexAttestationMetadata BuildAttestationMetadata(
|
||||
VexInTotoStatement statement,
|
||||
DsseEnvelope envelope,
|
||||
ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
VexRekorReference? rekor = null;
|
||||
if (metadata.TryGetValue("vex.signature.transparencyLogReference", out var rekorValue) && !string.IsNullOrWhiteSpace(rekorValue))
|
||||
{
|
||||
rekor = new VexRekorReference("0.1", rekorValue);
|
||||
}
|
||||
|
||||
DateTimeOffset signedAt;
|
||||
if (metadata.TryGetValue("vex.signature.verifiedAt", out var signedAtRaw)
|
||||
&& DateTimeOffset.TryParse(signedAtRaw, out var parsedSignedAt))
|
||||
{
|
||||
signedAt = parsedSignedAt;
|
||||
}
|
||||
else
|
||||
{
|
||||
signedAt = _timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
return new VexAttestationMetadata(
|
||||
statement.PredicateType ?? "https://stella-ops.org/attestations/vex-export",
|
||||
rekor,
|
||||
VexDsseBuilder.ComputeEnvelopeDigest(envelope),
|
||||
signedAt);
|
||||
}
|
||||
|
||||
private VexSignatureMetadata BuildSignatureMetadata(
|
||||
VexInTotoStatement statement,
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
VexAttestationMetadata attestationMetadata,
|
||||
ImmutableDictionary<string, string> diagnostics)
|
||||
{
|
||||
metadata.TryGetValue("vex.signature.type", out var type);
|
||||
metadata.TryGetValue("vex.provenance.cosign.subject", out var subject);
|
||||
metadata.TryGetValue("vex.provenance.cosign.issuer", out var issuer);
|
||||
metadata.TryGetValue("vex.signature.keyId", out var keyId);
|
||||
metadata.TryGetValue("vex.signature.transparencyLogReference", out var transparencyReference);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
type = statement.PredicateType?.Contains("attest", StringComparison.OrdinalIgnoreCase) == true
|
||||
? "cosign"
|
||||
: "attestation";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subject) && statement.Subject is { Count: > 0 })
|
||||
{
|
||||
subject = statement.Subject[0].Name;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(transparencyReference) && attestationMetadata.Rekor is not null)
|
||||
{
|
||||
transparencyReference = attestationMetadata.Rekor.Location;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(issuer)
|
||||
&& diagnostics.TryGetValue("verification.issuer", out var diagnosticIssuer)
|
||||
&& !string.IsNullOrWhiteSpace(diagnosticIssuer))
|
||||
{
|
||||
issuer = diagnosticIssuer;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(keyId)
|
||||
&& diagnostics.TryGetValue("verification.keyId", out var diagnosticKeyId)
|
||||
&& !string.IsNullOrWhiteSpace(diagnosticKeyId))
|
||||
{
|
||||
keyId = diagnosticKeyId;
|
||||
}
|
||||
|
||||
var verifiedAt = attestationMetadata.SignedAt ?? _timeProvider.GetUtcNow();
|
||||
|
||||
return new VexSignatureMetadata(
|
||||
type!,
|
||||
subject,
|
||||
issuer,
|
||||
keyId,
|
||||
verifiedAt,
|
||||
transparencyReference);
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
|
||||
? digest
|
||||
: $"sha256:{digest}";
|
||||
}
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> content)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
if (!SHA256.TryHashData(content, buffer, out _))
|
||||
{
|
||||
var hash = SHA256.HashData(content.ToArray());
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexSignatureMetadata? ExtractSignatureMetadata(ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (!metadata.TryGetValue("vex.signature.type", out var type) || string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.TryGetValue("vex.signature.subject", out var subject);
|
||||
metadata.TryGetValue("vex.signature.issuer", out var issuer);
|
||||
metadata.TryGetValue("vex.signature.keyId", out var keyId);
|
||||
metadata.TryGetValue("vex.signature.verifiedAt", out var verifiedAtRaw);
|
||||
metadata.TryGetValue("vex.signature.transparencyLogReference", out var tlog);
|
||||
|
||||
DateTimeOffset? verifiedAt = null;
|
||||
if (!string.IsNullOrWhiteSpace(verifiedAtRaw) && DateTimeOffset.TryParse(verifiedAtRaw, out var parsed))
|
||||
{
|
||||
verifiedAt = parsed;
|
||||
}
|
||||
|
||||
return new VexSignatureMetadata(type, subject, issuer, keyId, verifiedAt, tlog);
|
||||
}
|
||||
|
||||
private static void RecordVerification(string providerId, ImmutableDictionary<string, string> metadata, string result)
|
||||
{
|
||||
var tags = new List<KeyValuePair<string, object?>>(3)
|
||||
{
|
||||
new("source", providerId),
|
||||
new("result", result),
|
||||
};
|
||||
|
||||
if (!metadata.TryGetValue("tenant", out var tenant) || string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
tenant = "tenant-default";
|
||||
}
|
||||
|
||||
tags.Add(new KeyValuePair<string, object?>("tenant", tenant));
|
||||
|
||||
SignatureVerificationCounter.Add(1, tags.ToArray());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk.Worker">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0-rc.2.25502.107" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
19
src/Excititor/StellaOps.Excititor.Worker/TASKS.md
Normal file
19
src/Excititor/StellaOps.Excititor.Worker/TASKS.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# TASKS — Epic 1: Aggregation-Only Contract
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|---|
|
||||
| EXCITITOR-WORKER-AOC-19-001 `Raw pipeline rewiring` | DONE (2025-10-31) | Excititor Worker Guild | EXCITITOR-CORE-AOC-19-001 | Update ingest pipelines to persist upstream documents directly into `vex_raw` via the new repository guard. Remove consensus/folding hooks and ensure retries respect append-only semantics. |
|
||||
> 2025-10-31: Worker now runs in raw-only mode; `DefaultVexProviderRunner` no longer normalizes or schedules consensus refresh and logs document counts only. Tests updated to assert the normalizer is not invoked.
|
||||
| EXCITITOR-WORKER-AOC-19-002 `Signature & checksum enforcement` | DONE (2025-10-28) | Excititor Worker Guild | EXCITITOR-WORKER-AOC-19-001 | Add signature verification + checksum computation before writes, capturing failure reasons mapped to `ERR_AOC_005`, with structured logs/metrics for verification results. |
|
||||
> 2025-10-28: Resuming implementation to finish attestation metadata plumbing, wiring into runner, and tests (`WorkerSignatureVerifier`, `DefaultVexProviderRunner`).
|
||||
> 2025-10-28: Attestation verification now enriches signature metadata & runner tests cover DSSE path; metrics unchanged.
|
||||
> 2025-10-31: Worker wraps raw sink with checksum enforcement. Digest mismatches raise `ERR_AOC_005`, signature metadata is captured when present, and `ingestion_signature_verified_total` is emitted (`result=ok|fail|skipped`).
|
||||
| EXCITITOR-WORKER-AOC-19-003 `Deterministic batching tests` | DONE (2025-10-28) | QA Guild | EXCITITOR-WORKER-AOC-19-001 | Extend worker integration tests to replay large VEX batches ensuring idempotent upserts, supersedes chaining, and guard enforcement across restart scenarios. |
|
||||
> 2025-10-28: Added Mongo-backed integration suite validating large batch replay, guard-triggered failures, and restart idempotency (`DefaultVexProviderRunnerIntegrationTests`). Worker unit tests now exercise the verifying sink path, and `dotnet test` passes after attestation envelope fixes.
|
||||
|
||||
## Orchestrator Dashboard
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| EXCITITOR-ORCH-32-001 `Worker SDK adoption` | TODO | Excititor Worker Guild | ORCH-SVC-32-005, WORKER-GO-32-001, WORKER-PY-32-001 | Integrate orchestrator worker SDK in Excititor ingestion jobs, emit heartbeats/progress/artifact hashes, and register source metadata. |
|
||||
| EXCITITOR-ORCH-33-001 `Control compliance` | TODO | Excititor Worker Guild | EXCITITOR-ORCH-32-001, ORCH-SVC-33-001, ORCH-SVC-33-002 | Honor orchestrator pause/throttle/retry actions, classify error outputs, and persist restart checkpoints. |
|
||||
| EXCITITOR-ORCH-34-001 `Backfill & circuit breaker` | TODO | Excititor Worker Guild | EXCITITOR-ORCH-33-001, ORCH-SVC-33-003, ORCH-SVC-34-001 | Implement orchestrator-driven backfills, apply circuit breaker reset rules, and ensure artifact dedupe alignment. |
|
||||
Reference in New Issue
Block a user