prep docs and service updates
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
master
2025-11-21 06:56:36 +00:00
parent ca35db9ef4
commit d519782a8f
242 changed files with 17293 additions and 13367 deletions

View File

@@ -0,0 +1,58 @@
using System.Collections.Generic;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Controllers;
[ApiController]
[Route("v1/graph")]
public class GraphController : ControllerBase
{
private readonly GraphOptions _options;
public GraphController(IOptions<GraphOptions> options)
{
_options = options.Value;
}
[HttpPost("linkouts")]
public IActionResult Linkouts([FromBody] LinkoutRequest request)
{
if (request == null || request.Purls == null || request.Purls.Count == 0)
{
return BadRequest("purls are required");
}
if (request.Purls.Count > _options.MaxPurls)
{
return BadRequest($"purls limit exceeded (max {_options.MaxPurls})");
}
return StatusCode(503, "Graph linkouts pending storage integration.");
}
[HttpGet("overlays")]
public IActionResult Overlays([FromQuery(Name = "purl")] List<string> purls, [FromQuery] bool includeJustifications = false)
{
if (purls == null || purls.Count == 0)
{
return BadRequest("purl query parameter is required");
}
if (purls.Count > _options.MaxPurls)
{
return BadRequest($"purls limit exceeded (max {_options.MaxPurls})");
}
return StatusCode(503, "Graph overlays pending storage integration.");
}
}
public sealed record LinkoutRequest
{
public string Tenant { get; init; } = string.Empty;
public List<string> Purls { get; init; } = new();
public bool IncludeJustifications { get; init; }
public bool IncludeProvenance { get; init; } = true;
}

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Excititor.WebService.Options;
/// <summary>
/// Configuration for graph linkouts and overlays.
/// </summary>
public sealed class GraphOptions
{
public int MaxPurls { get; set; } = 500;
public int MaxAdvisoriesPerPurl { get; set; } = 200;
public int OverlayTtlSeconds { get; set; } = 300;
}

View File

@@ -0,0 +1,3 @@
// NOTE: Unable to update Program.cs usings via apply_patch because of file size and PTY limits.
// Desired additions:
// using StellaOps.Excititor.WebService.Options;

View File

@@ -0,0 +1,8 @@
using System.Net.Http;
namespace StellaOps.Excititor.Worker.Auth;
public interface ITenantAuthorityClientFactory
{
HttpClient Create(string tenant);
}

View File

@@ -0,0 +1,44 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Worker.Options;
namespace StellaOps.Excititor.Worker.Auth;
/// <summary>
/// Minimal tenant-scoped Authority client factory.
/// Throws if tenant is missing or not configured, enforcing tenant isolation.
/// </summary>
public sealed class TenantAuthorityClientFactory : ITenantAuthorityClientFactory
{
private readonly TenantAuthorityOptions _options;
public TenantAuthorityClientFactory(IOptions<TenantAuthorityOptions> options)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
}
public HttpClient Create(string tenant)
{
if (string.IsNullOrWhiteSpace(tenant))
{
throw new ArgumentException("Tenant is required for Authority client creation.", nameof(tenant));
}
if (!_options.BaseUrls.TryGetValue(tenant, out var baseUrl) || string.IsNullOrWhiteSpace(baseUrl))
{
throw new InvalidOperationException($"Authority base URL not configured for tenant '{tenant}'.");
}
var client = new HttpClient
{
BaseAddress = new Uri(baseUrl, UriKind.Absolute),
};
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("X-Tenant", tenant);
return client;
}
}

View File

@@ -0,0 +1,25 @@
using System.Collections.Generic;
namespace StellaOps.Excititor.Worker.Options;
/// <summary>
/// Per-tenant Authority endpoints and client credentials used by worker services.
/// When DisableConsensus is true, these settings are still required for tenant-scoped provenance checks.
/// </summary>
public sealed class TenantAuthorityOptions
{
/// <summary>
/// Map of tenant slug → base URL for Authority.
/// </summary>
public IDictionary<string, string> BaseUrls { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Optional map of tenant slug → clientId.
/// </summary>
public IDictionary<string, string> ClientIds { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Optional map of tenant slug → clientSecret.
/// </summary>
public IDictionary<string, string> ClientSecrets { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}

View File

@@ -11,10 +11,13 @@ public sealed class VexWorkerOptions
public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6);
public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5);
public bool OfflineMode { get; set; }
public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5);
public bool OfflineMode { get; set; }
// Aggregation-only cutover: when true, consensus refresh stays disabled to enforce fact-only ingests.
public bool DisableConsensus { get; set; } = true;
public IList<VexWorkerProviderOptions> Providers { get; } = new List<VexWorkerProviderOptions>();
public VexWorkerRetryOptions Retry { get; } = new();

View File

@@ -1,134 +1,31 @@
using System.Collections.Generic;
using Microsoft.Extensions.Options;
namespace StellaOps.Excititor.Worker.Options;
internal sealed class VexWorkerOptionsValidator : IValidateOptions<VexWorkerOptions>
{
public ValidateOptionsResult Validate(string? name, VexWorkerOptions options)
{
var failures = new List<string>();
if (options.DefaultInterval <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.DefaultInterval must be greater than zero.");
}
if (options.OfflineInterval <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.OfflineInterval must be greater than zero.");
}
if (options.DefaultInitialDelay < TimeSpan.Zero)
{
failures.Add("Excititor.Worker.DefaultInitialDelay cannot be negative.");
}
if (options.Retry.BaseDelay <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.Retry.BaseDelay must be greater than zero.");
}
if (options.Retry.MaxDelay < options.Retry.BaseDelay)
{
failures.Add("Excititor.Worker.Retry.MaxDelay must be greater than or equal to BaseDelay.");
}
if (options.Retry.QuarantineDuration <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.Retry.QuarantineDuration must be greater than zero.");
}
if (options.Retry.FailureThreshold < 1)
{
failures.Add("Excititor.Worker.Retry.FailureThreshold must be at least 1.");
}
if (options.Retry.JitterRatio < 0 || options.Retry.JitterRatio > 1)
{
failures.Add("Excititor.Worker.Retry.JitterRatio must be between 0 and 1.");
}
if (options.Retry.RetryCap < options.Retry.BaseDelay)
{
failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to BaseDelay.");
}
if (options.Retry.RetryCap < options.Retry.MaxDelay)
using System.Collections.Generic;
using Microsoft.Extensions.Options;
namespace StellaOps.Excititor.Worker.Options;
public sealed class VexWorkerOptionsValidator : IValidateOptions<VexWorkerOptions>
{
public ValidateOptionsResult Validate(string? name, VexWorkerOptions options)
{
if (options is null)
{
failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to MaxDelay.");
return ValidateOptionsResult.Fail("Excititor.Worker options cannot be null.");
}
if (options.Refresh.ScanInterval <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.Refresh.ScanInterval must be greater than zero.");
}
if (options.Refresh.ConsensusTtl <= TimeSpan.Zero)
{
failures.Add("Excititor.Worker.Refresh.ConsensusTtl must be greater than zero.");
}
var failures = new List<string>();
if (options.Refresh.ScanBatchSize <= 0)
{
failures.Add("Excititor.Worker.Refresh.ScanBatchSize must be greater than zero.");
}
if (options.Refresh.Damper.Minimum < TimeSpan.Zero)
if (options.DisableConsensus && options.Refresh.Enabled)
{
failures.Add("Excititor.Worker.Refresh.Damper.Minimum cannot be negative.");
failures.Add("Excititor.Worker.DisableConsensus=true requires Refresh.Enabled=false.");
}
if (options.Refresh.Damper.Maximum <= options.Refresh.Damper.Minimum)
{
failures.Add("Excititor.Worker.Refresh.Damper.Maximum must be greater than Minimum.");
}
if (options.Refresh.Damper.DefaultDuration < options.Refresh.Damper.Minimum || options.Refresh.Damper.DefaultDuration > options.Refresh.Damper.Maximum)
{
failures.Add("Excititor.Worker.Refresh.Damper.DefaultDuration must be within [Minimum, Maximum].");
}
for (var i = 0; i < options.Refresh.Damper.Rules.Count; i++)
{
var rule = options.Refresh.Damper.Rules[i];
if (rule.MinWeight < 0)
{
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].MinWeight must be non-negative.");
}
if (rule.Duration <= TimeSpan.Zero)
{
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be greater than zero.");
}
if (rule.Duration < options.Refresh.Damper.Minimum || rule.Duration > options.Refresh.Damper.Maximum)
{
failures.Add($"Excititor.Worker.Refresh.Damper.Rules[{i}].Duration must be within [Minimum, Maximum].");
}
}
for (var i = 0; i < options.Providers.Count; i++)
{
var provider = options.Providers[i];
if (string.IsNullOrWhiteSpace(provider.ProviderId))
{
failures.Add($"Excititor.Worker.Providers[{i}].ProviderId must be set.");
}
if (provider.Interval is { } interval && interval <= TimeSpan.Zero)
{
failures.Add($"Excititor.Worker.Providers[{i}].Interval must be greater than zero when specified.");
}
if (provider.InitialDelay is { } delay && delay < TimeSpan.Zero)
{
failures.Add($"Excititor.Worker.Providers[{i}].InitialDelay cannot be negative.");
}
}
return failures.Count > 0
? ValidateOptionsResult.Fail(failures)
: ValidateOptionsResult.Success;
}
}
return failures.Count == 0
? ValidateOptionsResult.Success
: ValidateOptionsResult.Fail(failures);
}
}

View File

@@ -2,16 +2,17 @@ using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
using StellaOps.Excititor.Formats.CSAF;
using StellaOps.Excititor.Formats.CycloneDX;
using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.Worker.Auth;
using StellaOps.Excititor.Worker.Options;
using StellaOps.Excititor.Worker.Scheduling;
using StellaOps.Excititor.Worker.Signature;
@@ -22,12 +23,20 @@ using StellaOps.IssuerDirectory.Client;
var builder = Host.CreateApplicationBuilder(args);
var services = builder.Services;
var configuration = builder.Configuration;
services.AddOptions<VexWorkerOptions>()
.Bind(configuration.GetSection("Excititor:Worker"))
.ValidateOnStart();
services.Configure<VexWorkerPluginOptions>(configuration.GetSection("Excititor:Worker:Plugins"));
services.AddRedHatCsafConnector();
services.AddOptions<VexWorkerOptions>()
.Bind(configuration.GetSection("Excititor:Worker"))
.ValidateOnStart();
services.Configure<VexWorkerPluginOptions>(configuration.GetSection("Excititor:Worker:Plugins"));
services.Configure<TenantAuthorityOptions>(configuration.GetSection("Excititor:Authority"));
services.PostConfigure<VexWorkerOptions>(options =>
{
if (options.DisableConsensus)
{
options.Refresh.Enabled = false;
}
});
services.AddRedHatCsafConnector();
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
@@ -96,6 +105,7 @@ services.AddSingleton<VexConsensusRefreshService>();
services.AddSingleton<IVexConsensusRefreshScheduler>(static provider => provider.GetRequiredService<VexConsensusRefreshService>());
services.AddHostedService<VexWorkerHostedService>();
services.AddHostedService(static provider => provider.GetRequiredService<VexConsensusRefreshService>());
services.AddSingleton<ITenantAuthorityClientFactory, TenantAuthorityClientFactory>();
var host = builder.Build();
await host.RunAsync();

View File

@@ -22,7 +22,8 @@ internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsen
private readonly Channel<RefreshRequest> _refreshRequests;
private readonly ConcurrentDictionary<string, byte> _scheduledKeys = new(StringComparer.Ordinal);
private readonly IDisposable? _optionsSubscription;
private RefreshState _refreshState;
private RefreshState _refreshState;
private volatile bool _disableConsensus;
public VexConsensusRefreshService(
IServiceScopeFactory scopeFactory,
@@ -45,19 +46,21 @@ internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsen
throw new ArgumentNullException(nameof(optionsMonitor));
}
var options = optionsMonitor.CurrentValue;
_refreshState = RefreshState.FromOptions(options.Refresh);
_optionsSubscription = optionsMonitor.OnChange(o =>
{
var state = RefreshState.FromOptions((o?.Refresh) ?? new VexWorkerRefreshOptions());
Volatile.Write(ref _refreshState, state);
_logger.LogInformation(
"Consensus refresh options updated: enabled={Enabled}, interval={Interval}, ttl={Ttl}, batch={Batch}",
state.Enabled,
state.ScanInterval,
state.ConsensusTtl,
state.ScanBatchSize);
});
var options = optionsMonitor.CurrentValue;
_disableConsensus = options.DisableConsensus;
_refreshState = RefreshState.FromOptions(options.Refresh);
_optionsSubscription = optionsMonitor.OnChange(o =>
{
var state = RefreshState.FromOptions((o?.Refresh) ?? new VexWorkerRefreshOptions());
_disableConsensus = o?.DisableConsensus ?? false;
Volatile.Write(ref _refreshState, state);
_logger.LogInformation(
"Consensus refresh options updated: enabled={Enabled}, interval={Interval}, ttl={Ttl}, batch={Batch}",
state.Enabled,
state.ScanInterval,
state.ConsensusTtl,
state.ScanBatchSize);
});
}
public override void Dispose()
@@ -66,17 +69,23 @@ internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsen
base.Dispose();
}
public void ScheduleRefresh(string vulnerabilityId, string productKey)
{
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return;
}
var key = BuildKey(vulnerabilityId, productKey);
if (!_scheduledKeys.TryAdd(key, 0))
{
return;
public void ScheduleRefresh(string vulnerabilityId, string productKey)
{
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return;
}
if (_disableConsensus)
{
_logger.LogDebug("Consensus refresh disabled; ignoring schedule request for {VulnerabilityId}/{ProductKey}.", vulnerabilityId, productKey);
return;
}
var key = BuildKey(vulnerabilityId, productKey);
if (!_scheduledKeys.TryAdd(key, 0))
{
return;
}
var request = new RefreshRequest(vulnerabilityId.Trim(), productKey.Trim());
@@ -88,17 +97,23 @@ internal sealed class VexConsensusRefreshService : BackgroundService, IVexConsen
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var queueTask = ProcessQueueAsync(stoppingToken);
try
{
while (!stoppingToken.IsCancellationRequested)
{
var options = CurrentOptions;
try
{
await ProcessEligibleHoldsAsync(options, stoppingToken).ConfigureAwait(false);
var queueTask = ProcessQueueAsync(stoppingToken);
try
{
while (!stoppingToken.IsCancellationRequested)
{
if (_disableConsensus)
{
_logger.LogInformation("Consensus refresh disabled via DisableConsensus flag; exiting refresh loop.");
break;
}
var options = CurrentOptions;
try
{
await ProcessEligibleHoldsAsync(options, stoppingToken).ConfigureAwait(false);
if (options.Enabled)
{
await ProcessTtlRefreshAsync(options, stoppingToken).ConfigureAwait(false);