Rename Vexer to Excititor

This commit is contained in:
2025-10-18 20:00:46 +03:00
parent fbd1826ef3
commit 7e1b10d3b2
263 changed files with 848 additions and 848 deletions

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for Ubuntu CSAF advisories (USN VEX data), managing discovery, incremental pulls, and raw document persistence.
## Scope
- Ubuntu CSAF metadata discovery, release channel awareness, and pagination handling.
- HTTP client with retries/backoff, checksum validation, and deduplication.
- Mapping Ubuntu identifiers (USN numbers, package metadata) into connector metadata for downstream policy.
- Emitting trust configuration (GPG fingerprints, cosign options) for policy weighting.
## Participants
- Worker schedules regular pulls; WebService can initiate manual ingest/resume.
- CSAF normalizer converts raw documents into claims.
- Policy engine leverages connector-supplied trust metadata.
## Interfaces & contracts
- Implements `IVexConnector`, using shared abstractions for HTTP/resume markers and telemetry.
- Provides options for release channels (stable/LTS) and offline seed bundles.
## In/Out of scope
In: data fetching, metadata mapping, raw persistence, trust hints.
Out: normalization/export, storage internals, attestation.
## Observability & security expectations
- Log release window fetch metrics, rate limits, and deduplication stats; mask secrets.
- Emit counters for newly ingested vs unchanged USNs and quota usage.
## Tests
- Connector tests with mocked Ubuntu CSAF endpoints will live in `../StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests`.

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
public sealed class UbuntuConnectorOptions
{
public const string HttpClientName = "excititor.connector.ubuntu.catalog";
/// <summary>
/// Root index that lists Ubuntu CSAF channels.
/// </summary>
public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json");
/// <summary>
/// Channels to include (e.g. stable, esm, lts).
/// </summary>
public IList<string> Channels { get; } = new List<string> { "stable" };
/// <summary>
/// Duration to cache discovery metadata.
/// </summary>
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(4);
/// <summary>
/// Prefer offline snapshot when available.
/// </summary>
public bool PreferOfflineSnapshot { get; set; }
/// <summary>
/// Optional file path for offline index snapshot.
/// </summary>
public string? OfflineSnapshotPath { get; set; }
/// <summary>
/// Controls persistence of network responses to <see cref="OfflineSnapshotPath"/>.
/// </summary>
public bool PersistOfflineSnapshot { get; set; } = true;
public void Validate(IFileSystem? fileSystem = null)
{
if (IndexUri is null || !IndexUri.IsAbsoluteUri)
{
throw new InvalidOperationException("IndexUri must be an absolute URI.");
}
if (IndexUri.Scheme is not ("http" or "https"))
{
throw new InvalidOperationException("IndexUri must use HTTP or HTTPS.");
}
if (Channels.Count == 0)
{
throw new InvalidOperationException("At least one channel must be specified.");
}
for (var i = Channels.Count - 1; i >= 0; i--)
{
if (string.IsNullOrWhiteSpace(Channels[i]))
{
Channels.RemoveAt(i);
}
}
if (Channels.Count == 0)
{
throw new InvalidOperationException("Channel names cannot be empty.");
}
if (MetadataCacheDuration <= TimeSpan.Zero)
{
throw new InvalidOperationException("MetadataCacheDuration must be positive.");
}
if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled.");
}
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
public sealed class UbuntuConnectorOptionsValidator : IVexConnectorOptionsValidator<UbuntuConnectorOptions>
{
private readonly IFileSystem _fileSystem;
public UbuntuConnectorOptionsValidator(IFileSystem fileSystem)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
}
public void Validate(VexConnectorDescriptor descriptor, UbuntuConnectorOptions options, IList<string> errors)
{
ArgumentNullException.ThrowIfNull(descriptor);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(errors);
try
{
options.Validate(_fileSystem);
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
}
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core;
using System.IO.Abstractions;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.DependencyInjection;
public static class UbuntuConnectorServiceCollectionExtensions
{
public static IServiceCollection AddUbuntuCsafConnector(this IServiceCollection services, Action<UbuntuConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<UbuntuConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddSingleton<IVexConnectorOptionsValidator<UbuntuConnectorOptions>, UbuntuConnectorOptionsValidator>();
services.AddHttpClient(UbuntuConnectorOptions.HttpClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Ubuntu.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<UbuntuCatalogLoader>();
services.AddSingleton<IVexConnector, UbuntuCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,248 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO.Abstractions;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
public sealed class UbuntuCatalogLoader
{
public const string CachePrefix = "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Index";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _memoryCache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<UbuntuCatalogLoader> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
public UbuntuCatalogLoader(
IHttpClientFactory httpClientFactory,
IMemoryCache memoryCache,
IFileSystem fileSystem,
ILogger<UbuntuCatalogLoader> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<UbuntuCatalogResult> LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
options.Validate(_fileSystem);
var cacheKey = CreateCacheKey(options);
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
{
return cached.ToResult(fromCache: true);
}
CacheEntry? entry = null;
if (options.PreferOfflineSnapshot)
{
entry = LoadFromOffline(options);
if (entry is null)
{
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found.");
}
}
else
{
entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false)
?? LoadFromOffline(options);
}
if (entry is null)
{
throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot.");
}
var cacheOptions = new MemoryCacheEntryOptions();
if (entry.MetadataCacheDuration > TimeSpan.Zero)
{
cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration);
}
_memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions);
return entry.ToResult(fromCache: false);
}
finally
{
_semaphore.Release();
}
}
private async Task<CacheEntry?> TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
{
try
{
var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var metadata = ParseMetadata(payload, options.Channels);
var now = _timeProvider.GetUtcNow();
var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false);
PersistSnapshotIfNeeded(options, metadata, now);
return entry;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri);
return null;
}
}
private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options)
{
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return null;
}
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
{
_logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
return null;
}
try
{
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
var snapshot = JsonSerializer.Deserialize<UbuntuCatalogSnapshot>(payload, _serializerOptions);
if (snapshot is null)
{
throw new InvalidOperationException("Offline snapshot payload was empty.");
}
return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath);
return null;
}
}
private UbuntuCatalogMetadata ParseMetadata(string payload, IList<string> channels)
{
if (string.IsNullOrWhiteSpace(payload))
{
throw new InvalidOperationException("Ubuntu index payload was empty.");
}
using var document = JsonDocument.Parse(payload);
var root = document.RootElement;
var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated)
? generated
: _timeProvider.GetUtcNow();
var channelSet = new HashSet<string>(channels, StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array)
{
throw new InvalidOperationException("Ubuntu index did not include a channels array.");
}
var builder = ImmutableArray.CreateBuilder<UbuChannelCatalog>();
foreach (var channelElement in channelsElement.EnumerateArray())
{
var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name))
{
continue;
}
if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri))
{
_logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name);
continue;
}
string? sha256 = null;
if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String)
{
sha256 = shaElement.GetString();
}
DateTimeOffset? lastUpdated = null;
if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated))
{
lastUpdated = updated;
}
builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated));
}
if (builder.Count == 0)
{
throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index.");
}
return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable());
}
private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt)
{
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
{
return;
}
try
{
var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt);
var payload = JsonSerializer.Serialize(snapshot, _serializerOptions);
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
_logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
}
}
private static string CreateCacheKey(UbuntuConnectorOptions options)
=> $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}";
private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot)
{
public bool IsExpired(DateTimeOffset now)
=> MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration;
public UbuntuCatalogResult ToResult(bool fromCache)
=> new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot);
}
private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt);
}
public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray<UbuChannelCatalog> Channels);
public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated);
public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot);

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-UBUNTU-01-001 Ubuntu CSAF discovery & channels|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.|
|EXCITITOR-CONN-UBUNTU-01-002 Incremental fetch & deduplication|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003|TODO Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.|
|EXCITITOR-CONN-UBUNTU-01-003 Trust metadata & provenance|Team Excititor Connectors Ubuntu|EXCITITOR-CONN-UBUNTU-01-002, EXCITITOR-POLICY-01-001|TODO Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.|

View File

@@ -0,0 +1,80 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF;
public sealed class UbuntuCsafConnector : VexConnectorBase
{
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:ubuntu",
kind: VexProviderKind.Distro,
displayName: "Ubuntu CSAF")
{
Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"),
};
private readonly UbuntuCatalogLoader _catalogLoader;
private readonly IEnumerable<IVexConnectorOptionsValidator<UbuntuConnectorOptions>> _validators;
private UbuntuConnectorOptions? _options;
private UbuntuCatalogResult? _catalog;
public UbuntuCsafConnector(
UbuntuCatalogLoader catalogLoader,
IEnumerable<IVexConnectorOptionsValidator<UbuntuConnectorOptions>> validators,
ILogger<UbuntuCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<UbuntuConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
_options = VexConnectorOptionsBinder.Bind(
Descriptor,
settings,
validators: _validators);
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary<string, object?>
{
["channelCount"] = _catalog.Metadata.Channels.Length,
["fromOffline"] = _catalog.FromOfflineSnapshot,
});
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{
throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_catalog is null)
{
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
}
LogConnectorEvent(LogLevel.Debug, "fetch", "Ubuntu CSAF discovery ready; channel catalogs handled in subsequent task.", new Dictionary<string, object?>
{
["since"] = context.Since?.ToString("O"),
});
yield break;
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing.");
public UbuntuCatalogResult? GetCachedCatalog() => _catalog;
}