Rename Vexer to Excititor
This commit is contained in:
master
25
src/StellaOps.Excititor.Connectors.RedHat.CSAF/AGENTS.md
Normal file
25
src/StellaOps.Excititor.Connectors.RedHat.CSAF/AGENTS.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization.
|
||||
## Scope
|
||||
- Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches.
|
||||
- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots.
|
||||
- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage.
|
||||
- Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents.
|
||||
- Emitting structured telemetry and resume markers for incremental pulls.
|
||||
- Supplying Red Hat-specific trust overrides and provenance hints to normalization.
|
||||
## Participants
|
||||
- Worker schedules pulls using this connector; WebService triggers ad-hoc runs.
|
||||
- CSAF normalizer consumes fetched documents to produce claims.
|
||||
- Policy/consensus rely on Red Hat trust metadata captured here.
|
||||
## Interfaces & contracts
|
||||
- Implements `IVexConnector` with Red Hat-specific options (parallelism, token auth if configured).
|
||||
- Uses abstractions from `StellaOps.Excititor.Connectors.Abstractions` for HTTP/resume helpers.
|
||||
## In/Out of scope
|
||||
In: data acquisition, HTTP retries, raw document persistence, provider metadata population.
|
||||
Out: normalization, storage internals, attestation, general connector abstractions (covered elsewhere).
|
||||
## Observability & security expectations
|
||||
- Log provider metadata URL, revision ids, fetch durations; redact tokens.
|
||||
- Emit counters for documents fetched, skipped (304), quarantined.
|
||||
## Tests
|
||||
- Connector harness tests (mock HTTP) and resume regression cases will live in `../StellaOps.Excititor.Connectors.RedHat.CSAF.Tests`.
|
||||
@@ -0,0 +1,104 @@
|
||||
using System.Collections.Generic;
|
||||
using System.IO.Abstractions;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
|
||||
public sealed class RedHatConnectorOptions
|
||||
{
|
||||
public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json");
|
||||
|
||||
/// <summary>
|
||||
/// HTTP client name registered for the connector.
|
||||
/// </summary>
|
||||
public const string HttpClientName = "excititor.connector.redhat";
|
||||
|
||||
/// <summary>
|
||||
/// URI of the CSAF provider metadata document.
|
||||
/// </summary>
|
||||
public Uri MetadataUri { get; set; } = DefaultMetadataUri;
|
||||
|
||||
/// <summary>
|
||||
/// Duration to cache loaded metadata before refreshing.
|
||||
/// </summary>
|
||||
public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Optional file path used to store or source offline metadata snapshots.
|
||||
/// </summary>
|
||||
public string? OfflineSnapshotPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When true, the loader prefers the offline snapshot without attempting a network fetch.
|
||||
/// </summary>
|
||||
public bool PreferOfflineSnapshot { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enables writing fresh metadata responses to <see cref="OfflineSnapshotPath"/>.
|
||||
/// </summary>
|
||||
public bool PersistOfflineSnapshot { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Explicit trust weight override applied to the provider entry.
|
||||
/// </summary>
|
||||
public double TrustWeight { get; set; } = 1.0;
|
||||
|
||||
/// <summary>
|
||||
/// Sigstore/Cosign issuer used to verify CSAF signatures, if published.
|
||||
/// </summary>
|
||||
public string? CosignIssuer { get; set; } = "https://access.redhat.com";
|
||||
|
||||
/// <summary>
|
||||
/// Identity pattern matched against the Cosign certificate subject.
|
||||
/// </summary>
|
||||
public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$";
|
||||
|
||||
/// <summary>
|
||||
/// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts.
|
||||
/// </summary>
|
||||
public IList<string> PgpFingerprints { get; } = new List<string>();
|
||||
|
||||
public void Validate(IFileSystem? fileSystem = null)
|
||||
{
|
||||
if (MetadataUri is null || !MetadataUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("Metadata URI must be absolute.");
|
||||
}
|
||||
|
||||
if (MetadataUri.Scheme is not ("http" or "https"))
|
||||
{
|
||||
throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS.");
|
||||
}
|
||||
|
||||
if (MetadataCacheDuration <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Metadata cache duration must be positive.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath))
|
||||
{
|
||||
var fs = fileSystem ?? new FileSystem();
|
||||
var directory = Path.GetDirectoryName(OfflineSnapshotPath);
|
||||
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
|
||||
{
|
||||
fs.Directory.CreateDirectory(directory);
|
||||
}
|
||||
}
|
||||
|
||||
if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0)
|
||||
{
|
||||
TrustWeight = 1.0;
|
||||
}
|
||||
else if (TrustWeight > 1.0)
|
||||
{
|
||||
TrustWeight = 1.0;
|
||||
}
|
||||
|
||||
if (CosignIssuer is not null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(CosignIdentityPattern))
|
||||
{
|
||||
throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using System.IO.Abstractions;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
|
||||
public static class RedHatConnectorServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action<RedHatConnectorOptions>? configure = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
services.AddOptions<RedHatConnectorOptions>()
|
||||
.Configure(options =>
|
||||
{
|
||||
configure?.Invoke(options);
|
||||
})
|
||||
.PostConfigure(options => options.Validate());
|
||||
|
||||
services.TryAddSingleton<IMemoryCache, MemoryCache>();
|
||||
services.TryAddSingleton<IFileSystem, FileSystem>();
|
||||
|
||||
services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client =>
|
||||
{
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.RedHat/1.0");
|
||||
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler
|
||||
{
|
||||
AutomaticDecompression = DecompressionMethods.All,
|
||||
});
|
||||
|
||||
services.AddSingleton<RedHatProviderMetadataLoader>();
|
||||
services.AddSingleton<IVexConnector, RedHatCsafConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Core;
|
||||
using System.IO.Abstractions;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
|
||||
public sealed class RedHatProviderMetadataLoader
|
||||
{
|
||||
public const string CacheKey = "StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata";
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly ILogger<RedHatProviderMetadataLoader> _logger;
|
||||
private readonly RedHatConnectorOptions _options;
|
||||
private readonly IFileSystem _fileSystem;
|
||||
private readonly JsonSerializerOptions _serializerOptions;
|
||||
private readonly SemaphoreSlim _refreshSemaphore = new(1, 1);
|
||||
|
||||
public RedHatProviderMetadataLoader(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IMemoryCache memoryCache,
|
||||
IOptions<RedHatConnectorOptions> options,
|
||||
ILogger<RedHatProviderMetadataLoader> logger,
|
||||
IFileSystem? fileSystem = null)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_fileSystem = fileSystem ?? new FileSystem();
|
||||
_serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RedHatProviderMetadataResult> LoadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_cache.TryGetValue<CacheEntry>(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired())
|
||||
{
|
||||
_logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt);
|
||||
return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline);
|
||||
}
|
||||
|
||||
await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_cache.TryGetValue<CacheEntry>(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired())
|
||||
{
|
||||
return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline);
|
||||
}
|
||||
|
||||
CacheEntry? previous = cached;
|
||||
|
||||
// Attempt live fetch unless offline preferred.
|
||||
if (!_options.PreferOfflineSnapshot)
|
||||
{
|
||||
var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false);
|
||||
if (httpResult is not null)
|
||||
{
|
||||
StoreCache(httpResult);
|
||||
return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
var offlineResult = TryLoadFromOffline();
|
||||
if (offlineResult is not null)
|
||||
{
|
||||
var offlineEntry = offlineResult with
|
||||
{
|
||||
FetchedAt = DateTimeOffset.UtcNow,
|
||||
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
|
||||
FromOffline = true,
|
||||
};
|
||||
StoreCache(offlineEntry);
|
||||
return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_refreshSemaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void StoreCache(CacheEntry entry)
|
||||
{
|
||||
var cacheEntryOptions = new MemoryCacheEntryOptions
|
||||
{
|
||||
AbsoluteExpiration = entry.ExpiresAt,
|
||||
};
|
||||
_cache.Set(CacheKey, entry, cacheEntryOptions);
|
||||
}
|
||||
|
||||
private async Task<CacheEntry?> TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri);
|
||||
if (!string.IsNullOrWhiteSpace(previous?.ETag))
|
||||
{
|
||||
if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag))
|
||||
{
|
||||
request.Headers.IfNoneMatch.Add(etag);
|
||||
}
|
||||
}
|
||||
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.NotModified && previous is not null)
|
||||
{
|
||||
_logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag);
|
||||
return previous with
|
||||
{
|
||||
FetchedAt = DateTimeOffset.UtcNow,
|
||||
ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
|
||||
};
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var provider = ParseAndValidate(payload);
|
||||
var etagHeader = response.Headers.ETag?.ToString();
|
||||
|
||||
if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
|
||||
{
|
||||
try
|
||||
{
|
||||
_fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload);
|
||||
_logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath);
|
||||
}
|
||||
}
|
||||
|
||||
return new CacheEntry(
|
||||
provider,
|
||||
DateTimeOffset.UtcNow,
|
||||
DateTimeOffset.UtcNow + _options.MetadataCacheDuration,
|
||||
etagHeader,
|
||||
FromOffline: false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private CacheEntry? TryLoadFromOffline()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath))
|
||||
{
|
||||
_logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath);
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath);
|
||||
var provider = ParseAndValidate(payload);
|
||||
return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private VexProvider ParseAndValidate(string payload)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(payload))
|
||||
{
|
||||
throw new InvalidOperationException("Provider metadata payload was empty.");
|
||||
}
|
||||
|
||||
ProviderMetadataDocument? document;
|
||||
try
|
||||
{
|
||||
document = JsonSerializer.Deserialize<ProviderMetadataDocument>(payload, _serializerOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex);
|
||||
}
|
||||
|
||||
if (document is null)
|
||||
{
|
||||
throw new InvalidOperationException("Provider metadata payload was null after parsing.");
|
||||
}
|
||||
|
||||
if (document.Metadata?.Provider?.Name is null)
|
||||
{
|
||||
throw new InvalidOperationException("Provider metadata missing provider name.");
|
||||
}
|
||||
|
||||
var distributions = document.Distributions?
|
||||
.Select(static d => d.Directory)
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory)))
|
||||
.ToImmutableArray() ?? ImmutableArray<Uri>.Empty;
|
||||
|
||||
if (distributions.IsDefaultOrEmpty)
|
||||
{
|
||||
throw new InvalidOperationException("Provider metadata did not include any valid distribution directories.");
|
||||
}
|
||||
|
||||
Uri? rolieFeed = null;
|
||||
if (document.Rolie?.Feeds is not null)
|
||||
{
|
||||
foreach (var feed in document.Rolie.Feeds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(feed.Url))
|
||||
{
|
||||
rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var trust = BuildTrust();
|
||||
return new VexProvider(
|
||||
id: "excititor:redhat",
|
||||
displayName: document.Metadata.Provider.Name,
|
||||
kind: VexProviderKind.Distro,
|
||||
baseUris: distributions,
|
||||
discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed),
|
||||
trust: trust);
|
||||
}
|
||||
|
||||
private VexProviderTrust BuildTrust()
|
||||
{
|
||||
VexCosignTrust? cosign = null;
|
||||
if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern))
|
||||
{
|
||||
cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!);
|
||||
}
|
||||
|
||||
return new VexProviderTrust(
|
||||
_options.TrustWeight,
|
||||
cosign,
|
||||
_options.PgpFingerprints);
|
||||
}
|
||||
|
||||
private static Uri CreateUri(string value, string propertyName)
|
||||
{
|
||||
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https"))
|
||||
{
|
||||
throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI.");
|
||||
}
|
||||
|
||||
return uri;
|
||||
}
|
||||
|
||||
private sealed record ProviderMetadataDocument(
|
||||
[property: JsonPropertyName("metadata")] ProviderMetadata? Metadata,
|
||||
[property: JsonPropertyName("distributions")] IReadOnlyList<ProviderMetadataDistribution>? Distributions,
|
||||
[property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie);
|
||||
|
||||
private sealed record ProviderMetadata(
|
||||
[property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider);
|
||||
|
||||
private sealed record ProviderMetadataProvider(
|
||||
[property: JsonPropertyName("name")] string? Name);
|
||||
|
||||
private sealed record ProviderMetadataDistribution(
|
||||
[property: JsonPropertyName("directory")] string? Directory);
|
||||
|
||||
private sealed record ProviderMetadataRolie(
|
||||
[property: JsonPropertyName("feeds")] IReadOnlyList<ProviderMetadataRolieFeed>? Feeds);
|
||||
|
||||
private sealed record ProviderMetadataRolieFeed(
|
||||
[property: JsonPropertyName("url")] string? Url);
|
||||
|
||||
private sealed record CacheEntry(
|
||||
VexProvider Provider,
|
||||
DateTimeOffset FetchedAt,
|
||||
DateTimeOffset ExpiresAt,
|
||||
string? ETag,
|
||||
bool FromOffline)
|
||||
{
|
||||
public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record RedHatProviderMetadataResult(
|
||||
VexProvider Provider,
|
||||
DateTimeOffset FetchedAt,
|
||||
bool FromCache,
|
||||
bool FromOfflineSnapshot);
|
||||
@@ -0,0 +1,186 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF;
|
||||
|
||||
public sealed class RedHatCsafConnector : VexConnectorBase
|
||||
{
|
||||
private readonly RedHatProviderMetadataLoader _metadataLoader;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
public RedHatCsafConnector(
|
||||
VexConnectorDescriptor descriptor,
|
||||
RedHatProviderMetadataLoader metadataLoader,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
ILogger<RedHatCsafConnector> logger,
|
||||
TimeProvider timeProvider)
|
||||
: base(descriptor, logger, timeProvider)
|
||||
{
|
||||
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
}
|
||||
|
||||
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
// No connector-specific settings yet.
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (metadataResult.Provider.Discovery.RolIeService is null)
|
||||
{
|
||||
throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed.");
|
||||
}
|
||||
|
||||
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var sinceTimestamp = context.Since;
|
||||
if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp))
|
||||
{
|
||||
sinceTimestamp = persisted;
|
||||
}
|
||||
|
||||
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
|
||||
var digestList = new List<string>(knownDigests);
|
||||
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
|
||||
var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue;
|
||||
var stateChanged = false;
|
||||
|
||||
foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.DocumentUri is null)
|
||||
{
|
||||
Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!digestSet.Add(rawDocument.Digest))
|
||||
{
|
||||
Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
digestList.Add(rawDocument.Digest);
|
||||
stateChanged = true;
|
||||
|
||||
if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated)
|
||||
{
|
||||
latestUpdated = entryUpdated;
|
||||
}
|
||||
|
||||
yield return rawDocument;
|
||||
}
|
||||
|
||||
if (stateChanged)
|
||||
{
|
||||
var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated;
|
||||
var updatedState = new VexConnectorState(
|
||||
Descriptor.Id,
|
||||
newLastUpdated,
|
||||
digestList.ToImmutableArray());
|
||||
|
||||
await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
// This connector relies on format-specific normalizers registered elsewhere.
|
||||
throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component.");
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<RolieEntry>> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = XDocument.Load(stream);
|
||||
var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom";
|
||||
|
||||
var entries = document.Root?
|
||||
.Elements(ns + "entry")
|
||||
.Select(e => new RolieEntry(
|
||||
Id: (string?)e.Element(ns + "id"),
|
||||
Updated: ParseUpdated((string?)e.Element(ns + "updated")),
|
||||
DocumentUri: ParseDocumentLink(e, ns)))
|
||||
.Where(entry => entry.Id is not null && entry.Updated is not null)
|
||||
.OrderBy(entry => entry.Updated)
|
||||
.ToList() ?? new List<RolieEntry>();
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseUpdated(string? value)
|
||||
=> DateTimeOffset.TryParse(value, out var parsed) ? parsed : null;
|
||||
|
||||
private static Uri? ParseDocumentLink(XElement entry, XNamespace ns)
|
||||
{
|
||||
var linkElements = entry.Elements(ns + "link");
|
||||
foreach (var link in linkElements)
|
||||
{
|
||||
var rel = (string?)link.Attribute("rel");
|
||||
var href = (string?)link.Attribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (Uri.TryCreate(href, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return uri;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<VexRawDocument> DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI.");
|
||||
|
||||
var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
var metadata = BuildMetadata(builder => builder
|
||||
.Add("redhat.csaf.entryId", entry.Id)
|
||||
.Add("redhat.csaf.documentUri", documentUri.ToString())
|
||||
.Add("redhat.csaf.updated", entry.Updated?.ToString("O")));
|
||||
|
||||
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata);
|
||||
}
|
||||
|
||||
private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
|
||||
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
10
src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md
Normal file
10
src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md
Normal file
@@ -0,0 +1,10 @@
|
||||
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|EXCITITOR-CONN-RH-01-001 – Provider metadata discovery|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `RedHatProviderMetadataLoader` with HTTP/ETag caching, offline snapshot handling, and validation; exposed DI helper + tests covering live, cached, and offline scenarios.|
|
||||
|EXCITITOR-CONN-RH-01-002 – Incremental CSAF pulls|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Implemented `RedHatCsafConnector` with ROLIE feed parsing, incremental filtering via `context.Since`, CSAF document download + metadata capture, and persistence through `IVexRawDocumentSink`; tests cover live fetch/cache/offline scenarios with ETag handling.|
|
||||
|EXCITITOR-CONN-RH-01-003 – Trust metadata emission|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-POLICY-01-001|**DONE (2025-10-17)** – Provider metadata loader now emits trust overrides (weight, cosign issuer/pattern, PGP fingerprints) and the connector surfaces provenance hints for policy/consensus layers.|
|
||||
|EXCITITOR-CONN-RH-01-004 – Resume state persistence|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Connector now loads/saves resume state via `IVexConnectorStateRepository`, tracking last update timestamp and recent document digests to avoid duplicate CSAF ingestion; regression covers state persistence and duplicate skips.|
|
||||
|EXCITITOR-CONN-RH-01-005 – Worker/WebService integration|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002|**DONE (2025-10-17)** – Worker/WebService now call `AddRedHatCsafConnector`, register the connector + state repo, and default worker scheduling adds the `excititor:redhat` provider so background jobs and orchestration can activate the connector without extra wiring.|
|
||||
|EXCITITOR-CONN-RH-01-006 – CSAF normalization parity tests|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-FMT-CSAF-01-001|**DONE (2025-10-17)** – Added RHSA fixture-driven regression verifying CSAF normalizer retains Red Hat product metadata, tracking fields, and timestamps (`rhsa-sample.json` + `CsafNormalizerTests.NormalizeAsync_PreservesRedHatSpecificMetadata`).|
|
||||
Reference in New Issue
Block a user