249 lines
10 KiB
C#
249 lines
10 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Collections.Immutable;
|
|
using System.IO.Abstractions;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.Extensions.Caching.Memory;
|
|
using Microsoft.Extensions.Logging;
|
|
using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration;
|
|
|
|
namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata;
|
|
|
|
public sealed class UbuntuCatalogLoader
|
|
{
|
|
public const string CachePrefix = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Index";
|
|
|
|
private readonly IHttpClientFactory _httpClientFactory;
|
|
private readonly IMemoryCache _memoryCache;
|
|
private readonly IFileSystem _fileSystem;
|
|
private readonly ILogger<UbuntuCatalogLoader> _logger;
|
|
private readonly TimeProvider _timeProvider;
|
|
private readonly SemaphoreSlim _semaphore = new(1, 1);
|
|
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
|
|
|
|
public UbuntuCatalogLoader(
|
|
IHttpClientFactory httpClientFactory,
|
|
IMemoryCache memoryCache,
|
|
IFileSystem fileSystem,
|
|
ILogger<UbuntuCatalogLoader> logger,
|
|
TimeProvider? timeProvider = null)
|
|
{
|
|
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
|
_memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache));
|
|
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
|
}
|
|
|
|
public async Task<UbuntuCatalogResult> LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(options);
|
|
options.Validate(_fileSystem);
|
|
|
|
var cacheKey = CreateCacheKey(options);
|
|
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
|
|
{
|
|
return cached.ToResult(fromCache: true);
|
|
}
|
|
|
|
await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
if (_memoryCache.TryGetValue<CacheEntry>(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow()))
|
|
{
|
|
return cached.ToResult(fromCache: true);
|
|
}
|
|
|
|
CacheEntry? entry = null;
|
|
if (options.PreferOfflineSnapshot)
|
|
{
|
|
entry = LoadFromOffline(options);
|
|
if (entry is null)
|
|
{
|
|
throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found.");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false)
|
|
?? LoadFromOffline(options);
|
|
}
|
|
|
|
if (entry is null)
|
|
{
|
|
throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot.");
|
|
}
|
|
|
|
var cacheOptions = new MemoryCacheEntryOptions();
|
|
if (entry.MetadataCacheDuration > TimeSpan.Zero)
|
|
{
|
|
cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration);
|
|
}
|
|
|
|
_memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions);
|
|
return entry.ToResult(fromCache: false);
|
|
}
|
|
finally
|
|
{
|
|
_semaphore.Release();
|
|
}
|
|
}
|
|
|
|
private async Task<CacheEntry?> TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
|
|
using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
|
response.EnsureSuccessStatusCode();
|
|
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
var metadata = ParseMetadata(payload, options.Channels);
|
|
var now = _timeProvider.GetUtcNow();
|
|
var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false);
|
|
|
|
PersistSnapshotIfNeeded(options, metadata, now);
|
|
return entry;
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if (!_fileSystem.File.Exists(options.OfflineSnapshotPath))
|
|
{
|
|
_logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath);
|
|
return null;
|
|
}
|
|
|
|
try
|
|
{
|
|
var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath);
|
|
var snapshot = JsonSerializer.Deserialize<UbuntuCatalogSnapshot>(payload, _serializerOptions);
|
|
if (snapshot is null)
|
|
{
|
|
throw new InvalidOperationException("Offline snapshot payload was empty.");
|
|
}
|
|
|
|
return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private UbuntuCatalogMetadata ParseMetadata(string payload, IList<string> channels)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(payload))
|
|
{
|
|
throw new InvalidOperationException("Ubuntu index payload was empty.");
|
|
}
|
|
|
|
using var document = JsonDocument.Parse(payload);
|
|
var root = document.RootElement;
|
|
|
|
var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated)
|
|
? generated
|
|
: _timeProvider.GetUtcNow();
|
|
|
|
var channelSet = new HashSet<string>(channels, StringComparer.OrdinalIgnoreCase);
|
|
|
|
if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array)
|
|
{
|
|
throw new InvalidOperationException("Ubuntu index did not include a channels array.");
|
|
}
|
|
|
|
var builder = ImmutableArray.CreateBuilder<UbuChannelCatalog>();
|
|
foreach (var channelElement in channelsElement.EnumerateArray())
|
|
{
|
|
var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null;
|
|
if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri))
|
|
{
|
|
_logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name);
|
|
continue;
|
|
}
|
|
|
|
string? sha256 = null;
|
|
if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String)
|
|
{
|
|
sha256 = shaElement.GetString();
|
|
}
|
|
|
|
DateTimeOffset? lastUpdated = null;
|
|
if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated))
|
|
{
|
|
lastUpdated = updated;
|
|
}
|
|
|
|
builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated));
|
|
}
|
|
|
|
if (builder.Count == 0)
|
|
{
|
|
throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index.");
|
|
}
|
|
|
|
return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable());
|
|
}
|
|
|
|
private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt)
|
|
{
|
|
if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath))
|
|
{
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt);
|
|
var payload = JsonSerializer.Serialize(snapshot, _serializerOptions);
|
|
_fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload);
|
|
_logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath);
|
|
}
|
|
}
|
|
|
|
private static string CreateCacheKey(UbuntuConnectorOptions options)
|
|
=> $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}";
|
|
|
|
private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot)
|
|
{
|
|
public bool IsExpired(DateTimeOffset now)
|
|
=> MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration;
|
|
|
|
public UbuntuCatalogResult ToResult(bool fromCache)
|
|
=> new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot);
|
|
}
|
|
|
|
private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt);
|
|
}
|
|
|
|
public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray<UbuChannelCatalog> Channels);
|
|
|
|
public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated);
|
|
|
|
public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot);
|