using System; using System.Collections.Generic; using System.Collections.Immutable; using System.IO.Abstractions; using System.Net; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; public sealed class UbuntuCatalogLoader { public const string CachePrefix = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Index"; private readonly IHttpClientFactory _httpClientFactory; private readonly IMemoryCache _memoryCache; private readonly IFileSystem _fileSystem; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; private readonly SemaphoreSlim _semaphore = new(1, 1); private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); public UbuntuCatalogLoader( IHttpClientFactory httpClientFactory, IMemoryCache memoryCache, IFileSystem fileSystem, ILogger logger, TimeProvider? timeProvider = null) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; } public async Task LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(options); options.Validate(_fileSystem); var cacheKey = CreateCacheKey(options); if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) { return cached.ToResult(fromCache: true); } await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) { return cached.ToResult(fromCache: true); } CacheEntry? entry = null; if (options.PreferOfflineSnapshot) { entry = LoadFromOffline(options); if (entry is null) { throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found."); } } else { entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) ?? LoadFromOffline(options); } if (entry is null) { throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot."); } var cacheOptions = new MemoryCacheEntryOptions(); if (entry.MetadataCacheDuration > TimeSpan.Zero) { cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); } _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions); return entry.ToResult(fromCache: false); } finally { _semaphore.Release(); } } private async Task TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) { try { var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var metadata = ParseMetadata(payload, options.Channels); var now = _timeProvider.GetUtcNow(); var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false); PersistSnapshotIfNeeded(options, metadata, now); return entry; } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri); return null; } } private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options) { if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) { return null; } if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) { _logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); return null; } try { var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); if (snapshot is null) { throw new InvalidOperationException("Offline snapshot payload was empty."); } return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); } catch (Exception ex) { _logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath); return null; } } private UbuntuCatalogMetadata ParseMetadata(string payload, IList channels) { if (string.IsNullOrWhiteSpace(payload)) { throw new InvalidOperationException("Ubuntu index payload was empty."); } using var document = JsonDocument.Parse(payload); var root = document.RootElement; var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) ? generated : _timeProvider.GetUtcNow(); var channelSet = new HashSet(channels, StringComparer.OrdinalIgnoreCase); if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array) { throw new InvalidOperationException("Ubuntu index did not include a channels array."); } var builder = ImmutableArray.CreateBuilder(); foreach (var channelElement in channelsElement.EnumerateArray()) { var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name)) { continue; } if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri)) { _logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name); continue; } string? sha256 = null; if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String) { sha256 = shaElement.GetString(); } DateTimeOffset? lastUpdated = null; if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated)) { lastUpdated = updated; } builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated)); } if (builder.Count == 0) { throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index."); } return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable()); } private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt) { if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) { return; } try { var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt); var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); _logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); } catch (Exception ex) { _logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); } } private static string CreateCacheKey(UbuntuConnectorOptions options) => $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}"; private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) { public bool IsExpired(DateTimeOffset now) => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; public UbuntuCatalogResult ToResult(bool fromCache) => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); } private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt); } public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray Channels); public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated); public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot);