using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Feedser.Source.CertCc.Configuration; using StellaOps.Feedser.Source.CertCc.Internal; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Fetch; using StellaOps.Feedser.Source.Common.Cursors; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Plugin; namespace StellaOps.Feedser.Source.CertCc; public sealed class CertCcConnector : IFeedConnector { private readonly CertCcSummaryPlanner _summaryPlanner; private readonly SourceFetchService _fetchService; private readonly IDocumentStore _documentStore; private readonly ISourceStateRepository _stateRepository; private readonly CertCcOptions _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; public CertCcConnector( CertCcSummaryPlanner summaryPlanner, SourceFetchService fetchService, IDocumentStore documentStore, ISourceStateRepository stateRepository, IOptions options, TimeProvider? timeProvider, ILogger logger) { _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner)); _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); _options.Validate(); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public string SourceName => CertCcConnectorPlugin.SourceName; public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) { var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var plan = _summaryPlanner.CreatePlan(cursor.SummaryState); if (plan.Requests.Count == 0) { await UpdateCursorAsync(cursor.WithSummaryState(plan.NextState).WithLastRun(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false); return; } foreach (var request in plan.Requests) { cancellationToken.ThrowIfCancellationRequested(); try { var uri = request.Uri; var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(), ["certcc.year"] = request.Year.ToString("D4"), }; if (request.Month.HasValue) { metadata["certcc.month"] = request.Month.Value.ToString("D2"); } var fetchRequest = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri) { Metadata = metadata, AcceptHeaders = new[] { "application/json" }, ETag = existing?.Etag, LastModified = existing?.LastModified, }; var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); if (result.IsNotModified) { _logger.LogDebug("CERT/CC summary {Uri} returned 304 Not Modified", uri); } } catch (Exception ex) { _logger.LogError(ex, "CERT/CC summary fetch failed for {Uri}", request.Uri); await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); throw; } } var updatedCursor = cursor .WithSummaryState(plan.NextState) .WithLastRun(_timeProvider.GetUtcNow()); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; private async Task GetCursorAsync(CancellationToken cancellationToken) { var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); return CertCcCursor.FromBson(record?.Cursor); } private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken) { var document = cursor.ToBsonDocument(); await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); } }