git.stella-ops.org/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.CertCc.Configuration;
using StellaOps.Feedser.Source.CertCc.Internal;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Common.Cursors;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Plugin;

namespace StellaOps.Feedser.Source.CertCc;

public sealed class CertCcConnector : IFeedConnector
{
    private readonly CertCcSummaryPlanner _summaryPlanner;
    private readonly SourceFetchService _fetchService;
    private readonly IDocumentStore _documentStore;
    private readonly ISourceStateRepository _stateRepository;
    private readonly CertCcOptions _options;
    private readonly TimeProvider _timeProvider;
    private readonly ILogger<CertCcConnector> _logger;

    public CertCcConnector(
        CertCcSummaryPlanner summaryPlanner,
        SourceFetchService fetchService,
        IDocumentStore documentStore,
        ISourceStateRepository stateRepository,
        IOptions<CertCcOptions> options,
        TimeProvider? timeProvider,
        ILogger<CertCcConnector> logger)
    {
        _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner));
        _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
        _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
        _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
        _options.Validate();
        _timeProvider = timeProvider ?? TimeProvider.System;
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public string SourceName => CertCcConnectorPlugin.SourceName;

    public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        var plan = _summaryPlanner.CreatePlan(cursor.SummaryState);
        if (plan.Requests.Count == 0)
        {
            await UpdateCursorAsync(cursor.WithSummaryState(plan.NextState).WithLastRun(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false);
            return;
        }

        foreach (var request in plan.Requests)
        {
            cancellationToken.ThrowIfCancellationRequested();

            try
            {
                var uri = request.Uri;
                var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false);
                var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
                {
                    ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(),
                    ["certcc.year"] = request.Year.ToString("D4"),
                };

                if (request.Month.HasValue)
                {
                    metadata["certcc.month"] = request.Month.Value.ToString("D2");
                }

                var fetchRequest = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri)
                {
                    Metadata = metadata,
                    AcceptHeaders = new[] { "application/json" },
                    ETag = existing?.Etag,
                    LastModified = existing?.LastModified,
                };

                var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false);
                if (result.IsNotModified)
                {
                    _logger.LogDebug("CERT/CC summary {Uri} returned 304 Not Modified", uri);
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "CERT/CC summary fetch failed for {Uri}", request.Uri);
                await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
                throw;
            }
        }

        var updatedCursor = cursor
            .WithSummaryState(plan.NextState)
            .WithLastRun(_timeProvider.GetUtcNow());

        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
    }

    public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
        => Task.CompletedTask;

    public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
        => Task.CompletedTask;

    private async Task<CertCcCursor> GetCursorAsync(CancellationToken cancellationToken)
    {
        var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
        return CertCcCursor.FromBson(record?.Cursor);
    }

    private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
    {
        var document = cursor.ToBsonDocument();
        await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
    }
}