Initial commit (history squashed)

src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using StellaOps.Feedser.Source.CertCc.Configuration;
+using StellaOps.Feedser.Source.CertCc.Internal;
+using StellaOps.Feedser.Source.Common;
+using StellaOps.Feedser.Source.Common.Fetch;
+using StellaOps.Feedser.Source.Common.Cursors;
+using StellaOps.Feedser.Storage.Mongo;
+using StellaOps.Feedser.Storage.Mongo.Documents;
+using StellaOps.Plugin;
+
+namespace StellaOps.Feedser.Source.CertCc;
+
+public sealed class CertCcConnector : IFeedConnector
+{
+    private readonly CertCcSummaryPlanner _summaryPlanner;
+    private readonly SourceFetchService _fetchService;
+    private readonly IDocumentStore _documentStore;
+    private readonly ISourceStateRepository _stateRepository;
+    private readonly CertCcOptions _options;
+    private readonly TimeProvider _timeProvider;
+    private readonly ILogger<CertCcConnector> _logger;
+
+    public CertCcConnector(
+        CertCcSummaryPlanner summaryPlanner,
+        SourceFetchService fetchService,
+        IDocumentStore documentStore,
+        ISourceStateRepository stateRepository,
+        IOptions<CertCcOptions> options,
+        TimeProvider? timeProvider,
+        ILogger<CertCcConnector> logger)
+    {
+        _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner));
+        _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
+        _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
+        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
+        _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
+        _options.Validate();
+        _timeProvider = timeProvider ?? TimeProvider.System;
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+    }
+
+    public string SourceName => CertCcConnectorPlugin.SourceName;
+
+    public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
+    {
+        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
+        var plan = _summaryPlanner.CreatePlan(cursor.SummaryState);
+        if (plan.Requests.Count == 0)
+        {
+            await UpdateCursorAsync(cursor.WithSummaryState(plan.NextState).WithLastRun(_timeProvider.GetUtcNow()), cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        foreach (var request in plan.Requests)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            try
+            {
+                var uri = request.Uri;
+                var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false);
+                var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(),
+                    ["certcc.year"] = request.Year.ToString("D4"),
+                };
+
+                if (request.Month.HasValue)
+                {
+                    metadata["certcc.month"] = request.Month.Value.ToString("D2");
+                }
+
+                var fetchRequest = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri)
+                {
+                    Metadata = metadata,
+                    AcceptHeaders = new[] { "application/json" },
+                    ETag = existing?.Etag,
+                    LastModified = existing?.LastModified,
+                };
+
+                var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false);
+                if (result.IsNotModified)
+                {
+                    _logger.LogDebug("CERT/CC summary {Uri} returned 304 Not Modified", uri);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "CERT/CC summary fetch failed for {Uri}", request.Uri);
+                await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
+                throw;
+            }
+        }
+
+        var updatedCursor = cursor
+            .WithSummaryState(plan.NextState)
+            .WithLastRun(_timeProvider.GetUtcNow());
+
+        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
+    }
+
+    public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
+        => Task.CompletedTask;
+
+    public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
+        => Task.CompletedTask;
+
+    private async Task<CertCcCursor> GetCursorAsync(CancellationToken cancellationToken)
+    {
+        var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
+        return CertCcCursor.FromBson(record?.Cursor);
+    }
+
+    private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
+    {
+        var document = cursor.ToBsonDocument();
+        await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
+    }
+}