Initial commit (history squashed)

src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs

@@ -0,0 +1,263 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Http;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Time.Testing;
+using MongoDB.Bson;
+using StellaOps.Feedser.Source.CertCc;
+using StellaOps.Feedser.Source.CertCc.Configuration;
+using StellaOps.Feedser.Source.CertCc.Internal;
+using StellaOps.Feedser.Source.Common;
+using StellaOps.Feedser.Source.Common.Http;
+using StellaOps.Feedser.Source.Common.Cursors;
+using StellaOps.Feedser.Source.Common.Testing;
+using StellaOps.Feedser.Storage.Mongo;
+using StellaOps.Feedser.Storage.Mongo.Documents;
+using StellaOps.Feedser.Testing;
+using Xunit;
+
+namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc;
+
+[Collection("mongo-fixture")]
+public sealed class CertCcConnectorFetchTests : IAsyncLifetime
+{
+    private const string TestNoteId = "294418";
+
+    private readonly MongoIntegrationFixture _fixture;
+    private readonly FakeTimeProvider _timeProvider;
+    private readonly CannedHttpMessageHandler _handler;
+    private ServiceProvider? _serviceProvider;
+
+    public CertCcConnectorFetchTests(MongoIntegrationFixture fixture)
+    {
+        _fixture = fixture;
+        _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero));
+        _handler = new CannedHttpMessageHandler();
+    }
+
+    [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")]
+    public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor()
+    {
+        var template = new CertCcOptions
+        {
+            BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute),
+            SummaryWindow = new TimeWindowCursorOptions
+            {
+                WindowSize = TimeSpan.FromDays(30),
+                Overlap = TimeSpan.FromDays(5),
+                InitialBackfill = TimeSpan.FromDays(60),
+                MinimumWindowSize = TimeSpan.FromDays(1),
+            },
+            MaxMonthlySummaries = 3,
+            MaxNotesPerFetch = 3,
+            DetailRequestDelay = TimeSpan.Zero,
+        };
+
+        await EnsureServiceProviderAsync(template);
+        var provider = _serviceProvider!;
+
+        _handler.Clear();
+
+        var planner = provider.GetRequiredService<CertCcSummaryPlanner>();
+        var plan = planner.CreatePlan(state: null);
+        Assert.NotEmpty(plan.Requests);
+
+        foreach (var request in plan.Requests)
+        {
+            _handler.AddJsonResponse(request.Uri, BuildSummaryPayload());
+        }
+
+        RegisterDetailResponses();
+
+        var connector = provider.GetRequiredService<CertCcConnector>();
+        await connector.FetchAsync(provider, CancellationToken.None);
+
+        var documentStore = provider.GetRequiredService<IDocumentStore>();
+        foreach (var request in plan.Requests)
+        {
+            var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None);
+            Assert.NotNull(record);
+            Assert.Equal(DocumentStatuses.PendingParse, record!.Status);
+            Assert.NotNull(record.Metadata);
+            Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]);
+            Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]);
+            if (request.Month.HasValue)
+            {
+                Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]);
+            }
+            else
+            {
+                Assert.False(record.Metadata.ContainsKey("certcc.month"));
+            }
+        }
+
+        foreach (var uri in EnumerateDetailUris())
+        {
+            var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None);
+            Assert.NotNull(record);
+            Assert.Equal(DocumentStatuses.PendingParse, record!.Status);
+            Assert.NotNull(record.Metadata);
+            Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]);
+        }
+
+        var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
+        var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None);
+        Assert.NotNull(state);
+
+        BsonValue summaryValue;
+        Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue));
+        var summaryDocument = Assert.IsType<BsonDocument>(summaryValue);
+        Assert.True(summaryDocument.TryGetValue("start", out _));
+        Assert.True(summaryDocument.TryGetValue("end", out _));
+
+        var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue)
+            ? pendingNotesValue.AsBsonArray.Count
+            : 0;
+        Assert.Equal(0, pendingNotesCount);
+
+        var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue)
+            ? pendingSummariesValue.AsBsonArray.Count
+            : 0;
+        Assert.Equal(0, pendingSummariesCount);
+
+        Assert.True(state.Cursor.TryGetValue("lastRun", out _));
+
+        Assert.True(_handler.Requests.Count >= plan.Requests.Count);
+        foreach (var request in _handler.Requests)
+        {
+            if (request.Headers.TryGetValue("Accept", out var accept))
+            {
+                Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase);
+            }
+        }
+    }
+
+    private static string BuildSummaryPayload()
+    {
+        return $$"""
+        {
+            "count": 1,
+            "notes": [
+                "VU#{TestNoteId}"
+            ]
+        }
+        """;
+    }
+
+    private void RegisterDetailResponses()
+    {
+        foreach (var uri in EnumerateDetailUris())
+        {
+            var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase)
+                ? "vu-294418-vendors.json"
+                : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase)
+                    ? "vu-294418-vuls.json"
+                    : "vu-294418.json";
+
+            _handler.AddJsonResponse(uri, ReadFixture(fixtureName));
+        }
+    }
+
+    private static IEnumerable<Uri> EnumerateDetailUris()
+    {
+        var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
+        yield return new Uri(baseUri, $"{TestNoteId}/");
+        yield return new Uri(baseUri, $"{TestNoteId}/vendors/");
+        yield return new Uri(baseUri, $"{TestNoteId}/vuls/");
+    }
+
+    private async Task EnsureServiceProviderAsync(CertCcOptions template)
+    {
+        await DisposeServiceProviderAsync();
+        await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
+
+        var services = new ServiceCollection();
+        services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
+        services.AddSingleton<TimeProvider>(_timeProvider);
+        services.AddSingleton(_handler);
+
+        services.AddMongoStorage(options =>
+        {
+            options.ConnectionString = _fixture.Runner.ConnectionString;
+            options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
+            options.CommandTimeout = TimeSpan.FromSeconds(5);
+            options.RawDocumentRetention = TimeSpan.Zero;
+            options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5);
+            options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1);
+        });
+
+        services.AddSourceCommon();
+        services.AddCertCcConnector(options =>
+        {
+            options.BaseApiUri = template.BaseApiUri;
+            options.SummaryWindow = new TimeWindowCursorOptions
+            {
+                WindowSize = template.SummaryWindow.WindowSize,
+                Overlap = template.SummaryWindow.Overlap,
+                InitialBackfill = template.SummaryWindow.InitialBackfill,
+                MinimumWindowSize = template.SummaryWindow.MinimumWindowSize,
+            };
+            options.MaxMonthlySummaries = template.MaxMonthlySummaries;
+            options.MaxNotesPerFetch = template.MaxNotesPerFetch;
+            options.DetailRequestDelay = template.DetailRequestDelay;
+            options.EnableDetailMapping = template.EnableDetailMapping;
+        });
+
+        services.Configure<HttpClientFactoryOptions>(CertCcOptions.HttpClientName, builderOptions =>
+        {
+            builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler);
+        });
+
+        _serviceProvider = services.BuildServiceProvider();
+        var bootstrapper = _serviceProvider.GetRequiredService<MongoBootstrapper>();
+        await bootstrapper.InitializeAsync(CancellationToken.None);
+    }
+
+    private async Task DisposeServiceProviderAsync()
+    {
+        if (_serviceProvider is null)
+        {
+            return;
+        }
+
+        if (_serviceProvider is IAsyncDisposable asyncDisposable)
+        {
+            await asyncDisposable.DisposeAsync();
+        }
+        else
+        {
+            _serviceProvider.Dispose();
+        }
+
+        _serviceProvider = null;
+    }
+
+    private static string ReadFixture(string filename)
+    {
+        var baseDirectory = AppContext.BaseDirectory;
+        var primary = Path.Combine(baseDirectory, "Fixtures", filename);
+        if (File.Exists(primary))
+        {
+            return File.ReadAllText(primary);
+        }
+
+        return File.ReadAllText(Path.Combine(baseDirectory, filename));
+    }
+
+    public Task InitializeAsync()
+    {
+        _handler.Clear();
+        return Task.CompletedTask;
+    }
+
+    public async Task DisposeAsync()
+    {
+        await DisposeServiceProviderAsync();
+    }
+}