git.stella-ops.org/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Bson;
using StellaOps.Feedser.Source.CertCc;
using StellaOps.Feedser.Source.CertCc.Configuration;
using StellaOps.Feedser.Source.CertCc.Internal;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common.Cursors;
using StellaOps.Feedser.Source.Common.Testing;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Testing;
using Xunit;

namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc;

[Collection("mongo-fixture")]
public sealed class CertCcConnectorFetchTests : IAsyncLifetime
{
    private const string TestNoteId = "294418";

    private readonly MongoIntegrationFixture _fixture;
    private readonly FakeTimeProvider _timeProvider;
    private readonly CannedHttpMessageHandler _handler;
    private ServiceProvider? _serviceProvider;

    public CertCcConnectorFetchTests(MongoIntegrationFixture fixture)
    {
        _fixture = fixture;
        _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero));
        _handler = new CannedHttpMessageHandler();
    }

    [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")]
    public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor()
    {
        var template = new CertCcOptions
        {
            BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute),
            SummaryWindow = new TimeWindowCursorOptions
            {
                WindowSize = TimeSpan.FromDays(30),
                Overlap = TimeSpan.FromDays(5),
                InitialBackfill = TimeSpan.FromDays(60),
                MinimumWindowSize = TimeSpan.FromDays(1),
            },
            MaxMonthlySummaries = 3,
            MaxNotesPerFetch = 3,
            DetailRequestDelay = TimeSpan.Zero,
        };

        await EnsureServiceProviderAsync(template);
        var provider = _serviceProvider!;

        _handler.Clear();

        var planner = provider.GetRequiredService<CertCcSummaryPlanner>();
        var plan = planner.CreatePlan(state: null);
        Assert.NotEmpty(plan.Requests);

        foreach (var request in plan.Requests)
        {
            _handler.AddJsonResponse(request.Uri, BuildSummaryPayload());
        }

        RegisterDetailResponses();

        var connector = provider.GetRequiredService<CertCcConnector>();
        await connector.FetchAsync(provider, CancellationToken.None);

        var documentStore = provider.GetRequiredService<IDocumentStore>();
        foreach (var request in plan.Requests)
        {
            var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None);
            Assert.NotNull(record);
            Assert.Equal(DocumentStatuses.PendingParse, record!.Status);
            Assert.NotNull(record.Metadata);
            Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]);
            Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]);
            if (request.Month.HasValue)
            {
                Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]);
            }
            else
            {
                Assert.False(record.Metadata.ContainsKey("certcc.month"));
            }
        }

        foreach (var uri in EnumerateDetailUris())
        {
            var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None);
            Assert.NotNull(record);
            Assert.Equal(DocumentStatuses.PendingParse, record!.Status);
            Assert.NotNull(record.Metadata);
            Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]);
        }

        var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
        var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None);
        Assert.NotNull(state);

        BsonValue summaryValue;
        Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue));
        var summaryDocument = Assert.IsType<BsonDocument>(summaryValue);
        Assert.True(summaryDocument.TryGetValue("start", out _));
        Assert.True(summaryDocument.TryGetValue("end", out _));

        var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue)
            ? pendingNotesValue.AsBsonArray.Count
            : 0;
        Assert.Equal(0, pendingNotesCount);

        var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue)
            ? pendingSummariesValue.AsBsonArray.Count
            : 0;
        Assert.Equal(0, pendingSummariesCount);

        Assert.True(state.Cursor.TryGetValue("lastRun", out _));

        Assert.True(_handler.Requests.Count >= plan.Requests.Count);
        foreach (var request in _handler.Requests)
        {
            if (request.Headers.TryGetValue("Accept", out var accept))
            {
                Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase);
            }
        }
    }

    private static string BuildSummaryPayload()
    {
        return $$"""
        {
            "count": 1,
            "notes": [
                "VU#{TestNoteId}"
            ]
        }
        """;
    }

    private void RegisterDetailResponses()
    {
        foreach (var uri in EnumerateDetailUris())
        {
            var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase)
                ? "vu-294418-vendors.json"
                : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase)
                    ? "vu-294418-vuls.json"
                    : "vu-294418.json";

            _handler.AddJsonResponse(uri, ReadFixture(fixtureName));
        }
    }

    private static IEnumerable<Uri> EnumerateDetailUris()
    {
        var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
        yield return new Uri(baseUri, $"{TestNoteId}/");
        yield return new Uri(baseUri, $"{TestNoteId}/vendors/");
        yield return new Uri(baseUri, $"{TestNoteId}/vuls/");
    }

    private async Task EnsureServiceProviderAsync(CertCcOptions template)
    {
        await DisposeServiceProviderAsync();
        await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);

        var services = new ServiceCollection();
        services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
        services.AddSingleton<TimeProvider>(_timeProvider);
        services.AddSingleton(_handler);

        services.AddMongoStorage(options =>
        {
            options.ConnectionString = _fixture.Runner.ConnectionString;
            options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
            options.CommandTimeout = TimeSpan.FromSeconds(5);
            options.RawDocumentRetention = TimeSpan.Zero;
            options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5);
            options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1);
        });

        services.AddSourceCommon();
        services.AddCertCcConnector(options =>
        {
            options.BaseApiUri = template.BaseApiUri;
            options.SummaryWindow = new TimeWindowCursorOptions
            {
                WindowSize = template.SummaryWindow.WindowSize,
                Overlap = template.SummaryWindow.Overlap,
                InitialBackfill = template.SummaryWindow.InitialBackfill,
                MinimumWindowSize = template.SummaryWindow.MinimumWindowSize,
            };
            options.MaxMonthlySummaries = template.MaxMonthlySummaries;
            options.MaxNotesPerFetch = template.MaxNotesPerFetch;
            options.DetailRequestDelay = template.DetailRequestDelay;
            options.EnableDetailMapping = template.EnableDetailMapping;
        });

        services.Configure<HttpClientFactoryOptions>(CertCcOptions.HttpClientName, builderOptions =>
        {
            builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler);
        });

        _serviceProvider = services.BuildServiceProvider();
        var bootstrapper = _serviceProvider.GetRequiredService<MongoBootstrapper>();
        await bootstrapper.InitializeAsync(CancellationToken.None);
    }

    private async Task DisposeServiceProviderAsync()
    {
        if (_serviceProvider is null)
        {
            return;
        }

        if (_serviceProvider is IAsyncDisposable asyncDisposable)
        {
            await asyncDisposable.DisposeAsync();
        }
        else
        {
            _serviceProvider.Dispose();
        }

        _serviceProvider = null;
    }

    private static string ReadFixture(string filename)
    {
        var baseDirectory = AppContext.BaseDirectory;
        var primary = Path.Combine(baseDirectory, "Fixtures", filename);
        if (File.Exists(primary))
        {
            return File.ReadAllText(primary);
        }

        return File.ReadAllText(Path.Combine(baseDirectory, filename));
    }

    public Task InitializeAsync()
    {
        _handler.Clear();
        return Task.CompletedTask;
    }

    public async Task DisposeAsync()
    {
        await DisposeServiceProviderAsync();
    }
}