using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using StellaOps.Feedser.Source.CertCc; using StellaOps.Feedser.Source.CertCc.Configuration; using StellaOps.Feedser.Source.CertCc.Internal; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Cursors; using StellaOps.Feedser.Source.Common.Testing; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Testing; using Xunit; namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; [Collection("mongo-fixture")] public sealed class CertCcConnectorFetchTests : IAsyncLifetime { private const string TestNoteId = "294418"; private readonly MongoIntegrationFixture _fixture; private readonly FakeTimeProvider _timeProvider; private readonly CannedHttpMessageHandler _handler; private ServiceProvider? _serviceProvider; public CertCcConnectorFetchTests(MongoIntegrationFixture fixture) { _fixture = fixture; _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero)); _handler = new CannedHttpMessageHandler(); } [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")] public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor() { var template = new CertCcOptions { BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute), SummaryWindow = new TimeWindowCursorOptions { WindowSize = TimeSpan.FromDays(30), Overlap = TimeSpan.FromDays(5), InitialBackfill = TimeSpan.FromDays(60), MinimumWindowSize = TimeSpan.FromDays(1), }, MaxMonthlySummaries = 3, MaxNotesPerFetch = 3, DetailRequestDelay = TimeSpan.Zero, }; await EnsureServiceProviderAsync(template); var provider = _serviceProvider!; _handler.Clear(); var planner = provider.GetRequiredService(); var plan = planner.CreatePlan(state: null); Assert.NotEmpty(plan.Requests); foreach (var request in plan.Requests) { _handler.AddJsonResponse(request.Uri, BuildSummaryPayload()); } RegisterDetailResponses(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); var documentStore = provider.GetRequiredService(); foreach (var request in plan.Requests) { var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None); Assert.NotNull(record); Assert.Equal(DocumentStatuses.PendingParse, record!.Status); Assert.NotNull(record.Metadata); Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]); Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]); if (request.Month.HasValue) { Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]); } else { Assert.False(record.Metadata.ContainsKey("certcc.month")); } } foreach (var uri in EnumerateDetailUris()) { var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); Assert.NotNull(record); Assert.Equal(DocumentStatuses.PendingParse, record!.Status); Assert.NotNull(record.Metadata); Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]); } var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); BsonValue summaryValue; Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue)); var summaryDocument = Assert.IsType(summaryValue); Assert.True(summaryDocument.TryGetValue("start", out _)); Assert.True(summaryDocument.TryGetValue("end", out _)); var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) ? pendingNotesValue.AsBsonArray.Count : 0; Assert.Equal(0, pendingNotesCount); var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) ? pendingSummariesValue.AsBsonArray.Count : 0; Assert.Equal(0, pendingSummariesCount); Assert.True(state.Cursor.TryGetValue("lastRun", out _)); Assert.True(_handler.Requests.Count >= plan.Requests.Count); foreach (var request in _handler.Requests) { if (request.Headers.TryGetValue("Accept", out var accept)) { Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase); } } } private static string BuildSummaryPayload() { return $$""" { "count": 1, "notes": [ "VU#{TestNoteId}" ] } """; } private void RegisterDetailResponses() { foreach (var uri in EnumerateDetailUris()) { var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase) ? "vu-294418-vendors.json" : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase) ? "vu-294418-vuls.json" : "vu-294418.json"; _handler.AddJsonResponse(uri, ReadFixture(fixtureName)); } } private static IEnumerable EnumerateDetailUris() { var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); yield return new Uri(baseUri, $"{TestNoteId}/"); yield return new Uri(baseUri, $"{TestNoteId}/vendors/"); yield return new Uri(baseUri, $"{TestNoteId}/vuls/"); } private async Task EnsureServiceProviderAsync(CertCcOptions template) { await DisposeServiceProviderAsync(); await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); services.AddSingleton(_timeProvider); services.AddSingleton(_handler); services.AddMongoStorage(options => { options.ConnectionString = _fixture.Runner.ConnectionString; options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; options.CommandTimeout = TimeSpan.FromSeconds(5); options.RawDocumentRetention = TimeSpan.Zero; options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5); options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1); }); services.AddSourceCommon(); services.AddCertCcConnector(options => { options.BaseApiUri = template.BaseApiUri; options.SummaryWindow = new TimeWindowCursorOptions { WindowSize = template.SummaryWindow.WindowSize, Overlap = template.SummaryWindow.Overlap, InitialBackfill = template.SummaryWindow.InitialBackfill, MinimumWindowSize = template.SummaryWindow.MinimumWindowSize, }; options.MaxMonthlySummaries = template.MaxMonthlySummaries; options.MaxNotesPerFetch = template.MaxNotesPerFetch; options.DetailRequestDelay = template.DetailRequestDelay; options.EnableDetailMapping = template.EnableDetailMapping; }); services.Configure(CertCcOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); }); _serviceProvider = services.BuildServiceProvider(); var bootstrapper = _serviceProvider.GetRequiredService(); await bootstrapper.InitializeAsync(CancellationToken.None); } private async Task DisposeServiceProviderAsync() { if (_serviceProvider is null) { return; } if (_serviceProvider is IAsyncDisposable asyncDisposable) { await asyncDisposable.DisposeAsync(); } else { _serviceProvider.Dispose(); } _serviceProvider = null; } private static string ReadFixture(string filename) { var baseDirectory = AppContext.BaseDirectory; var primary = Path.Combine(baseDirectory, "Fixtures", filename); if (File.Exists(primary)) { return File.ReadAllText(primary); } return File.ReadAllText(Path.Combine(baseDirectory, filename)); } public Task InitializeAsync() { _handler.Clear(); return Task.CompletedTask; } public async Task DisposeAsync() { await DisposeServiceProviderAsync(); } }