using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Testing; using StellaOps.Feedser.Source.Ru.Nkcki; using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Testing; using StellaOps.Feedser.Models; using MongoDB.Driver; using Xunit; namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; [Collection("mongo-fixture")] public sealed class RuNkckiConnectorTests : IAsyncLifetime { private static readonly Uri ListingUri = new("https://cert.gov.ru/materialy/uyazvimosti/"); private static readonly Uri BulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-sample.json.zip"); private readonly MongoIntegrationFixture _fixture; private readonly FakeTimeProvider _timeProvider; private readonly CannedHttpMessageHandler _handler; public RuNkckiConnectorTests(MongoIntegrationFixture fixture) { _fixture = fixture; _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); _handler = new CannedHttpMessageHandler(); } [Fact] public async Task FetchParseMap_ProducesExpectedSnapshot() { await using var provider = await BuildServiceProviderAsync(); SeedListingAndBulletin(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); _timeProvider.Advance(TimeSpan.FromMinutes(1)); await connector.ParseAsync(provider, CancellationToken.None); await connector.MapAsync(provider, CancellationToken.None); var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.Single(advisories); var snapshot = SnapshotSerializer.ToSnapshot(advisories); WriteOrAssertSnapshot(snapshot, "nkcki-advisories.snapshot.json"); var documentStore = provider.GetRequiredService(); var document = await documentStore.FindBySourceAndUriAsync(RuNkckiConnectorPlugin.SourceName, "https://cert.gov.ru/materialy/uyazvimosti/BDU:2025-01001", CancellationToken.None); Assert.NotNull(document); Assert.Equal(DocumentStatuses.Mapped, document!.Status); var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(RuNkckiConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); Assert.True(IsEmptyArray(state!.Cursor, "pendingDocuments")); Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); } [Fact] public async Task Fetch_ReusesCachedBulletinWhenListingFails() { await using var provider = await BuildServiceProviderAsync(); SeedListingAndBulletin(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); _handler.Clear(); _handler.AddResponse(ListingUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) { Content = new StringContent("error", Encoding.UTF8, "text/plain"), }); var advisoryStore = provider.GetRequiredService(); var before = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.NotEmpty(before); await connector.FetchAsync(provider, CancellationToken.None); var after = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.Equal(before.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key), after.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key)); _handler.AssertNoPendingResponses(); } private async Task BuildServiceProviderAsync() { try { await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); } catch (MongoConnectionException ex) { Assert.Skip($"Mongo runner unavailable: {ex.Message}"); } catch (TimeoutException ex) { Assert.Skip($"Mongo runner unavailable: {ex.Message}"); } _handler.Clear(); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); services.AddSingleton(_timeProvider); services.AddMongoStorage(options => { options.ConnectionString = _fixture.Runner.ConnectionString; options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; options.CommandTimeout = TimeSpan.FromSeconds(5); }); services.AddSourceCommon(); services.AddRuNkckiConnector(options => { options.BaseAddress = new Uri("https://cert.gov.ru/"); options.ListingPath = "/materialy/uyazvimosti/"; options.MaxBulletinsPerFetch = 2; options.MaxVulnerabilitiesPerFetch = 50; var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName); Directory.CreateDirectory(cacheRoot); options.CacheDirectory = Path.Combine(cacheRoot, "ru-nkcki"); options.RequestDelay = TimeSpan.Zero; }); services.Configure(RuNkckiOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); }); try { var provider = services.BuildServiceProvider(); var bootstrapper = provider.GetRequiredService(); await bootstrapper.InitializeAsync(CancellationToken.None); return provider; } catch (MongoConnectionException ex) { Assert.Skip($"Mongo runner unavailable: {ex.Message}"); throw; // Unreachable } catch (TimeoutException ex) { Assert.Skip($"Mongo runner unavailable: {ex.Message}"); throw; } } private void SeedListingAndBulletin() { var listingHtml = ReadFixture("listing.html"); _handler.AddTextResponse(ListingUri, listingHtml, "text/html"); var bulletinBytes = ReadBulletinFixture("bulletin-sample.json.zip"); _handler.AddResponse(BulletinUri, () => { var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(bulletinBytes), }; response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); response.Content.Headers.LastModified = new DateTimeOffset(2025, 9, 22, 0, 0, 0, TimeSpan.Zero); return response; }); } private static bool IsEmptyArray(BsonDocument document, string field) { if (!document.TryGetValue(field, out var value) || value is not BsonArray array) { return false; } return array.Count == 0; } private static string ReadFixture(string filename) { var path = Path.Combine("Fixtures", filename); var resolved = ResolveFixturePath(path); return File.ReadAllText(resolved); } private static byte[] ReadBulletinFixture(string filename) { var path = Path.Combine("Fixtures", filename); var resolved = ResolveFixturePath(path); return File.ReadAllBytes(resolved); } private static string ResolveFixturePath(string relativePath) { var projectRoot = GetProjectRoot(); var projectPath = Path.Combine(projectRoot, relativePath); if (File.Exists(projectPath)) { return projectPath; } var binaryPath = Path.Combine(AppContext.BaseDirectory, relativePath); if (File.Exists(binaryPath)) { return Path.GetFullPath(binaryPath); } throw new FileNotFoundException($"Fixture not found: {relativePath}"); } private static void WriteOrAssertSnapshot(string snapshot, string filename) { if (ShouldUpdateFixtures()) { var path = GetWritableFixturePath(filename); Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllText(path, snapshot); return; } var expectedPath = ResolveFixturePath(Path.Combine("Fixtures", filename)); if (!File.Exists(expectedPath)) { throw new FileNotFoundException($"Expected snapshot missing: {expectedPath}. Set UPDATE_NKCKI_FIXTURES=1 to generate."); } var expected = File.ReadAllText(expectedPath); Assert.Equal(Normalize(expected), Normalize(snapshot)); } private static string GetWritableFixturePath(string filename) { var projectRoot = GetProjectRoot(); return Path.Combine(projectRoot, "Fixtures", filename); } private static bool ShouldUpdateFixtures() { var value = Environment.GetEnvironmentVariable("UPDATE_NKCKI_FIXTURES"); return string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); } private static string Normalize(string text) => text.Replace("\r\n", "\n", StringComparison.Ordinal); private static string GetProjectRoot() { var current = AppContext.BaseDirectory; while (!string.IsNullOrEmpty(current)) { var candidate = Path.Combine(current, "StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj"); if (File.Exists(candidate)) { return current; } current = Path.GetDirectoryName(current); } throw new InvalidOperationException("Unable to locate project root for Ru.Nkcki tests."); } public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() => await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); }