using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Driver; using StellaOps.Concelier.Models; using StellaOps.Concelier.Connector.Common.Testing; using StellaOps.Concelier.Connector.Ru.Bdu; using StellaOps.Concelier.Connector.Ru.Bdu.Configuration; using StellaOps.Concelier.Connector.Ru.Bdu.Internal; using StellaOps.Concelier.Storage.Mongo; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Concelier.Testing; using Xunit; using Xunit.Sdk; namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests; [Collection("mongo-fixture")] public sealed class RuBduConnectorSnapshotTests : IAsyncLifetime { private const string UpdateFixturesVariable = "UPDATE_BDU_FIXTURES"; private static readonly Uri ArchiveUri = new("https://bdu.fstec.ru/files/documents/vulxml.zip"); private readonly MongoIntegrationFixture _fixture; private ConnectorTestHarness? _harness; public RuBduConnectorSnapshotTests(MongoIntegrationFixture fixture) { _fixture = fixture; } [Fact] public async Task FetchParseMap_ProducesDeterministicSnapshots() { var harness = await EnsureHarnessAsync(); harness.Handler.AddResponse(ArchiveUri, BuildArchiveResponse); var connector = harness.ServiceProvider.GetRequiredService(); await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); var stateRepository = harness.ServiceProvider.GetRequiredService(); var initialState = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(initialState); var cursorBeforeParse = initialState!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(initialState.Cursor); Assert.NotEmpty(cursorBeforeParse.PendingDocuments); var expectedDocumentIds = cursorBeforeParse.PendingDocuments.ToArray(); await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); var documentsCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Document); var documentCount = await documentsCollection.CountDocumentsAsync(Builders.Filter.Empty); Assert.True(documentCount > 0, "Expected persisted documents after map stage"); var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider, expectedDocumentIds); WriteOrAssertSnapshot(documentsSnapshot, "ru-bdu-documents.snapshot.json"); var dtoSnapshot = await BuildDtoSnapshotAsync(harness.ServiceProvider); WriteOrAssertSnapshot(dtoSnapshot, "ru-bdu-dtos.snapshot.json"); var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); WriteOrAssertSnapshot(advisoriesSnapshot, "ru-bdu-advisories.snapshot.json"); var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); WriteOrAssertSnapshot(stateSnapshot, "ru-bdu-state.snapshot.json"); var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); WriteOrAssertSnapshot(requestsSnapshot, "ru-bdu-requests.snapshot.json"); harness.Handler.AssertNoPendingResponses(); } public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() { if (_harness is not null) { await _harness.DisposeAsync(); _harness = null; } } private async Task EnsureHarnessAsync() { if (_harness is not null) { return _harness; } var initialTime = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero); var harness = new ConnectorTestHarness(_fixture, initialTime, RuBduOptions.HttpClientName); await harness.EnsureServiceProviderAsync(services => { services.AddLogging(builder => { builder.ClearProviders(); builder.AddProvider(NullLoggerProvider.Instance); }); services.AddRuBduConnector(options => { options.BaseAddress = new Uri("https://bdu.fstec.ru/"); options.DataArchivePath = "files/documents/vulxml.zip"; options.MaxVulnerabilitiesPerFetch = 25; options.RequestTimeout = TimeSpan.FromSeconds(30); var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName, "ru-bdu"); Directory.CreateDirectory(cacheRoot); options.CacheDirectory = cacheRoot; }); services.Configure(RuBduOptions.HttpClientName, options => { options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); }); }); _harness = harness; return harness; } private static HttpResponseMessage BuildArchiveResponse() { var archiveBytes = CreateArchiveBytes(); var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(archiveBytes), }; response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero); response.Content.Headers.ContentLength = archiveBytes.Length; return response; } private async Task BuildDocumentsSnapshotAsync(IServiceProvider provider, IReadOnlyCollection documentIds) { var documentStore = provider.GetRequiredService(); var records = new List(documentIds.Count); foreach (var documentId in documentIds) { var record = await documentStore.FindAsync(documentId, CancellationToken.None); if (record is null) { var existing = await _fixture.Database .GetCollection("documents") .Find(Builders.Filter.Empty) .Project(Builders.Projection.Include("Uri")) .ToListAsync(CancellationToken.None); var uris = existing .Select(document => document.GetValue("Uri", BsonValue.Create(string.Empty)).AsString) .ToArray(); throw new XunitException($"Document id not found: {documentId}. Known URIs: {string.Join(", ", uris)}"); } records.Add(new { record.Uri, record.Status, record.Sha256, Metadata = record.Metadata is null ? null : record.Metadata .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase) }); } var ordered = records .OrderBy(static entry => entry?.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) .ToArray(); return SnapshotSerializer.ToSnapshot(ordered); } private async Task BuildDtoSnapshotAsync(IServiceProvider provider) { var dtoStore = provider.GetRequiredService(); var documentStore = provider.GetRequiredService(); var records = await dtoStore.GetBySourceAsync(RuBduConnectorPlugin.SourceName, 25, CancellationToken.None); var entries = new List(records.Count); foreach (var record in records.OrderBy(static r => r.DocumentId)) { var document = await documentStore.FindAsync(record.DocumentId, CancellationToken.None); Assert.NotNull(document); var payload = BsonTypeMapper.MapToDotNetValue(record.Payload); entries.Add(new { DocumentUri = document!.Uri, record.SchemaVersion, Payload = payload, }); } return SnapshotSerializer.ToSnapshot(entries.OrderBy(static entry => entry.GetType().GetProperty("DocumentUri")!.GetValue(entry)?.ToString(), StringComparer.Ordinal).ToArray()); } private async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) { var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(25, CancellationToken.None); var ordered = advisories .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) .ToArray(); return SnapshotSerializer.ToSnapshot(ordered); } private async Task BuildStateSnapshotAsync(IServiceProvider provider) { var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); var cursor = state!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor); var snapshot = new { PendingDocuments = cursor.PendingDocuments.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), PendingMappings = cursor.PendingMappings.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), LastSuccessfulFetch = cursor.LastSuccessfulFetch?.ToUniversalTime().ToString("O"), }; return SnapshotSerializer.ToSnapshot(snapshot); } private static string BuildRequestsSnapshot(IReadOnlyCollection requests) { var ordered = requests .Select(record => new { Method = record.Method.Method, Uri = record.Uri.ToString(), Headers = record.Headers .OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase) .ToDictionary(static kvp => kvp.Key, static kvp => kvp.Value, StringComparer.OrdinalIgnoreCase), }) .OrderBy(static entry => entry.Uri, StringComparer.Ordinal) .ToArray(); return SnapshotSerializer.ToSnapshot(ordered); } private static string ReadFixtureText(string filename) { var path = GetSourceFixturePath(filename); return File.ReadAllText(path, Encoding.UTF8); } private static byte[] CreateArchiveBytes() { var xml = ReadFixtureText("export-sample.xml"); using var buffer = new MemoryStream(); using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) { var entry = archive.CreateEntry("export/export.xml", CompressionLevel.NoCompression); entry.LastWriteTime = new DateTimeOffset(2025, 10, 14, 9, 0, 0, TimeSpan.Zero); using var entryStream = entry.Open(); using var writer = new StreamWriter(entryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); writer.Write(xml); } return buffer.ToArray(); } private static bool ShouldUpdateFixtures() => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(UpdateFixturesVariable)); private static void WriteOrAssertSnapshot(string content, string filename) { var path = GetSourceFixturePath(filename); if (ShouldUpdateFixtures()) { Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllText(path, content, Encoding.UTF8); } else { Assert.True(File.Exists(path), $"Snapshot '{filename}' is missing. Run {UpdateFixturesVariable}=1 dotnet test to regenerate fixtures."); var expected = File.ReadAllText(path, Encoding.UTF8); Assert.Equal(expected, content); } } private static string GetSourceFixturePath(string relativeName) => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", relativeName)); }