304 lines
13 KiB
C#
304 lines
13 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Http;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using MongoDB.Bson;
|
|
using MongoDB.Bson.Serialization;
|
|
using MongoDB.Driver;
|
|
using StellaOps.Concelier.Models;
|
|
using StellaOps.Concelier.Connector.Common.Testing;
|
|
using StellaOps.Concelier.Connector.Ru.Bdu;
|
|
using StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
|
using StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
|
using StellaOps.Concelier.Storage.Mongo;
|
|
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
|
using StellaOps.Concelier.Storage.Mongo.Documents;
|
|
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
|
using StellaOps.Concelier.Testing;
|
|
using Xunit;
|
|
using Xunit.Sdk;
|
|
|
|
namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests;
|
|
|
|
[Collection("mongo-fixture")]
|
|
public sealed class RuBduConnectorSnapshotTests : IAsyncLifetime
|
|
{
|
|
private const string UpdateFixturesVariable = "UPDATE_BDU_FIXTURES";
|
|
private static readonly Uri ArchiveUri = new("https://bdu.fstec.ru/files/documents/vulxml.zip");
|
|
|
|
private readonly MongoIntegrationFixture _fixture;
|
|
private ConnectorTestHarness? _harness;
|
|
|
|
public RuBduConnectorSnapshotTests(MongoIntegrationFixture fixture)
|
|
{
|
|
_fixture = fixture;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FetchParseMap_ProducesDeterministicSnapshots()
|
|
{
|
|
var harness = await EnsureHarnessAsync();
|
|
harness.Handler.AddResponse(ArchiveUri, BuildArchiveResponse);
|
|
|
|
var connector = harness.ServiceProvider.GetRequiredService<RuBduConnector>();
|
|
await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None);
|
|
|
|
var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
|
|
var initialState = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None);
|
|
Assert.NotNull(initialState);
|
|
var cursorBeforeParse = initialState!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(initialState.Cursor);
|
|
Assert.NotEmpty(cursorBeforeParse.PendingDocuments);
|
|
var expectedDocumentIds = cursorBeforeParse.PendingDocuments.ToArray();
|
|
|
|
await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None);
|
|
await connector.MapAsync(harness.ServiceProvider, CancellationToken.None);
|
|
|
|
var documentsCollection = _fixture.Database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.Document);
|
|
var documentCount = await documentsCollection.CountDocumentsAsync(Builders<BsonDocument>.Filter.Empty);
|
|
Assert.True(documentCount > 0, "Expected persisted documents after map stage");
|
|
|
|
var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider, expectedDocumentIds);
|
|
WriteOrAssertSnapshot(documentsSnapshot, "ru-bdu-documents.snapshot.json");
|
|
|
|
var dtoSnapshot = await BuildDtoSnapshotAsync(harness.ServiceProvider);
|
|
WriteOrAssertSnapshot(dtoSnapshot, "ru-bdu-dtos.snapshot.json");
|
|
|
|
var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider);
|
|
WriteOrAssertSnapshot(advisoriesSnapshot, "ru-bdu-advisories.snapshot.json");
|
|
|
|
var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider);
|
|
WriteOrAssertSnapshot(stateSnapshot, "ru-bdu-state.snapshot.json");
|
|
|
|
var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests);
|
|
WriteOrAssertSnapshot(requestsSnapshot, "ru-bdu-requests.snapshot.json");
|
|
|
|
harness.Handler.AssertNoPendingResponses();
|
|
}
|
|
|
|
public Task InitializeAsync() => Task.CompletedTask;
|
|
|
|
public async Task DisposeAsync()
|
|
{
|
|
if (_harness is not null)
|
|
{
|
|
await _harness.DisposeAsync();
|
|
_harness = null;
|
|
}
|
|
}
|
|
|
|
private async Task<ConnectorTestHarness> EnsureHarnessAsync()
|
|
{
|
|
if (_harness is not null)
|
|
{
|
|
return _harness;
|
|
}
|
|
|
|
var initialTime = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero);
|
|
var harness = new ConnectorTestHarness(_fixture, initialTime, RuBduOptions.HttpClientName);
|
|
await harness.EnsureServiceProviderAsync(services =>
|
|
{
|
|
services.AddLogging(builder =>
|
|
{
|
|
builder.ClearProviders();
|
|
builder.AddProvider(NullLoggerProvider.Instance);
|
|
});
|
|
|
|
services.AddRuBduConnector(options =>
|
|
{
|
|
options.BaseAddress = new Uri("https://bdu.fstec.ru/");
|
|
options.DataArchivePath = "files/documents/vulxml.zip";
|
|
options.MaxVulnerabilitiesPerFetch = 25;
|
|
options.RequestTimeout = TimeSpan.FromSeconds(30);
|
|
var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName, "ru-bdu");
|
|
Directory.CreateDirectory(cacheRoot);
|
|
options.CacheDirectory = cacheRoot;
|
|
});
|
|
|
|
services.Configure<HttpClientFactoryOptions>(RuBduOptions.HttpClientName, options =>
|
|
{
|
|
options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler);
|
|
});
|
|
});
|
|
|
|
_harness = harness;
|
|
return harness;
|
|
}
|
|
|
|
private static HttpResponseMessage BuildArchiveResponse()
|
|
{
|
|
var archiveBytes = CreateArchiveBytes();
|
|
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
|
{
|
|
Content = new ByteArrayContent(archiveBytes),
|
|
};
|
|
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip");
|
|
response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero);
|
|
response.Content.Headers.ContentLength = archiveBytes.Length;
|
|
return response;
|
|
}
|
|
|
|
private async Task<string> BuildDocumentsSnapshotAsync(IServiceProvider provider, IReadOnlyCollection<Guid> documentIds)
|
|
{
|
|
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
|
var records = new List<object>(documentIds.Count);
|
|
|
|
foreach (var documentId in documentIds)
|
|
{
|
|
var record = await documentStore.FindAsync(documentId, CancellationToken.None);
|
|
if (record is null)
|
|
{
|
|
var existing = await _fixture.Database
|
|
.GetCollection<BsonDocument>("documents")
|
|
.Find(Builders<BsonDocument>.Filter.Empty)
|
|
.Project(Builders<BsonDocument>.Projection.Include("Uri"))
|
|
.ToListAsync(CancellationToken.None);
|
|
var uris = existing
|
|
.Select(document => document.GetValue("Uri", BsonValue.Create(string.Empty)).AsString)
|
|
.ToArray();
|
|
throw new XunitException($"Document id not found: {documentId}. Known URIs: {string.Join(", ", uris)}");
|
|
}
|
|
|
|
records.Add(new
|
|
{
|
|
record.Uri,
|
|
record.Status,
|
|
record.Sha256,
|
|
Metadata = record.Metadata is null
|
|
? null
|
|
: record.Metadata
|
|
.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase)
|
|
});
|
|
}
|
|
|
|
var ordered = records
|
|
.OrderBy(static entry => entry?.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
return SnapshotSerializer.ToSnapshot(ordered);
|
|
}
|
|
|
|
private async Task<string> BuildDtoSnapshotAsync(IServiceProvider provider)
|
|
{
|
|
var dtoStore = provider.GetRequiredService<IDtoStore>();
|
|
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
|
var records = await dtoStore.GetBySourceAsync(RuBduConnectorPlugin.SourceName, 25, CancellationToken.None);
|
|
|
|
var entries = new List<object>(records.Count);
|
|
foreach (var record in records.OrderBy(static r => r.DocumentId))
|
|
{
|
|
var document = await documentStore.FindAsync(record.DocumentId, CancellationToken.None);
|
|
Assert.NotNull(document);
|
|
|
|
var payload = BsonTypeMapper.MapToDotNetValue(record.Payload);
|
|
entries.Add(new
|
|
{
|
|
DocumentUri = document!.Uri,
|
|
record.SchemaVersion,
|
|
Payload = payload,
|
|
});
|
|
}
|
|
|
|
return SnapshotSerializer.ToSnapshot(entries.OrderBy(static entry => entry.GetType().GetProperty("DocumentUri")!.GetValue(entry)?.ToString(), StringComparer.Ordinal).ToArray());
|
|
}
|
|
|
|
private async Task<string> BuildAdvisoriesSnapshotAsync(IServiceProvider provider)
|
|
{
|
|
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
|
var advisories = await advisoryStore.GetRecentAsync(25, CancellationToken.None);
|
|
var ordered = advisories
|
|
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
|
|
.ToArray();
|
|
return SnapshotSerializer.ToSnapshot(ordered);
|
|
}
|
|
|
|
private async Task<string> BuildStateSnapshotAsync(IServiceProvider provider)
|
|
{
|
|
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
|
var state = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None);
|
|
Assert.NotNull(state);
|
|
|
|
var cursor = state!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor);
|
|
var snapshot = new
|
|
{
|
|
PendingDocuments = cursor.PendingDocuments.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(),
|
|
PendingMappings = cursor.PendingMappings.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(),
|
|
LastSuccessfulFetch = cursor.LastSuccessfulFetch?.ToUniversalTime().ToString("O"),
|
|
};
|
|
|
|
return SnapshotSerializer.ToSnapshot(snapshot);
|
|
}
|
|
|
|
private static string BuildRequestsSnapshot(IReadOnlyCollection<CannedHttpMessageHandler.CannedRequestRecord> requests)
|
|
{
|
|
var ordered = requests
|
|
.Select(record => new
|
|
{
|
|
Method = record.Method.Method,
|
|
Uri = record.Uri.ToString(),
|
|
Headers = record.Headers
|
|
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(static kvp => kvp.Key, static kvp => kvp.Value, StringComparer.OrdinalIgnoreCase),
|
|
})
|
|
.OrderBy(static entry => entry.Uri, StringComparer.Ordinal)
|
|
.ToArray();
|
|
|
|
return SnapshotSerializer.ToSnapshot(ordered);
|
|
}
|
|
|
|
private static string ReadFixtureText(string filename)
|
|
{
|
|
var path = GetSourceFixturePath(filename);
|
|
return File.ReadAllText(path, Encoding.UTF8);
|
|
}
|
|
|
|
private static byte[] CreateArchiveBytes()
|
|
{
|
|
var xml = ReadFixtureText("export-sample.xml");
|
|
using var buffer = new MemoryStream();
|
|
using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true))
|
|
{
|
|
var entry = archive.CreateEntry("export/export.xml", CompressionLevel.NoCompression);
|
|
entry.LastWriteTime = new DateTimeOffset(2025, 10, 14, 9, 0, 0, TimeSpan.Zero);
|
|
using var entryStream = entry.Open();
|
|
using var writer = new StreamWriter(entryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
|
writer.Write(xml);
|
|
}
|
|
|
|
return buffer.ToArray();
|
|
}
|
|
|
|
private static bool ShouldUpdateFixtures()
|
|
=> !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(UpdateFixturesVariable));
|
|
|
|
private static void WriteOrAssertSnapshot(string content, string filename)
|
|
{
|
|
var path = GetSourceFixturePath(filename);
|
|
if (ShouldUpdateFixtures())
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
File.WriteAllText(path, content, Encoding.UTF8);
|
|
}
|
|
else
|
|
{
|
|
Assert.True(File.Exists(path), $"Snapshot '{filename}' is missing. Run {UpdateFixturesVariable}=1 dotnet test to regenerate fixtures.");
|
|
var expected = File.ReadAllText(path, Encoding.UTF8);
|
|
Assert.Equal(expected, content);
|
|
}
|
|
}
|
|
|
|
private static string GetSourceFixturePath(string relativeName)
|
|
=> Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", relativeName));
|
|
}
|