Rename Concelier Source modules to Connector
This commit is contained in:
master
@@ -0,0 +1,303 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common.Testing;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Testing;
|
||||
using Xunit;
|
||||
using Xunit.Sdk;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests;
|
||||
|
||||
[Collection("mongo-fixture")]
|
||||
public sealed class RuBduConnectorSnapshotTests : IAsyncLifetime
|
||||
{
|
||||
private const string UpdateFixturesVariable = "UPDATE_BDU_FIXTURES";
|
||||
private static readonly Uri ArchiveUri = new("https://bdu.fstec.ru/files/documents/vulxml.zip");
|
||||
|
||||
private readonly MongoIntegrationFixture _fixture;
|
||||
private ConnectorTestHarness? _harness;
|
||||
|
||||
public RuBduConnectorSnapshotTests(MongoIntegrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_ProducesDeterministicSnapshots()
|
||||
{
|
||||
var harness = await EnsureHarnessAsync();
|
||||
harness.Handler.AddResponse(ArchiveUri, BuildArchiveResponse);
|
||||
|
||||
var connector = harness.ServiceProvider.GetRequiredService<RuBduConnector>();
|
||||
await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None);
|
||||
|
||||
var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
|
||||
var initialState = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(initialState);
|
||||
var cursorBeforeParse = initialState!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(initialState.Cursor);
|
||||
Assert.NotEmpty(cursorBeforeParse.PendingDocuments);
|
||||
var expectedDocumentIds = cursorBeforeParse.PendingDocuments.ToArray();
|
||||
|
||||
await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None);
|
||||
await connector.MapAsync(harness.ServiceProvider, CancellationToken.None);
|
||||
|
||||
var documentsCollection = _fixture.Database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.Document);
|
||||
var documentCount = await documentsCollection.CountDocumentsAsync(Builders<BsonDocument>.Filter.Empty);
|
||||
Assert.True(documentCount > 0, "Expected persisted documents after map stage");
|
||||
|
||||
var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider, expectedDocumentIds);
|
||||
WriteOrAssertSnapshot(documentsSnapshot, "ru-bdu-documents.snapshot.json");
|
||||
|
||||
var dtoSnapshot = await BuildDtoSnapshotAsync(harness.ServiceProvider);
|
||||
WriteOrAssertSnapshot(dtoSnapshot, "ru-bdu-dtos.snapshot.json");
|
||||
|
||||
var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider);
|
||||
WriteOrAssertSnapshot(advisoriesSnapshot, "ru-bdu-advisories.snapshot.json");
|
||||
|
||||
var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider);
|
||||
WriteOrAssertSnapshot(stateSnapshot, "ru-bdu-state.snapshot.json");
|
||||
|
||||
var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests);
|
||||
WriteOrAssertSnapshot(requestsSnapshot, "ru-bdu-requests.snapshot.json");
|
||||
|
||||
harness.Handler.AssertNoPendingResponses();
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_harness is not null)
|
||||
{
|
||||
await _harness.DisposeAsync();
|
||||
_harness = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ConnectorTestHarness> EnsureHarnessAsync()
|
||||
{
|
||||
if (_harness is not null)
|
||||
{
|
||||
return _harness;
|
||||
}
|
||||
|
||||
var initialTime = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero);
|
||||
var harness = new ConnectorTestHarness(_fixture, initialTime, RuBduOptions.HttpClientName);
|
||||
await harness.EnsureServiceProviderAsync(services =>
|
||||
{
|
||||
services.AddLogging(builder =>
|
||||
{
|
||||
builder.ClearProviders();
|
||||
builder.AddProvider(NullLoggerProvider.Instance);
|
||||
});
|
||||
|
||||
services.AddRuBduConnector(options =>
|
||||
{
|
||||
options.BaseAddress = new Uri("https://bdu.fstec.ru/");
|
||||
options.DataArchivePath = "files/documents/vulxml.zip";
|
||||
options.MaxVulnerabilitiesPerFetch = 25;
|
||||
options.RequestTimeout = TimeSpan.FromSeconds(30);
|
||||
var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName, "ru-bdu");
|
||||
Directory.CreateDirectory(cacheRoot);
|
||||
options.CacheDirectory = cacheRoot;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(RuBduOptions.HttpClientName, options =>
|
||||
{
|
||||
options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler);
|
||||
});
|
||||
});
|
||||
|
||||
_harness = harness;
|
||||
return harness;
|
||||
}
|
||||
|
||||
private static HttpResponseMessage BuildArchiveResponse()
|
||||
{
|
||||
var archiveBytes = CreateArchiveBytes();
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new ByteArrayContent(archiveBytes),
|
||||
};
|
||||
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip");
|
||||
response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero);
|
||||
response.Content.Headers.ContentLength = archiveBytes.Length;
|
||||
return response;
|
||||
}
|
||||
|
||||
private async Task<string> BuildDocumentsSnapshotAsync(IServiceProvider provider, IReadOnlyCollection<Guid> documentIds)
|
||||
{
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var records = new List<object>(documentIds.Count);
|
||||
|
||||
foreach (var documentId in documentIds)
|
||||
{
|
||||
var record = await documentStore.FindAsync(documentId, CancellationToken.None);
|
||||
if (record is null)
|
||||
{
|
||||
var existing = await _fixture.Database
|
||||
.GetCollection<BsonDocument>("documents")
|
||||
.Find(Builders<BsonDocument>.Filter.Empty)
|
||||
.Project(Builders<BsonDocument>.Projection.Include("Uri"))
|
||||
.ToListAsync(CancellationToken.None);
|
||||
var uris = existing
|
||||
.Select(document => document.GetValue("Uri", BsonValue.Create(string.Empty)).AsString)
|
||||
.ToArray();
|
||||
throw new XunitException($"Document id not found: {documentId}. Known URIs: {string.Join(", ", uris)}");
|
||||
}
|
||||
|
||||
records.Add(new
|
||||
{
|
||||
record.Uri,
|
||||
record.Status,
|
||||
record.Sha256,
|
||||
Metadata = record.Metadata is null
|
||||
? null
|
||||
: record.Metadata
|
||||
.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase)
|
||||
});
|
||||
}
|
||||
|
||||
var ordered = records
|
||||
.OrderBy(static entry => entry?.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return SnapshotSerializer.ToSnapshot(ordered);
|
||||
}
|
||||
|
||||
private async Task<string> BuildDtoSnapshotAsync(IServiceProvider provider)
|
||||
{
|
||||
var dtoStore = provider.GetRequiredService<IDtoStore>();
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var records = await dtoStore.GetBySourceAsync(RuBduConnectorPlugin.SourceName, 25, CancellationToken.None);
|
||||
|
||||
var entries = new List<object>(records.Count);
|
||||
foreach (var record in records.OrderBy(static r => r.DocumentId))
|
||||
{
|
||||
var document = await documentStore.FindAsync(record.DocumentId, CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
|
||||
var payload = BsonTypeMapper.MapToDotNetValue(record.Payload);
|
||||
entries.Add(new
|
||||
{
|
||||
DocumentUri = document!.Uri,
|
||||
record.SchemaVersion,
|
||||
Payload = payload,
|
||||
});
|
||||
}
|
||||
|
||||
return SnapshotSerializer.ToSnapshot(entries.OrderBy(static entry => entry.GetType().GetProperty("DocumentUri")!.GetValue(entry)?.ToString(), StringComparer.Ordinal).ToArray());
|
||||
}
|
||||
|
||||
private async Task<string> BuildAdvisoriesSnapshotAsync(IServiceProvider provider)
|
||||
{
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(25, CancellationToken.None);
|
||||
var ordered = advisories
|
||||
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
return SnapshotSerializer.ToSnapshot(ordered);
|
||||
}
|
||||
|
||||
private async Task<string> BuildStateSnapshotAsync(IServiceProvider provider)
|
||||
{
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
|
||||
var cursor = state!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor);
|
||||
var snapshot = new
|
||||
{
|
||||
PendingDocuments = cursor.PendingDocuments.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(),
|
||||
PendingMappings = cursor.PendingMappings.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(),
|
||||
LastSuccessfulFetch = cursor.LastSuccessfulFetch?.ToUniversalTime().ToString("O"),
|
||||
};
|
||||
|
||||
return SnapshotSerializer.ToSnapshot(snapshot);
|
||||
}
|
||||
|
||||
private static string BuildRequestsSnapshot(IReadOnlyCollection<CannedHttpMessageHandler.CannedRequestRecord> requests)
|
||||
{
|
||||
var ordered = requests
|
||||
.Select(record => new
|
||||
{
|
||||
Method = record.Method.Method,
|
||||
Uri = record.Uri.ToString(),
|
||||
Headers = record.Headers
|
||||
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(static kvp => kvp.Key, static kvp => kvp.Value, StringComparer.OrdinalIgnoreCase),
|
||||
})
|
||||
.OrderBy(static entry => entry.Uri, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return SnapshotSerializer.ToSnapshot(ordered);
|
||||
}
|
||||
|
||||
private static string ReadFixtureText(string filename)
|
||||
{
|
||||
var path = GetSourceFixturePath(filename);
|
||||
return File.ReadAllText(path, Encoding.UTF8);
|
||||
}
|
||||
|
||||
private static byte[] CreateArchiveBytes()
|
||||
{
|
||||
var xml = ReadFixtureText("export-sample.xml");
|
||||
using var buffer = new MemoryStream();
|
||||
using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true))
|
||||
{
|
||||
var entry = archive.CreateEntry("export/export.xml", CompressionLevel.NoCompression);
|
||||
entry.LastWriteTime = new DateTimeOffset(2025, 10, 14, 9, 0, 0, TimeSpan.Zero);
|
||||
using var entryStream = entry.Open();
|
||||
using var writer = new StreamWriter(entryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
writer.Write(xml);
|
||||
}
|
||||
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
private static bool ShouldUpdateFixtures()
|
||||
=> !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(UpdateFixturesVariable));
|
||||
|
||||
private static void WriteOrAssertSnapshot(string content, string filename)
|
||||
{
|
||||
var path = GetSourceFixturePath(filename);
|
||||
if (ShouldUpdateFixtures())
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
File.WriteAllText(path, content, Encoding.UTF8);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.True(File.Exists(path), $"Snapshot '{filename}' is missing. Run {UpdateFixturesVariable}=1 dotnet test to regenerate fixtures.");
|
||||
var expected = File.ReadAllText(path, Encoding.UTF8);
|
||||
Assert.Equal(expected, content);
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetSourceFixturePath(string relativeName)
|
||||
=> Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", relativeName));
|
||||
}
|
||||
Reference in New Issue
Block a user