using System; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using MongoDB.Bson; using StellaOps.Concelier.Connector.Cccs; using StellaOps.Concelier.Connector.Cccs.Configuration; using StellaOps.Concelier.Connector.Common; using StellaOps.Concelier.Connector.Common.Http; using StellaOps.Concelier.Connector.Common.Testing; using StellaOps.Concelier.Storage.Mongo; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; using StellaOps.Concelier.Testing; using Xunit; namespace StellaOps.Concelier.Connector.Cccs.Tests; [Collection("mongo-fixture")] public sealed class CccsConnectorTests : IAsyncLifetime { private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"); private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type"); private readonly MongoIntegrationFixture _fixture; private readonly CannedHttpMessageHandler _handler; public CccsConnectorTests(MongoIntegrationFixture fixture) { _fixture = fixture; _handler = new CannedHttpMessageHandler(); } [Fact] public async Task FetchParseMap_ProducesCanonicalAdvisory() { await using var provider = await BuildServiceProviderAsync(); SeedFeedResponses(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); await connector.ParseAsync(provider, CancellationToken.None); await connector.MapAsync(provider, CancellationToken.None); var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); advisories.Should().HaveCount(1); var advisory = advisories[0]; advisory.AdvisoryKey.Should().Be("TEST-001"); advisory.Title.Should().Be("Test Advisory Title"); advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" }); advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details"); advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"); advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0"); advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0"); var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None); state.Should().NotBeNull(); state!.Cursor.Should().NotBeNull(); state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); pendingDocs!.AsBsonArray.Should().BeEmpty(); state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); pendingMappings!.AsBsonArray.Should().BeEmpty(); } [Fact] public async Task Fetch_PersistsRawDocumentWithMetadata() { await using var provider = await BuildServiceProviderAsync(); SeedFeedResponses(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); var documentStore = provider.GetRequiredService(); var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None); document.Should().NotBeNull(); document!.Status.Should().Be(DocumentStatuses.PendingParse); document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en"); document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001"); document.ContentType.Should().Be("application/json"); } private async Task BuildServiceProviderAsync() { await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); _handler.Clear(); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); services.AddSingleton(_handler); services.AddMongoStorage(options => { options.ConnectionString = _fixture.Runner.ConnectionString; options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; options.CommandTimeout = TimeSpan.FromSeconds(5); }); services.AddSourceCommon(); services.AddCccsConnector(options => { options.Feeds.Clear(); options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri)); options.RequestDelay = TimeSpan.Zero; options.MaxEntriesPerFetch = 10; options.MaxKnownEntries = 32; }); services.Configure(CccsOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => { builder.PrimaryHandler = _handler; }); }); var provider = services.BuildServiceProvider(); var bootstrapper = provider.GetRequiredService(); await bootstrapper.InitializeAsync(CancellationToken.None); return provider; } private void SeedFeedResponses() { AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json")); AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json")); } private void AddJsonResponse(Uri uri, string json, string? etag = null) { _handler.AddResponse(uri, () => { var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent(json, Encoding.UTF8, "application/json"), }; if (!string.IsNullOrWhiteSpace(etag)) { response.Headers.ETag = new EntityTagHeaderValue(etag); } return response; }); } private static string ReadFixture(string fileName) => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); public Task InitializeAsync() => Task.CompletedTask; public Task DisposeAsync() => Task.CompletedTask; }