Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,163 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Cccs;
|
||||
using StellaOps.Concelier.Connector.Cccs.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Testing;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Tests;
|
||||
|
||||
[Collection("mongo-fixture")]
|
||||
public sealed class CccsConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat");
|
||||
private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type");
|
||||
|
||||
private readonly MongoIntegrationFixture _fixture;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
|
||||
public CccsConnectorTests(MongoIntegrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_ProducesCanonicalAdvisory()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedFeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CccsConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
|
||||
advisories.Should().HaveCount(1);
|
||||
|
||||
var advisory = advisories[0];
|
||||
advisory.AdvisoryKey.Should().Be("TEST-001");
|
||||
advisory.Title.Should().Be("Test Advisory Title");
|
||||
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en");
|
||||
advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0");
|
||||
advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0");
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None);
|
||||
state.Should().NotBeNull();
|
||||
state!.Cursor.Should().NotBeNull();
|
||||
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
|
||||
pendingDocs!.AsBsonArray.Should().BeEmpty();
|
||||
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
|
||||
pendingMappings!.AsBsonArray.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_PersistsRawDocumentWithMetadata()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedFeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CccsConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None);
|
||||
document.Should().NotBeNull();
|
||||
document!.Status.Should().Be(DocumentStatuses.PendingParse);
|
||||
document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en");
|
||||
document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001");
|
||||
document.ContentType.Should().Be("application/json");
|
||||
}
|
||||
|
||||
private async Task<ServiceProvider> BuildServiceProviderAsync()
|
||||
{
|
||||
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
|
||||
_handler.Clear();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
|
||||
services.AddSingleton(_handler);
|
||||
|
||||
services.AddMongoStorage(options =>
|
||||
{
|
||||
options.ConnectionString = _fixture.Runner.ConnectionString;
|
||||
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
});
|
||||
|
||||
services.AddSourceCommon();
|
||||
services.AddCccsConnector(options =>
|
||||
{
|
||||
options.Feeds.Clear();
|
||||
options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri));
|
||||
options.RequestDelay = TimeSpan.Zero;
|
||||
options.MaxEntriesPerFetch = 10;
|
||||
options.MaxKnownEntries = 32;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(CccsOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
|
||||
{
|
||||
builder.PrimaryHandler = _handler;
|
||||
});
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
|
||||
await bootstrapper.InitializeAsync(CancellationToken.None);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private void SeedFeedResponses()
|
||||
{
|
||||
AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json"));
|
||||
AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json"));
|
||||
}
|
||||
|
||||
private void AddJsonResponse(Uri uri, string json, string? etag = null)
|
||||
{
|
||||
_handler.AddResponse(uri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(etag))
|
||||
{
|
||||
response.Headers.ETag = new EntityTagHeaderValue(etag);
|
||||
}
|
||||
|
||||
return response;
|
||||
});
|
||||
}
|
||||
|
||||
private static string ReadFixture(string fileName)
|
||||
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"ERROR": false,
|
||||
"response": [
|
||||
{
|
||||
"nid": 1001,
|
||||
"title": "Test Advisory Title",
|
||||
"uuid": "uuid-test-001",
|
||||
"banner": null,
|
||||
"lang": "en",
|
||||
"date_modified": "2025-08-11",
|
||||
"date_modified_ts": "2025-08-11T12:00:00Z",
|
||||
"date_created": "2025-08-10T15:30:00Z",
|
||||
"summary": "Summary of advisory.",
|
||||
"body": [
|
||||
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
|
||||
],
|
||||
"url": "/en/alerts-advisories/test-advisory",
|
||||
"alert_type": 397,
|
||||
"serial_number": "TEST-001",
|
||||
"subject": "Infrastructure",
|
||||
"moderation_state": "published",
|
||||
"external_url": "https://example.com/external/advisory"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sourceId": "TEST-002-FR",
|
||||
"serialNumber": "TEST-002-FR",
|
||||
"uuid": "uuid-test-002",
|
||||
"language": "fr",
|
||||
"title": "Avis de sécurité – Mise à jour urgente",
|
||||
"summary": "Résumé de l'avis en français.",
|
||||
"canonicalUrl": "https://www.cyber.gc.ca/fr/alertes-avis/test-avis",
|
||||
"externalUrl": "https://exemple.ca/avis",
|
||||
"bodyHtml": "<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>",
|
||||
"bodySegments": [
|
||||
"<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>"
|
||||
],
|
||||
"alertType": "Alerte",
|
||||
"subject": "Infrastructure critique",
|
||||
"banner": null,
|
||||
"published": "2025-08-15T13:45:00Z",
|
||||
"modified": "2025-08-16T09:15:00Z",
|
||||
"rawCreated": "15 août 2025",
|
||||
"rawModified": "2025-08-16T09:15:00Z"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sourceId": "TEST-001",
|
||||
"serialNumber": "TEST-001",
|
||||
"uuid": "uuid-test-001",
|
||||
"language": "en",
|
||||
"title": "Test Advisory Title",
|
||||
"summary": "Summary of advisory.",
|
||||
"canonicalUrl": "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory",
|
||||
"externalUrl": "https://example.com/external/advisory",
|
||||
"bodyHtml": "<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>",
|
||||
"bodySegments": [
|
||||
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
|
||||
],
|
||||
"alertType": "Advisory",
|
||||
"subject": "Infrastructure",
|
||||
"banner": null,
|
||||
"published": "2025-08-10T15:30:00Z",
|
||||
"modified": "2025-08-11T12:00:00Z",
|
||||
"rawCreated": "August 10, 2025",
|
||||
"rawModified": "2025-08-11T12:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"ERROR": false,
|
||||
"response": [
|
||||
{
|
||||
"id": 396,
|
||||
"title": "Advisory"
|
||||
},
|
||||
{
|
||||
"id": 397,
|
||||
"title": "Alert"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Tests.Internal;
|
||||
|
||||
public sealed class CccsHtmlParserTests
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
private static readonly HtmlContentSanitizer Sanitizer = new();
|
||||
private static readonly CccsHtmlParser Parser = new(Sanitizer);
|
||||
|
||||
public CccsHtmlParserTests(ITestOutputHelper output)
|
||||
{
|
||||
_output = output ?? throw new ArgumentNullException(nameof(output));
|
||||
}
|
||||
|
||||
public static IEnumerable<object[]> ParserCases()
|
||||
{
|
||||
yield return new object[]
|
||||
{
|
||||
"cccs-raw-advisory.json",
|
||||
"TEST-001",
|
||||
"en",
|
||||
new[] { "Vendor Widget 1.0", "Vendor Widget 2.0" },
|
||||
new[]
|
||||
{
|
||||
"https://example.com/details",
|
||||
"https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"
|
||||
},
|
||||
new[] { "CVE-2020-1234", "CVE-2021-9999" }
|
||||
};
|
||||
|
||||
yield return new object[]
|
||||
{
|
||||
"cccs-raw-advisory-fr.json",
|
||||
"TEST-002-FR",
|
||||
"fr",
|
||||
new[] { "Produit Exemple 3.1", "Produit Exemple 3.2", "Variante 3.2.1" },
|
||||
new[]
|
||||
{
|
||||
"https://exemple.ca/details",
|
||||
"https://www.cyber.gc.ca/fr/contact-centre-cyber"
|
||||
},
|
||||
new[] { "CVE-2024-1111" }
|
||||
};
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(ParserCases))]
|
||||
public void Parse_ExtractsExpectedFields(
|
||||
string fixtureName,
|
||||
string expectedSerial,
|
||||
string expectedLanguage,
|
||||
string[] expectedProducts,
|
||||
string[] expectedReferenceUrls,
|
||||
string[] expectedCves)
|
||||
{
|
||||
var raw = LoadFixture<CccsRawAdvisoryDocument>(fixtureName);
|
||||
|
||||
var dto = Parser.Parse(raw);
|
||||
|
||||
_output.WriteLine("Products: {0}", string.Join("|", dto.Products));
|
||||
_output.WriteLine("References: {0}", string.Join("|", dto.References.Select(r => $"{r.Url} ({r.Label})")));
|
||||
_output.WriteLine("CVEs: {0}", string.Join("|", dto.CveIds));
|
||||
|
||||
dto.SerialNumber.Should().Be(expectedSerial);
|
||||
dto.Language.Should().Be(expectedLanguage);
|
||||
dto.Products.Should().BeEquivalentTo(expectedProducts);
|
||||
foreach (var url in expectedReferenceUrls)
|
||||
{
|
||||
dto.References.Should().Contain(reference => reference.Url == url);
|
||||
}
|
||||
|
||||
dto.CveIds.Should().BeEquivalentTo(expectedCves);
|
||||
dto.ContentHtml.Should().Contain("<ul>").And.Contain("<li>");
|
||||
dto.ContentHtml.Should().Contain("<h2", because: "heading structure must survive sanitisation for UI rendering");
|
||||
}
|
||||
|
||||
internal static T LoadFixture<T>(string fileName)
|
||||
{
|
||||
var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName);
|
||||
var json = File.ReadAllText(path);
|
||||
return JsonSerializer.Deserialize<T>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using System;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Concelier.Connector.Cccs.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cccs.Tests.Internal;
|
||||
|
||||
public sealed class CccsMapperTests
|
||||
{
|
||||
[Fact]
|
||||
public void Map_CreatesCanonicalAdvisory()
|
||||
{
|
||||
var raw = CccsHtmlParserTests.LoadFixture<CccsRawAdvisoryDocument>("cccs-raw-advisory.json");
|
||||
var dto = new CccsHtmlParser(new HtmlContentSanitizer()).Parse(raw);
|
||||
var document = new DocumentRecord(
|
||||
Guid.NewGuid(),
|
||||
CccsConnectorPlugin.SourceName,
|
||||
dto.CanonicalUrl,
|
||||
DateTimeOffset.UtcNow,
|
||||
"sha-test",
|
||||
DocumentStatuses.PendingMap,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: null,
|
||||
Etag: null,
|
||||
LastModified: dto.Modified,
|
||||
GridFsId: null);
|
||||
|
||||
var recordedAt = DateTimeOffset.Parse("2025-08-12T00:00:00Z");
|
||||
var advisory = CccsMapper.Map(dto, document, recordedAt);
|
||||
|
||||
advisory.AdvisoryKey.Should().Be("TEST-001");
|
||||
advisory.Title.Should().Be(dto.Title);
|
||||
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
|
||||
advisory.References.Should().Contain(reference => reference.Url == dto.CanonicalUrl && reference.Kind == "details");
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
|
||||
advisory.AffectedPackages.Should().HaveCount(2);
|
||||
advisory.Provenance.Should().ContainSingle(p => p.Source == CccsConnectorPlugin.SourceName && p.Kind == "advisory");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="Fixtures\*.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user