stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 1 Releases Wiki Activity

Initial commit (history squashed)

Browse Source
This commit is contained in:
master
2025-10-07 10:14:21 +03:00
commit 016c5a3fe7
1132 changed files with 117842 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs Normal file
View File

@@ -0,0 +1,313 @@
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Feedser.Source.CertFr;
using StellaOps.Feedser.Source.CertFr.Configuration;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common.Testing;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Feedser.Testing;
using StellaOps.Feedser.Models;
namespace StellaOps.Feedser.Source.CertFr.Tests;
[Collection("mongo-fixture")]
public sealed class CertFrConnectorTests : IAsyncLifetime
{
private static readonly Uri FeedUri = new("https://www.cert.ssi.gouv.fr/feed/alertes/");
private static readonly Uri FirstDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/");
private static readonly Uri SecondDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/");
private readonly MongoIntegrationFixture _fixture;
private readonly FakeTimeProvider _timeProvider;
private readonly CannedHttpMessageHandler _handler;
public CertFrConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 10, 3, 0, 0, 0, TimeSpan.Zero));
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_ProducesDeterministicSnapshot()
{
await using var provider = await BuildServiceProviderAsync();
SeedFeed();
SeedDetailResponses();
var connector = provider.GetRequiredService<CertFrConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
_timeProvider.Advance(TimeSpan.FromMinutes(1));
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
Assert.Equal(2, advisories.Count);
var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray());
var expected = ReadFixture("certfr-advisories.snapshot.json");
var normalizedSnapshot = Normalize(snapshot);
var normalizedExpected = Normalize(expected);
if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal))
{
var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", "certfr-advisories.actual.json");
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
File.WriteAllText(actualPath, snapshot);
}
Assert.Equal(normalizedExpected, normalizedSnapshot);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status);
var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0);
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0);
}
[Fact]
public async Task FetchFailure_RecordsBackoffAndReason()
{
await using var provider = await BuildServiceProviderAsync();
_handler.AddResponse(FeedUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = new StringContent("feed error", Encoding.UTF8, "text/plain"),
});
var connector = provider.GetRequiredService<CertFrConnector>();
await Assert.ThrowsAsync<HttpRequestException>(() => connector.FetchAsync(provider, CancellationToken.None));
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.Equal(1, state!.FailCount);
Assert.NotNull(state.LastFailureReason);
Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal);
Assert.NotNull(state.BackoffUntil);
Assert.True(state.BackoffUntil > _timeProvider.GetUtcNow());
}
[Fact]
public async Task Fetch_NotModifiedResponsesMaintainDocumentState()
{
await using var provider = await BuildServiceProviderAsync();
SeedFeed();
SeedDetailResponses();
var connector = provider.GetRequiredService<CertFrConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status);
var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status);
SeedFeed();
SeedNotModifiedDetailResponses();
await connector.FetchAsync(provider, CancellationToken.None);
firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status);
secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0);
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0);
}
[Fact]
public async Task Fetch_DuplicateContentSkipsRequeue()
{
await using var provider = await BuildServiceProviderAsync();
SeedFeed();
SeedDetailResponses();
var connector = provider.GetRequiredService<CertFrConnector>();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status);
var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status);
SeedFeed();
SeedDetailResponses();
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status);
secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0);
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0);
}
private async Task<ServiceProvider> BuildServiceProviderAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
_handler.Clear();
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton<TimeProvider>(_timeProvider);
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddCertFrConnector(opts =>
{
opts.FeedUri = FeedUri;
opts.InitialBackfill = TimeSpan.FromDays(30);
opts.WindowOverlap = TimeSpan.FromDays(2);
opts.MaxItemsPerFetch = 50;
});
services.Configure<HttpClientFactoryOptions>(CertFrOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
var provider = services.BuildServiceProvider();
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
return provider;
}
private void SeedFeed()
{
_handler.AddTextResponse(FeedUri, ReadFixture("certfr-feed.xml"), "application/atom+xml");
}
private void SeedDetailResponses()
{
AddDetailResponse(FirstDetailUri, "certfr-detail-AV-2024-001.html", "\"certfr-001\"");
AddDetailResponse(SecondDetailUri, "certfr-detail-AV-2024-002.html", "\"certfr-002\"");
}
private void SeedNotModifiedDetailResponses()
{
AddNotModifiedResponse(FirstDetailUri, "\"certfr-001\"");
AddNotModifiedResponse(SecondDetailUri, "\"certfr-002\"");
}
private void AddDetailResponse(Uri uri, string fixture, string? etag)
{
_handler.AddResponse(uri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"),
};
if (!string.IsNullOrEmpty(etag))
{
response.Headers.ETag = new EntityTagHeaderValue(etag);
}
return response;
});
}
private void AddNotModifiedResponse(Uri uri, string? etag)
{
_handler.AddResponse(uri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.NotModified);
if (!string.IsNullOrEmpty(etag))
{
response.Headers.ETag = new EntityTagHeaderValue(etag);
}
return response;
});
}
private static string ReadFixture(string filename)
{
var baseDirectory = AppContext.BaseDirectory;
var primary = Path.Combine(baseDirectory, "Source", "CertFr", "Fixtures", filename);
if (File.Exists(primary))
{
return File.ReadAllText(primary);
}
var fallback = Path.Combine(baseDirectory, "CertFr", "Fixtures", filename);
return File.ReadAllText(fallback);
}
private static string Normalize(string value)
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
}
}

205
src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json Normal file
View File

@@ -0,0 +1,205 @@
[
{
"advisoryKey": "cert-fr/AV-2024.001",
"affectedPackages": [
{
"identifier": "AV-2024.001",
"platform": null,
"provenance": [
{
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
}
],
"statuses": [],
"type": "vendor",
"versionRanges": [
{
"fixedVersion": null,
"introducedVersion": null,
"lastAffectedVersion": null,
"primitives": {
"evr": null,
"hasVendorExtensions": true,
"nevra": null,
"semVer": null,
"vendorExtensions": {
"certfr.summary": "Résumé de la première alerte.",
"certfr.content": "AV-2024.001 Alerte CERT-FR AV-2024.001 L'exploitation active de la vulnérabilité est surveillée. Consultez les indications du fournisseur .",
"certfr.reference.count": "1"
}
},
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
},
"rangeExpression": null,
"rangeKind": "vendor"
}
]
}
],
"aliases": [
"CERT-FR:AV-2024.001"
],
"cvssMetrics": [],
"exploitKnown": false,
"language": "fr",
"modified": null,
"provenance": [
{
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
}
],
"published": "2024-10-03T00:00:00+00:00",
"references": [
{
"kind": "reference",
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
},
"sourceTag": null,
"summary": null,
"url": "https://vendor.example.com/patch"
},
{
"kind": "advisory",
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
},
"sourceTag": "cert-fr",
"summary": "Résumé de la première alerte.",
"url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"
}
],
"severity": null,
"summary": "Résumé de la première alerte.",
"title": "AV-2024.001 - Première alerte"
},
{
"advisoryKey": "cert-fr/AV-2024.002",
"affectedPackages": [
{
"identifier": "AV-2024.002",
"platform": null,
"provenance": [
{
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
}
],
"statuses": [],
"type": "vendor",
"versionRanges": [
{
"fixedVersion": null,
"introducedVersion": null,
"lastAffectedVersion": null,
"primitives": {
"evr": null,
"hasVendorExtensions": true,
"nevra": null,
"semVer": null,
"vendorExtensions": {
"certfr.summary": "Résumé de la deuxième alerte.",
"certfr.content": "AV-2024.002 Alerte CERT-FR AV-2024.002 Des correctifs sont disponibles pour plusieurs produits. Note de mise à jour Correctif",
"certfr.reference.count": "2"
}
},
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
},
"rangeExpression": null,
"rangeKind": "vendor"
}
]
}
],
"aliases": [
"CERT-FR:AV-2024.002"
],
"cvssMetrics": [],
"exploitKnown": false,
"language": "fr",
"modified": null,
"provenance": [
{
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
}
],
"published": "2024-10-03T00:00:00+00:00",
"references": [
{
"kind": "reference",
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
},
"sourceTag": null,
"summary": null,
"url": "https://support.example.com/kb/KB-1234"
},
{
"kind": "reference",
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
},
"sourceTag": null,
"summary": null,
"url": "https://support.example.com/kb/KB-5678"
},
{
"kind": "advisory",
"provenance": {
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-10-03T00:01:00+00:00",
"source": "cert-fr",
"value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
},
"sourceTag": "cert-fr",
"summary": "Résumé de la deuxième alerte.",
"url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"
}
],
"severity": null,
"summary": "Résumé de la deuxième alerte.",
"title": "AV-2024.002 - Deuxième alerte"
}
]

8
src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html Normal file
View File

@@ -0,0 +1,8 @@
<html>
<head><title>AV-2024.001</title></head>
<body>
<h1>Alerte CERT-FR AV-2024.001</h1>
<p>L'exploitation active de la vulnérabilité est surveillée.</p>
<p>Consultez les indications du <a href="https://vendor.example.com/patch">fournisseur</a>.</p>
</body>
</html>

11
src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html Normal file
View File

@@ -0,0 +1,11 @@
<html>
<head><title>AV-2024.002</title></head>
<body>
<h1>Alerte CERT-FR AV-2024.002</h1>
<p>Des correctifs sont disponibles pour plusieurs produits.</p>
<ul>
<li><a href="https://support.example.com/kb/KB-1234">Note de mise à jour</a></li>
<li><a href="https://support.example.com/kb/KB-5678">Correctif</a></li>
</ul>
</body>
</html>

22
src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml Normal file
View File

@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>CERT-FR Alertes</title>
<link>https://www.cert.ssi.gouv.fr/</link>
<description>Alertes example feed</description>
<item>
<title>AV-2024.001 - Première alerte</title>
<link>https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/</link>
<description><![CDATA[Résumé de la première alerte.]]></description>
<pubDate>Thu, 03 Oct 2024 09:00:00 +0000</pubDate>
<guid>AV-2024.001</guid>
</item>
<item>
<title>AV-2024.002 - Deuxième alerte</title>
<link>https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/</link>
<description><![CDATA[Résumé de la deuxième alerte.]]></description>
<pubDate>Thu, 03 Oct 2024 11:30:00 +0000</pubDate>
<guid>AV-2024.002</guid>
</item>
</channel>
</rss>

16
src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj Normal file
View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="CertFr/Fixtures/**" CopyToOutputDirectory="Always" />
</ItemGroup>
</Project>