Restructure solution layout by module

This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions

View File

@@ -0,0 +1,350 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertIn;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Testing;
namespace StellaOps.Concelier.Connector.CertIn.Tests;
[Collection("mongo-fixture")]
public sealed class CertInConnectorTests : IAsyncLifetime
{
private readonly MongoIntegrationFixture _fixture;
private readonly FakeTimeProvider _timeProvider;
private readonly CannedHttpMessageHandler _handler;
private ServiceProvider? _serviceProvider;
public CertInConnectorTests(MongoIntegrationFixture fixture)
{
_fixture = fixture;
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 20, 0, 0, 0, TimeSpan.Zero));
_handler = new CannedHttpMessageHandler();
}
[Fact]
public async Task FetchParseMap_GeneratesExpectedSnapshot()
{
var options = new CertInOptions
{
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
WindowSize = TimeSpan.FromDays(60),
WindowOverlap = TimeSpan.FromDays(7),
MaxPagesPerFetch = 1,
RequestDelay = TimeSpan.Zero,
};
await EnsureServiceProviderAsync(options);
var provider = _serviceProvider!;
_handler.Clear();
_handler.AddTextResponse(options.AlertsEndpoint, ReadFixture("alerts-page1.json"), "application/json");
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
_handler.AddTextResponse(detailUri, ReadFixture("detail-CIAD-2024-0005.html"), "text/html");
var connector = new CertInConnectorPlugin().Create(provider);
await connector.FetchAsync(provider, CancellationToken.None);
_timeProvider.Advance(TimeSpan.FromMinutes(1));
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
Assert.Single(advisories);
var canonical = SnapshotSerializer.ToSnapshot(advisories.Single());
var expected = ReadFixture("expected-advisory.json");
var normalizedExpected = NormalizeLineEndings(expected);
var normalizedActual = NormalizeLineEndings(canonical);
if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal))
{
var actualPath = ResolveFixturePath("expected-advisory.actual.json");
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
File.WriteAllText(actualPath, canonical);
}
Assert.Equal(normalizedExpected, normalizedActual);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pending));
Assert.Empty(pending.AsBsonArray);
}
[Fact]
public async Task FetchFailure_RecordsBackoffAndReason()
{
var options = new CertInOptions
{
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
WindowSize = TimeSpan.FromDays(60),
WindowOverlap = TimeSpan.FromDays(7),
MaxPagesPerFetch = 1,
RequestDelay = TimeSpan.Zero,
};
await EnsureServiceProviderAsync(options);
_handler.Clear();
_handler.AddResponse(options.AlertsEndpoint, () => new HttpResponseMessage(HttpStatusCode.InternalServerError)
{
Content = new StringContent("{}", Encoding.UTF8, "application/json"),
});
var provider = _serviceProvider!;
var connector = new CertInConnectorPlugin().Create(provider);
await Assert.ThrowsAsync<HttpRequestException>(() => connector.FetchAsync(provider, CancellationToken.None));
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.Equal(1, state!.FailCount);
Assert.NotNull(state.LastFailureReason);
Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal);
Assert.True(state.BackoffUntil.HasValue);
Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow());
}
[Fact]
public async Task Fetch_NotModifiedMaintainsDocumentState()
{
var options = new CertInOptions
{
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
WindowSize = TimeSpan.FromDays(30),
WindowOverlap = TimeSpan.FromDays(7),
MaxPagesPerFetch = 1,
RequestDelay = TimeSpan.Zero,
};
await EnsureServiceProviderAsync(options);
var provider = _serviceProvider!;
_handler.Clear();
var listingPayload = ReadFixture("alerts-page1.json");
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
var etag = new EntityTagHeaderValue("\"certin-2024-0005\"");
var lastModified = new DateTimeOffset(2024, 4, 15, 10, 0, 0, TimeSpan.Zero);
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
_handler.AddResponse(detailUri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"),
};
response.Headers.ETag = etag;
response.Content.Headers.LastModified = lastModified;
return response;
});
var connector = new CertInConnectorPlugin().Create(provider);
await connector.FetchAsync(provider, CancellationToken.None);
_timeProvider.Advance(TimeSpan.FromMinutes(1));
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
Assert.Equal(etag.Tag, document.Etag);
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
_handler.AddResponse(detailUri, () =>
{
var response = new HttpResponseMessage(HttpStatusCode.NotModified)
{
Content = new StringContent(string.Empty)
};
response.Headers.ETag = etag;
return response;
});
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
Assert.Equal(0, pendingDocs.AsBsonArray.Count);
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
Assert.Equal(0, pendingMappings.AsBsonArray.Count);
}
[Fact]
public async Task Fetch_DuplicateContentSkipsRequeue()
{
var options = new CertInOptions
{
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
WindowSize = TimeSpan.FromDays(30),
WindowOverlap = TimeSpan.FromDays(7),
MaxPagesPerFetch = 1,
RequestDelay = TimeSpan.Zero,
};
await EnsureServiceProviderAsync(options);
var provider = _serviceProvider!;
_handler.Clear();
var listingPayload = ReadFixture("alerts-page1.json");
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
var connector = new CertInConnectorPlugin().Create(provider);
await connector.FetchAsync(provider, CancellationToken.None);
_timeProvider.Advance(TimeSpan.FromMinutes(1));
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
await connector.FetchAsync(provider, CancellationToken.None);
await connector.ParseAsync(provider, CancellationToken.None);
await connector.MapAsync(provider, CancellationToken.None);
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
Assert.Equal(0, pendingDocs.AsBsonArray.Count);
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
Assert.Equal(0, pendingMappings.AsBsonArray.Count);
}
private async Task EnsureServiceProviderAsync(CertInOptions template)
{
if (_serviceProvider is not null)
{
await ResetDatabaseAsync();
return;
}
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
services.AddSingleton<TimeProvider>(_timeProvider);
services.AddSingleton(_handler);
services.AddMongoStorage(options =>
{
options.ConnectionString = _fixture.Runner.ConnectionString;
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
options.CommandTimeout = TimeSpan.FromSeconds(5);
});
services.AddSourceCommon();
services.AddCertInConnector(opts =>
{
opts.AlertsEndpoint = template.AlertsEndpoint;
opts.WindowSize = template.WindowSize;
opts.WindowOverlap = template.WindowOverlap;
opts.MaxPagesPerFetch = template.MaxPagesPerFetch;
opts.RequestDelay = template.RequestDelay;
});
services.Configure<HttpClientFactoryOptions>(CertInOptions.HttpClientName, builderOptions =>
{
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
{
builder.PrimaryHandler = _handler;
});
});
_serviceProvider = services.BuildServiceProvider();
var bootstrapper = _serviceProvider.GetRequiredService<MongoBootstrapper>();
await bootstrapper.InitializeAsync(CancellationToken.None);
}
private Task ResetDatabaseAsync()
=> _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
private static string ReadFixture(string filename)
=> File.ReadAllText(ResolveFixturePath(filename));
private static string ResolveFixturePath(string filename)
{
var baseDirectory = AppContext.BaseDirectory;
var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename);
if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase))
{
return primary;
}
return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename);
}
private static string NormalizeLineEndings(string value)
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
if (_serviceProvider is IAsyncDisposable asyncDisposable)
{
await asyncDisposable.DisposeAsync();
}
else
{
_serviceProvider?.Dispose();
}
}
}

View File

@@ -0,0 +1,9 @@
[
{
"advisoryId": "CIAD-2024-0005",
"title": "Multiple vulnerabilities in Example Gateway",
"publishedOn": "2024-04-15T10:00:00Z",
"detailUrl": "https://cert-in.example/advisory/CIAD-2024-0005",
"summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990)."
}
]

View File

@@ -0,0 +1,17 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>Multiple vulnerabilities in Example Gateway</title>
</head>
<body>
<article>
<h1>Multiple vulnerabilities in Example Gateway</h1>
<p>Severity: High</p>
<p>Vendor: Example Gateway Technologies Pvt Ltd</p>
<p>Organisation: Partner Systems Inc.</p>
<p>CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands.</p>
<p>Further information is available from the <a href="https://vendor.example.com/advisories/example-gateway-bulletin">vendor bulletin</a>.</p>
</article>
</body>
</html>

View File

@@ -0,0 +1,128 @@
{
"advisoryKey": "CIAD-2024-0005",
"affectedPackages": [
{
"identifier": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the",
"platform": null,
"provenance": [
{
"fieldMask": [],
"kind": "affected",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the"
}
],
"statuses": [],
"type": "ics-vendor",
"versionRanges": [
{
"fixedVersion": null,
"introducedVersion": null,
"lastAffectedVersion": null,
"primitives": {
"evr": null,
"hasVendorExtensions": true,
"nevra": null,
"semVer": null,
"vendorExtensions": {
"certin.vendor": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the "
}
},
"provenance": {
"fieldMask": [],
"kind": "affected",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the"
},
"rangeExpression": null,
"rangeKind": "vendor"
}
]
}
],
"aliases": [
"CIAD-2024-0005",
"CVE-2024-9990",
"CVE-2024-9991"
],
"cvssMetrics": [],
"exploitKnown": false,
"language": "en",
"modified": "2024-04-15T10:00:00+00:00",
"provenance": [
{
"fieldMask": [],
"kind": "document",
"recordedAt": "2024-04-20T00:00:00+00:00",
"source": "cert-in",
"value": "https://cert-in.example/advisory/CIAD-2024-0005"
},
{
"fieldMask": [],
"kind": "mapping",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "CIAD-2024-0005"
}
],
"published": "2024-04-15T10:00:00+00:00",
"references": [
{
"kind": "advisory",
"provenance": {
"fieldMask": [],
"kind": "reference",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "https://cert-in.example/advisory/CIAD-2024-0005"
},
"sourceTag": "cert-in",
"summary": null,
"url": "https://cert-in.example/advisory/CIAD-2024-0005"
},
{
"kind": "reference",
"provenance": {
"fieldMask": [],
"kind": "reference",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "https://vendor.example.com/advisories/example-gateway-bulletin"
},
"sourceTag": null,
"summary": null,
"url": "https://vendor.example.com/advisories/example-gateway-bulletin"
},
{
"kind": "advisory",
"provenance": {
"fieldMask": [],
"kind": "reference",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "https://www.cve.org/CVERecord?id=CVE-2024-9990"
},
"sourceTag": "CVE-2024-9990",
"summary": null,
"url": "https://www.cve.org/CVERecord?id=CVE-2024-9990"
},
{
"kind": "advisory",
"provenance": {
"fieldMask": [],
"kind": "reference",
"recordedAt": "2024-04-20T00:01:00+00:00",
"source": "cert-in",
"value": "https://www.cve.org/CVERecord?id=CVE-2024-9991"
},
"sourceTag": "CVE-2024-9991",
"summary": null,
"url": "https://www.cve.org/CVERecord?id=CVE-2024-9991"
}
],
"severity": "high",
"summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).",
"title": "Multiple vulnerabilities in Example Gateway"
}

View File

@@ -0,0 +1,17 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="CertIn/Fixtures/**" CopyToOutputDirectory="Always" />
</ItemGroup>
</Project>