up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,24 +1,24 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.CertIn;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.CertIn;
|
||||
using StellaOps.Concelier.Connector.CertIn.Configuration;
|
||||
using StellaOps.Concelier.Connector.CertIn.Internal;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Testing;
|
||||
@@ -28,254 +28,254 @@ using StellaOps.Concelier.Storage.Postgres;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Storage;
|
||||
using StellaOps.Concelier.Testing;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Tests;
|
||||
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertIn.Tests;
|
||||
|
||||
[Collection(ConcelierFixtureCollection.Name)]
|
||||
public sealed class CertInConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private readonly ConcelierPostgresFixture _fixture;
|
||||
private readonly FakeTimeProvider _timeProvider;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
private ServiceProvider? _serviceProvider;
|
||||
|
||||
public CertInConnectorTests(ConcelierPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 20, 0, 0, 0, TimeSpan.Zero));
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_GeneratesExpectedSnapshot()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(60),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
|
||||
_handler.Clear();
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, ReadFixture("alerts-page1.json"), "application/json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
_handler.AddTextResponse(detailUri, ReadFixture("detail-CIAD-2024-0005.html"), "text/html");
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
|
||||
Assert.Single(advisories);
|
||||
var canonical = SnapshotSerializer.ToSnapshot(advisories.Single());
|
||||
var expected = ReadFixture("expected-advisory.json");
|
||||
var normalizedExpected = NormalizeLineEndings(expected);
|
||||
var normalizedActual = NormalizeLineEndings(canonical);
|
||||
if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = ResolveFixturePath("expected-advisory.actual.json");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
|
||||
File.WriteAllText(actualPath, canonical);
|
||||
}
|
||||
|
||||
Assert.Equal(normalizedExpected, normalizedActual);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pending));
|
||||
Assert.Empty(pending.AsBsonArray);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchFailure_RecordsBackoffAndReason()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(60),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
_handler.Clear();
|
||||
_handler.AddResponse(options.AlertsEndpoint, () => new HttpResponseMessage(HttpStatusCode.InternalServerError)
|
||||
{
|
||||
Content = new StringContent("{}", Encoding.UTF8, "application/json"),
|
||||
});
|
||||
|
||||
var provider = _serviceProvider!;
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await Assert.ThrowsAsync<HttpRequestException>(() => connector.FetchAsync(provider, CancellationToken.None));
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.Equal(1, state!.FailCount);
|
||||
Assert.NotNull(state.LastFailureReason);
|
||||
Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal);
|
||||
Assert.True(state.BackoffUntil.HasValue);
|
||||
Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_NotModifiedMaintainsDocumentState()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
_handler.Clear();
|
||||
|
||||
var listingPayload = ReadFixture("alerts-page1.json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
|
||||
var etag = new EntityTagHeaderValue("\"certin-2024-0005\"");
|
||||
var lastModified = new DateTimeOffset(2024, 4, 15, 10, 0, 0, TimeSpan.Zero);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddResponse(detailUri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"),
|
||||
};
|
||||
|
||||
response.Headers.ETag = etag;
|
||||
response.Content.Headers.LastModified = lastModified;
|
||||
return response;
|
||||
});
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
Assert.Equal(etag.Tag, document.Etag);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddResponse(detailUri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.NotModified)
|
||||
{
|
||||
Content = new StringContent(string.Empty)
|
||||
};
|
||||
response.Headers.ETag = etag;
|
||||
return response;
|
||||
});
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
|
||||
Assert.Equal(0, pendingDocs.AsBsonArray.Count);
|
||||
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
|
||||
Assert.Equal(0, pendingMappings.AsBsonArray.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_DuplicateContentSkipsRequeue()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
_handler.Clear();
|
||||
|
||||
var listingPayload = ReadFixture("alerts-page1.json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
|
||||
Assert.Equal(0, pendingDocs.AsBsonArray.Count);
|
||||
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
|
||||
Assert.Equal(0, pendingMappings.AsBsonArray.Count);
|
||||
}
|
||||
|
||||
private async Task EnsureServiceProviderAsync(CertInOptions template)
|
||||
{
|
||||
if (_serviceProvider is not null)
|
||||
{
|
||||
await ResetDatabaseAsync();
|
||||
return;
|
||||
}
|
||||
public sealed class CertInConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private readonly ConcelierPostgresFixture _fixture;
|
||||
private readonly FakeTimeProvider _timeProvider;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
private ServiceProvider? _serviceProvider;
|
||||
|
||||
public CertInConnectorTests(ConcelierPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 20, 0, 0, 0, TimeSpan.Zero));
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_GeneratesExpectedSnapshot()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(60),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
|
||||
_handler.Clear();
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, ReadFixture("alerts-page1.json"), "application/json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
_handler.AddTextResponse(detailUri, ReadFixture("detail-CIAD-2024-0005.html"), "text/html");
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
|
||||
Assert.Single(advisories);
|
||||
var canonical = SnapshotSerializer.ToSnapshot(advisories.Single());
|
||||
var expected = ReadFixture("expected-advisory.json");
|
||||
var normalizedExpected = NormalizeLineEndings(expected);
|
||||
var normalizedActual = NormalizeLineEndings(canonical);
|
||||
if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = ResolveFixturePath("expected-advisory.actual.json");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
|
||||
File.WriteAllText(actualPath, canonical);
|
||||
}
|
||||
|
||||
Assert.Equal(normalizedExpected, normalizedActual);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pending));
|
||||
Assert.Empty(pending.AsDocumentArray);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchFailure_RecordsBackoffAndReason()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(60),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
_handler.Clear();
|
||||
_handler.AddResponse(options.AlertsEndpoint, () => new HttpResponseMessage(HttpStatusCode.InternalServerError)
|
||||
{
|
||||
Content = new StringContent("{}", Encoding.UTF8, "application/json"),
|
||||
});
|
||||
|
||||
var provider = _serviceProvider!;
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await Assert.ThrowsAsync<HttpRequestException>(() => connector.FetchAsync(provider, CancellationToken.None));
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.Equal(1, state!.FailCount);
|
||||
Assert.NotNull(state.LastFailureReason);
|
||||
Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal);
|
||||
Assert.True(state.BackoffUntil.HasValue);
|
||||
Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_NotModifiedMaintainsDocumentState()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
_handler.Clear();
|
||||
|
||||
var listingPayload = ReadFixture("alerts-page1.json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
|
||||
var etag = new EntityTagHeaderValue("\"certin-2024-0005\"");
|
||||
var lastModified = new DateTimeOffset(2024, 4, 15, 10, 0, 0, TimeSpan.Zero);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddResponse(detailUri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"),
|
||||
};
|
||||
|
||||
response.Headers.ETag = etag;
|
||||
response.Content.Headers.LastModified = lastModified;
|
||||
return response;
|
||||
});
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
Assert.Equal(etag.Tag, document.Etag);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddResponse(detailUri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.NotModified)
|
||||
{
|
||||
Content = new StringContent(string.Empty)
|
||||
};
|
||||
response.Headers.ETag = etag;
|
||||
return response;
|
||||
});
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
|
||||
Assert.Equal(0, pendingDocs.AsDocumentArray.Count);
|
||||
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
|
||||
Assert.Equal(0, pendingMappings.AsDocumentArray.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_DuplicateContentSkipsRequeue()
|
||||
{
|
||||
var options = new CertInOptions
|
||||
{
|
||||
AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute),
|
||||
WindowSize = TimeSpan.FromDays(30),
|
||||
WindowOverlap = TimeSpan.FromDays(7),
|
||||
MaxPagesPerFetch = 1,
|
||||
RequestDelay = TimeSpan.Zero,
|
||||
};
|
||||
|
||||
await EnsureServiceProviderAsync(options);
|
||||
var provider = _serviceProvider!;
|
||||
_handler.Clear();
|
||||
|
||||
var listingPayload = ReadFixture("alerts-page1.json");
|
||||
var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005");
|
||||
var detailHtml = ReadFixture("detail-CIAD-2024-0005.html");
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
|
||||
|
||||
var connector = new CertInConnectorPlugin().Create(provider);
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
_timeProvider.Advance(TimeSpan.FromMinutes(1));
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
_handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json");
|
||||
_handler.AddTextResponse(detailUri, detailHtml, "text/html");
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
|
||||
Assert.NotNull(document);
|
||||
Assert.Equal(DocumentStatuses.Mapped, document!.Status);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs));
|
||||
Assert.Equal(0, pendingDocs.AsDocumentArray.Count);
|
||||
Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings));
|
||||
Assert.Equal(0, pendingMappings.AsDocumentArray.Count);
|
||||
}
|
||||
|
||||
private async Task EnsureServiceProviderAsync(CertInOptions template)
|
||||
{
|
||||
if (_serviceProvider is not null)
|
||||
{
|
||||
await ResetDatabaseAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
@@ -294,19 +294,19 @@ public sealed class CertInConnectorTests : IAsyncLifetime
|
||||
services.AddSourceCommon();
|
||||
services.AddCertInConnector(opts =>
|
||||
{
|
||||
opts.AlertsEndpoint = template.AlertsEndpoint;
|
||||
opts.WindowSize = template.WindowSize;
|
||||
opts.WindowOverlap = template.WindowOverlap;
|
||||
opts.MaxPagesPerFetch = template.MaxPagesPerFetch;
|
||||
opts.RequestDelay = template.RequestDelay;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(CertInOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
|
||||
{
|
||||
builder.PrimaryHandler = _handler;
|
||||
});
|
||||
opts.AlertsEndpoint = template.AlertsEndpoint;
|
||||
opts.WindowSize = template.WindowSize;
|
||||
opts.WindowOverlap = template.WindowOverlap;
|
||||
opts.MaxPagesPerFetch = template.MaxPagesPerFetch;
|
||||
opts.RequestDelay = template.RequestDelay;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(CertInOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
|
||||
{
|
||||
builder.PrimaryHandler = _handler;
|
||||
});
|
||||
});
|
||||
|
||||
_serviceProvider = services.BuildServiceProvider();
|
||||
@@ -314,36 +314,36 @@ public sealed class CertInConnectorTests : IAsyncLifetime
|
||||
|
||||
private Task ResetDatabaseAsync()
|
||||
=> _fixture.TruncateAllTablesAsync();
|
||||
|
||||
private static string ReadFixture(string filename)
|
||||
=> File.ReadAllText(ResolveFixturePath(filename));
|
||||
|
||||
private static string ResolveFixturePath(string filename)
|
||||
{
|
||||
var baseDirectory = AppContext.BaseDirectory;
|
||||
var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename);
|
||||
if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return primary;
|
||||
}
|
||||
|
||||
return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_serviceProvider is IAsyncDisposable asyncDisposable)
|
||||
{
|
||||
await asyncDisposable.DisposeAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
_serviceProvider?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ReadFixture(string filename)
|
||||
=> File.ReadAllText(ResolveFixturePath(filename));
|
||||
|
||||
private static string ResolveFixturePath(string filename)
|
||||
{
|
||||
var baseDirectory = AppContext.BaseDirectory;
|
||||
var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename);
|
||||
if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return primary;
|
||||
}
|
||||
|
||||
return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_serviceProvider is IAsyncDisposable asyncDisposable)
|
||||
{
|
||||
await asyncDisposable.DisposeAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
_serviceProvider?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user