using System; using System.Collections.Generic; using System.IO; using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Json; using StellaOps.Feedser.Source.Common.Testing; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Vndr.Chromium; using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.PsirtFlags; using StellaOps.Feedser.Testing; namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; [Collection("mongo-fixture")] public sealed class ChromiumConnectorTests : IAsyncLifetime { private readonly MongoIntegrationFixture _fixture; private readonly FakeTimeProvider _timeProvider; private readonly List _allocatedDatabases = new(); public ChromiumConnectorTests(MongoIntegrationFixture fixture) { _fixture = fixture; _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 10, 18, 0, 0, TimeSpan.Zero)); } [Fact] public async Task FetchParseMap_ProducesSnapshot() { var databaseName = AllocateDatabaseName(); await DropDatabaseAsync(databaseName); try { var handler = new CannedHttpMessageHandler(); await using var provider = await BuildServiceProviderAsync(handler, databaseName); SeedHttpFixtures(handler); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); try { await connector.ParseAsync(provider, CancellationToken.None); } catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException) { // Parsing should flag document as failed even when schema validation rejects payloads. } await connector.MapAsync(provider, CancellationToken.None); var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); var advisory = Assert.Single(advisories); Assert.Equal("chromium/post/stable-channel-update-for-desktop", advisory.AdvisoryKey); Assert.Contains("CHROMIUM-POST:stable-channel-update-for-desktop", advisory.Aliases); Assert.Contains("CVE-2024-12345", advisory.Aliases); Assert.Contains("CVE-2024-22222", advisory.Aliases); Assert.Contains(advisory.AffectedPackages, package => package.Platform == "android" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.89")); Assert.Contains(advisory.AffectedPackages, package => package.Platform == "linux" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.137")); Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome:extended-stable" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); Assert.Contains(advisory.References, reference => reference.Url.Contains("chromium.googlesource.com", StringComparison.OrdinalIgnoreCase)); Assert.Contains(advisory.References, reference => reference.Url.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase)); var psirtStore = provider.GetRequiredService(); var psirtFlag = await psirtStore.FindAsync(advisory.AdvisoryKey, CancellationToken.None); Assert.NotNull(psirtFlag); Assert.Equal("Google", psirtFlag!.Vendor); var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim(); var snapshotPath = ResolveFixturePath("chromium-advisory.snapshot.json"); var expected = File.ReadAllText(snapshotPath).Trim(); if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal)) { var actualPath = ResolveFixturePath("chromium-advisory.actual.json"); Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); File.WriteAllText(actualPath, canonicalJson); } Assert.Equal(expected, canonicalJson); } finally { await DropDatabaseAsync(databaseName); } } [Fact] public async Task ParseFailure_MarksDocumentFailed() { var databaseName = AllocateDatabaseName(); await DropDatabaseAsync(databaseName); try { var handler = new CannedHttpMessageHandler(); var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); handler.AddTextResponse(detailUri, "
missing post body
", "text/html"); await using var provider = await BuildServiceProviderAsync(handler, databaseName); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); try { await connector.ParseAsync(provider, CancellationToken.None); } catch (JsonSchemaValidationException) { // Expected for malformed posts; connector should still flag the document as failed. } var documentStore = provider.GetRequiredService(); var document = await documentStore.FindBySourceAndUriAsync(VndrChromiumConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); Assert.NotNull(document); Assert.Equal(DocumentStatuses.Failed, document!.Status); var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) ? pendingDocsValue.AsBsonArray : new BsonArray(); Assert.Empty(pendingDocuments); } finally { await DropDatabaseAsync(databaseName); } } [Fact] public async Task Resume_CompletesPendingDocumentsAfterRestart() { var databaseName = AllocateDatabaseName(); await DropDatabaseAsync(databaseName); try { var fetchHandler = new CannedHttpMessageHandler(); Guid[] pendingDocumentIds; await using (var fetchProvider = await BuildServiceProviderAsync(fetchHandler, databaseName)) { SeedHttpFixtures(fetchHandler); var connector = fetchProvider.GetRequiredService(); await connector.FetchAsync(fetchProvider, CancellationToken.None); var stateRepository = fetchProvider.GetRequiredService(); var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) ? pendingDocsValue.AsBsonArray : new BsonArray(); Assert.NotEmpty(pendingDocuments); pendingDocumentIds = pendingDocuments.Select(value => Guid.Parse(value.AsString)).ToArray(); } var resumeHandler = new CannedHttpMessageHandler(); SeedHttpFixtures(resumeHandler); await using var resumeProvider = await BuildServiceProviderAsync(resumeHandler, databaseName); var stateRepositoryBefore = resumeProvider.GetRequiredService(); var resumeState = await stateRepositoryBefore.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(resumeState); var resumePendingDocs = resumeState!.Cursor.TryGetValue("pendingDocuments", out var resumePendingValue) ? resumePendingValue.AsBsonArray : new BsonArray(); Assert.Equal(pendingDocumentIds.Length, resumePendingDocs.Count); var resumeIds = resumePendingDocs.Select(value => Guid.Parse(value.AsString)).OrderBy(id => id).ToArray(); Assert.Equal(pendingDocumentIds.OrderBy(id => id).ToArray(), resumeIds); var resumeConnector = resumeProvider.GetRequiredService(); await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); var documentStore = resumeProvider.GetRequiredService(); foreach (var documentId in pendingDocumentIds) { var document = await documentStore.FindAsync(documentId, CancellationToken.None); Assert.NotNull(document); Assert.Equal(DocumentStatuses.Mapped, document!.Status); } var stateRepositoryAfter = resumeProvider.GetRequiredService(); var finalState = await stateRepositoryAfter.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(finalState); var finalPending = finalState!.Cursor.TryGetValue("pendingDocuments", out var finalPendingDocs) ? finalPendingDocs.AsBsonArray : new BsonArray(); Assert.Empty(finalPending); var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var finalPendingMappingsValue) ? finalPendingMappingsValue.AsBsonArray : new BsonArray(); Assert.Empty(finalPendingMappings); } finally { await DropDatabaseAsync(databaseName); } } [Fact] public async Task Fetch_SkipsUnchangedDocuments() { var databaseName = AllocateDatabaseName(); await DropDatabaseAsync(databaseName); try { var handler = new CannedHttpMessageHandler(); await using var provider = await BuildServiceProviderAsync(handler, databaseName); SeedHttpFixtures(handler); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); await connector.ParseAsync(provider, CancellationToken.None); await connector.MapAsync(provider, CancellationToken.None); var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.Single(advisories); // Re-seed responses and fetch again with unchanged content. SeedHttpFixtures(handler); await connector.FetchAsync(provider, CancellationToken.None); var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); var cursor = state!.Cursor; var pendingDocuments = cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) ? pendingDocsValue.AsBsonArray : new BsonArray(); Assert.Empty(pendingDocuments); var pendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) ? pendingMappingsValue.AsBsonArray : new BsonArray(); Assert.Empty(pendingMappings); } finally { await DropDatabaseAsync(databaseName); } } private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler, string databaseName) { var services = new ServiceCollection(); services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); services.AddSingleton(_timeProvider); services.AddSingleton(handler); services.AddMongoStorage(options => { options.ConnectionString = _fixture.Runner.ConnectionString; options.DatabaseName = databaseName; options.CommandTimeout = TimeSpan.FromSeconds(5); }); services.AddSourceCommon(); services.AddChromiumConnector(opts => { opts.FeedUri = new Uri("https://chromereleases.googleblog.com/atom.xml"); opts.InitialBackfill = TimeSpan.FromDays(30); opts.WindowOverlap = TimeSpan.FromDays(1); opts.MaxFeedPages = 1; opts.MaxEntriesPerPage = 50; }); services.Configure(ChromiumOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => { builder.PrimaryHandler = handler; }); }); var provider = services.BuildServiceProvider(); var bootstrapper = provider.GetRequiredService(); await bootstrapper.InitializeAsync(CancellationToken.None); return provider; } private string AllocateDatabaseName() { var name = $"chromium-tests-{Guid.NewGuid():N}"; _allocatedDatabases.Add(name); return name; } private async Task DropDatabaseAsync(string databaseName) { try { await _fixture.Client.DropDatabaseAsync(databaseName); } catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound") { } } private static void SeedHttpFixtures(CannedHttpMessageHandler handler) { var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); handler.AddTextResponse(detailUri, ReadFixture("chromium-detail.html"), "text/html"); } private static string ReadFixture(string filename) { var path = ResolveFixturePath(filename); return File.ReadAllText(path); } private static string ResolveFixturePath(string filename) { var baseDirectory = AppContext.BaseDirectory; var primary = Path.Combine(baseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename); if (File.Exists(primary)) { return primary; } return Path.Combine(baseDirectory, "Chromium", "Fixtures", filename); } public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() { foreach (var name in _allocatedDatabases.Distinct(StringComparer.Ordinal)) { await DropDatabaseAsync(name); } } }