git.stella-ops.org/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Common.Json;
using StellaOps.Feedser.Source.Common.Testing;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Vndr.Chromium;
using StellaOps.Feedser.Source.Vndr.Chromium.Configuration;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.PsirtFlags;
using StellaOps.Feedser.Testing;

namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests;

[Collection("mongo-fixture")]
public sealed class ChromiumConnectorTests : IAsyncLifetime
{
    private readonly MongoIntegrationFixture _fixture;
    private readonly FakeTimeProvider _timeProvider;
    private readonly List<string> _allocatedDatabases = new();

    public ChromiumConnectorTests(MongoIntegrationFixture fixture)
    {
        _fixture = fixture;
        _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 10, 18, 0, 0, TimeSpan.Zero));
    }

    [Fact]
    public async Task FetchParseMap_ProducesSnapshot()
    {
        var databaseName = AllocateDatabaseName();
        await DropDatabaseAsync(databaseName);

        try
        {
            var handler = new CannedHttpMessageHandler();
            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
            SeedHttpFixtures(handler);

            var connector = provider.GetRequiredService<ChromiumConnector>();
            await connector.FetchAsync(provider, CancellationToken.None);
            try
            {
                await connector.ParseAsync(provider, CancellationToken.None);
            }
            catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException)
            {
                // Parsing should flag document as failed even when schema validation rejects payloads.
            }
            await connector.MapAsync(provider, CancellationToken.None);

            var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
            var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
            var advisory = Assert.Single(advisories);

            Assert.Equal("chromium/post/stable-channel-update-for-desktop", advisory.AdvisoryKey);
            Assert.Contains("CHROMIUM-POST:stable-channel-update-for-desktop", advisory.Aliases);
            Assert.Contains("CVE-2024-12345", advisory.Aliases);
            Assert.Contains("CVE-2024-22222", advisory.Aliases);

            Assert.Contains(advisory.AffectedPackages, package => package.Platform == "android" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.89"));
            Assert.Contains(advisory.AffectedPackages, package => package.Platform == "linux" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.137"));
            Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138"));
            Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome:extended-stable" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138"));

            Assert.Contains(advisory.References, reference => reference.Url.Contains("chromium.googlesource.com", StringComparison.OrdinalIgnoreCase));
            Assert.Contains(advisory.References, reference => reference.Url.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase));

            var psirtStore = provider.GetRequiredService<IPsirtFlagStore>();
            var psirtFlag = await psirtStore.FindAsync(advisory.AdvisoryKey, CancellationToken.None);
            Assert.NotNull(psirtFlag);
            Assert.Equal("Google", psirtFlag!.Vendor);

        var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim();
        var snapshotPath = ResolveFixturePath("chromium-advisory.snapshot.json");
        var expected = File.ReadAllText(snapshotPath).Trim();
        if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal))
        {
            var actualPath = ResolveFixturePath("chromium-advisory.actual.json");
            Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
            File.WriteAllText(actualPath, canonicalJson);
        }
        Assert.Equal(expected, canonicalJson);
    }
        finally
        {
            await DropDatabaseAsync(databaseName);
        }
    }

    [Fact]
    public async Task ParseFailure_MarksDocumentFailed()
    {
        var databaseName = AllocateDatabaseName();
        await DropDatabaseAsync(databaseName);

        try
        {
            var handler = new CannedHttpMessageHandler();
            var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false");
            var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html");

            handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml");
            handler.AddTextResponse(detailUri, "<html><body><div>missing post body</div></body></html>", "text/html");

            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
            var connector = provider.GetRequiredService<ChromiumConnector>();

            await connector.FetchAsync(provider, CancellationToken.None);
            try
            {
                await connector.ParseAsync(provider, CancellationToken.None);
            }
            catch (JsonSchemaValidationException)
            {
                // Expected for malformed posts; connector should still flag the document as failed.
            }

            var documentStore = provider.GetRequiredService<IDocumentStore>();
            var document = await documentStore.FindBySourceAndUriAsync(VndrChromiumConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
            Assert.NotNull(document);
            Assert.Equal(DocumentStatuses.Failed, document!.Status);

            var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
            var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
            Assert.NotNull(state);
            var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
                ? pendingDocsValue.AsBsonArray
                : new BsonArray();
            Assert.Empty(pendingDocuments);
        }
        finally
        {
            await DropDatabaseAsync(databaseName);
        }
    }

    [Fact]
    public async Task Resume_CompletesPendingDocumentsAfterRestart()
    {
        var databaseName = AllocateDatabaseName();
        await DropDatabaseAsync(databaseName);

        try
        {
            var fetchHandler = new CannedHttpMessageHandler();
            Guid[] pendingDocumentIds;
            await using (var fetchProvider = await BuildServiceProviderAsync(fetchHandler, databaseName))
            {
                SeedHttpFixtures(fetchHandler);
                var connector = fetchProvider.GetRequiredService<ChromiumConnector>();
                await connector.FetchAsync(fetchProvider, CancellationToken.None);

                var stateRepository = fetchProvider.GetRequiredService<ISourceStateRepository>();
                var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
                Assert.NotNull(state);
                var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
                    ? pendingDocsValue.AsBsonArray
                    : new BsonArray();
                Assert.NotEmpty(pendingDocuments);
                pendingDocumentIds = pendingDocuments.Select(value => Guid.Parse(value.AsString)).ToArray();
            }

            var resumeHandler = new CannedHttpMessageHandler();
            SeedHttpFixtures(resumeHandler);
            await using var resumeProvider = await BuildServiceProviderAsync(resumeHandler, databaseName);
            var stateRepositoryBefore = resumeProvider.GetRequiredService<ISourceStateRepository>();
            var resumeState = await stateRepositoryBefore.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
            Assert.NotNull(resumeState);
            var resumePendingDocs = resumeState!.Cursor.TryGetValue("pendingDocuments", out var resumePendingValue)
                ? resumePendingValue.AsBsonArray
                : new BsonArray();
            Assert.Equal(pendingDocumentIds.Length, resumePendingDocs.Count);
            var resumeIds = resumePendingDocs.Select(value => Guid.Parse(value.AsString)).OrderBy(id => id).ToArray();
            Assert.Equal(pendingDocumentIds.OrderBy(id => id).ToArray(), resumeIds);

            var resumeConnector = resumeProvider.GetRequiredService<ChromiumConnector>();
            await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None);
            await resumeConnector.MapAsync(resumeProvider, CancellationToken.None);

            var documentStore = resumeProvider.GetRequiredService<IDocumentStore>();
            foreach (var documentId in pendingDocumentIds)
            {
                var document = await documentStore.FindAsync(documentId, CancellationToken.None);
                Assert.NotNull(document);
                Assert.Equal(DocumentStatuses.Mapped, document!.Status);
            }

            var stateRepositoryAfter = resumeProvider.GetRequiredService<ISourceStateRepository>();
            var finalState = await stateRepositoryAfter.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
            Assert.NotNull(finalState);
            var finalPending = finalState!.Cursor.TryGetValue("pendingDocuments", out var finalPendingDocs)
                ? finalPendingDocs.AsBsonArray
                : new BsonArray();
            Assert.Empty(finalPending);

            var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var finalPendingMappingsValue)
                ? finalPendingMappingsValue.AsBsonArray
                : new BsonArray();
            Assert.Empty(finalPendingMappings);
        }
        finally
        {
            await DropDatabaseAsync(databaseName);
        }
    }

    [Fact]
    public async Task Fetch_SkipsUnchangedDocuments()
    {
        var databaseName = AllocateDatabaseName();
        await DropDatabaseAsync(databaseName);

        try
        {
            var handler = new CannedHttpMessageHandler();
            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
            SeedHttpFixtures(handler);

            var connector = provider.GetRequiredService<ChromiumConnector>();
            await connector.FetchAsync(provider, CancellationToken.None);
            await connector.ParseAsync(provider, CancellationToken.None);
            await connector.MapAsync(provider, CancellationToken.None);

            var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
            var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
            Assert.Single(advisories);

            // Re-seed responses and fetch again with unchanged content.
            SeedHttpFixtures(handler);
            await connector.FetchAsync(provider, CancellationToken.None);

            var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
            var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
            Assert.NotNull(state);
            var cursor = state!.Cursor;
            var pendingDocuments = cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
                ? pendingDocsValue.AsBsonArray
                : new BsonArray();
            Assert.Empty(pendingDocuments);

            var pendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappingsValue)
                ? pendingMappingsValue.AsBsonArray
                : new BsonArray();
            Assert.Empty(pendingMappings);
        }
        finally
        {
            await DropDatabaseAsync(databaseName);
        }
    }

    private async Task<ServiceProvider> BuildServiceProviderAsync(CannedHttpMessageHandler handler, string databaseName)
    {
        var services = new ServiceCollection();
        services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
        services.AddSingleton<TimeProvider>(_timeProvider);
        services.AddSingleton(handler);

        services.AddMongoStorage(options =>
        {
            options.ConnectionString = _fixture.Runner.ConnectionString;
            options.DatabaseName = databaseName;
            options.CommandTimeout = TimeSpan.FromSeconds(5);
        });

        services.AddSourceCommon();
        services.AddChromiumConnector(opts =>
        {
            opts.FeedUri = new Uri("https://chromereleases.googleblog.com/atom.xml");
            opts.InitialBackfill = TimeSpan.FromDays(30);
            opts.WindowOverlap = TimeSpan.FromDays(1);
            opts.MaxFeedPages = 1;
            opts.MaxEntriesPerPage = 50;
        });

        services.Configure<HttpClientFactoryOptions>(ChromiumOptions.HttpClientName, builderOptions =>
        {
            builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
            {
                builder.PrimaryHandler = handler;
            });
        });

        var provider = services.BuildServiceProvider();

        var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
        await bootstrapper.InitializeAsync(CancellationToken.None);

        return provider;
    }

    private string AllocateDatabaseName()
    {
        var name = $"chromium-tests-{Guid.NewGuid():N}";
        _allocatedDatabases.Add(name);
        return name;
    }

    private async Task DropDatabaseAsync(string databaseName)
    {
        try
        {
            await _fixture.Client.DropDatabaseAsync(databaseName);
        }
        catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound")
        {
        }
    }

    private static void SeedHttpFixtures(CannedHttpMessageHandler handler)
    {
        var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false");
        var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html");

        handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml");
        handler.AddTextResponse(detailUri, ReadFixture("chromium-detail.html"), "text/html");
    }

    private static string ReadFixture(string filename)
    {
        var path = ResolveFixturePath(filename);
        return File.ReadAllText(path);
    }

    private static string ResolveFixturePath(string filename)
    {
        var baseDirectory = AppContext.BaseDirectory;
        var primary = Path.Combine(baseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename);
        if (File.Exists(primary))
        {
            return primary;
        }

        return Path.Combine(baseDirectory, "Chromium", "Fixtures", filename);
    }

    public Task InitializeAsync() => Task.CompletedTask;

    public async Task DisposeAsync()
    {
        foreach (var name in _allocatedDatabases.Distinct(StringComparer.Ordinal))
        {
            await DropDatabaseAsync(name);
        }
    }
}