Rename Concelier Source modules to Connector

src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs

@@ -0,0 +1,360 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Http;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
+using MongoDB.Bson;
+using MongoDB.Driver;
+using StellaOps.Concelier.Models;
+using StellaOps.Concelier.Connector.Common.Http;
+using StellaOps.Concelier.Connector.Common.Json;
+using StellaOps.Concelier.Connector.Common.Testing;
+using StellaOps.Concelier.Connector.Common;
+using StellaOps.Concelier.Connector.Vndr.Chromium;
+using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration;
+using StellaOps.Concelier.Storage.Mongo;
+using StellaOps.Concelier.Storage.Mongo.Advisories;
+using StellaOps.Concelier.Storage.Mongo.Documents;
+using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
+using StellaOps.Concelier.Testing;
+
+namespace StellaOps.Concelier.Connector.Vndr.Chromium.Tests;
+
+[Collection("mongo-fixture")]
+public sealed class ChromiumConnectorTests : IAsyncLifetime
+{
+    private readonly MongoIntegrationFixture _fixture;
+    private readonly FakeTimeProvider _timeProvider;
+    private readonly List<string> _allocatedDatabases = new();
+
+    public ChromiumConnectorTests(MongoIntegrationFixture fixture)
+    {
+        _fixture = fixture;
+        _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 10, 18, 0, 0, TimeSpan.Zero));
+    }
+
+    [Fact]
+    public async Task FetchParseMap_ProducesSnapshot()
+    {
+        var databaseName = AllocateDatabaseName();
+        await DropDatabaseAsync(databaseName);
+
+        try
+        {
+            var handler = new CannedHttpMessageHandler();
+            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
+            SeedHttpFixtures(handler);
+
+            var connector = provider.GetRequiredService<ChromiumConnector>();
+            await connector.FetchAsync(provider, CancellationToken.None);
+            try
+            {
+                await connector.ParseAsync(provider, CancellationToken.None);
+            }
+            catch (StellaOps.Concelier.Connector.Common.Json.JsonSchemaValidationException)
+            {
+                // Parsing should flag document as failed even when schema validation rejects payloads.
+            }
+            await connector.MapAsync(provider, CancellationToken.None);
+
+            var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
+            var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
+            var advisory = Assert.Single(advisories);
+
+            Assert.Equal("chromium/post/stable-channel-update-for-desktop", advisory.AdvisoryKey);
+            Assert.Contains("CHROMIUM-POST:stable-channel-update-for-desktop", advisory.Aliases);
+            Assert.Contains("CVE-2024-12345", advisory.Aliases);
+            Assert.Contains("CVE-2024-22222", advisory.Aliases);
+
+            Assert.Contains(advisory.AffectedPackages, package => package.Platform == "android" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.89"));
+            Assert.Contains(advisory.AffectedPackages, package => package.Platform == "linux" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.137"));
+            Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138"));
+            Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome:extended-stable" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138"));
+
+            Assert.Contains(advisory.References, reference => reference.Url.Contains("chromium.googlesource.com", StringComparison.OrdinalIgnoreCase));
+            Assert.Contains(advisory.References, reference => reference.Url.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase));
+
+            var psirtStore = provider.GetRequiredService<IPsirtFlagStore>();
+            var psirtFlag = await psirtStore.FindAsync(advisory.AdvisoryKey, CancellationToken.None);
+            Assert.NotNull(psirtFlag);
+            Assert.Equal("Google", psirtFlag!.Vendor);
+
+        var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim();
+        var snapshotPath = ResolveFixturePath("chromium-advisory.snapshot.json");
+        var expected = File.ReadAllText(snapshotPath).Trim();
+        if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal))
+        {
+            var actualPath = ResolveFixturePath("chromium-advisory.actual.json");
+            Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
+            File.WriteAllText(actualPath, canonicalJson);
+        }
+        Assert.Equal(expected, canonicalJson);
+    }
+        finally
+        {
+            await DropDatabaseAsync(databaseName);
+        }
+    }
+
+    [Fact]
+    public async Task ParseFailure_MarksDocumentFailed()
+    {
+        var databaseName = AllocateDatabaseName();
+        await DropDatabaseAsync(databaseName);
+
+        try
+        {
+            var handler = new CannedHttpMessageHandler();
+            var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false");
+            var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html");
+
+            handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml");
+            handler.AddTextResponse(detailUri, "<html><body><div>missing post body</div></body></html>", "text/html");
+
+            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
+            var connector = provider.GetRequiredService<ChromiumConnector>();
+
+            await connector.FetchAsync(provider, CancellationToken.None);
+            try
+            {
+                await connector.ParseAsync(provider, CancellationToken.None);
+            }
+            catch (JsonSchemaValidationException)
+            {
+                // Expected for malformed posts; connector should still flag the document as failed.
+            }
+
+            var documentStore = provider.GetRequiredService<IDocumentStore>();
+            var document = await documentStore.FindBySourceAndUriAsync(VndrChromiumConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None);
+            Assert.NotNull(document);
+            Assert.Equal(DocumentStatuses.Failed, document!.Status);
+
+            var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
+            var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
+            Assert.NotNull(state);
+            var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
+                ? pendingDocsValue.AsBsonArray
+                : new BsonArray();
+            Assert.Empty(pendingDocuments);
+        }
+        finally
+        {
+            await DropDatabaseAsync(databaseName);
+        }
+    }
+
+    [Fact]
+    public async Task Resume_CompletesPendingDocumentsAfterRestart()
+    {
+        var databaseName = AllocateDatabaseName();
+        await DropDatabaseAsync(databaseName);
+
+        try
+        {
+            var fetchHandler = new CannedHttpMessageHandler();
+            Guid[] pendingDocumentIds;
+            await using (var fetchProvider = await BuildServiceProviderAsync(fetchHandler, databaseName))
+            {
+                SeedHttpFixtures(fetchHandler);
+                var connector = fetchProvider.GetRequiredService<ChromiumConnector>();
+                await connector.FetchAsync(fetchProvider, CancellationToken.None);
+
+                var stateRepository = fetchProvider.GetRequiredService<ISourceStateRepository>();
+                var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
+                Assert.NotNull(state);
+                var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
+                    ? pendingDocsValue.AsBsonArray
+                    : new BsonArray();
+                Assert.NotEmpty(pendingDocuments);
+                pendingDocumentIds = pendingDocuments.Select(value => Guid.Parse(value.AsString)).ToArray();
+            }
+
+            var resumeHandler = new CannedHttpMessageHandler();
+            SeedHttpFixtures(resumeHandler);
+            await using var resumeProvider = await BuildServiceProviderAsync(resumeHandler, databaseName);
+            var stateRepositoryBefore = resumeProvider.GetRequiredService<ISourceStateRepository>();
+            var resumeState = await stateRepositoryBefore.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
+            Assert.NotNull(resumeState);
+            var resumePendingDocs = resumeState!.Cursor.TryGetValue("pendingDocuments", out var resumePendingValue)
+                ? resumePendingValue.AsBsonArray
+                : new BsonArray();
+            Assert.Equal(pendingDocumentIds.Length, resumePendingDocs.Count);
+            var resumeIds = resumePendingDocs.Select(value => Guid.Parse(value.AsString)).OrderBy(id => id).ToArray();
+            Assert.Equal(pendingDocumentIds.OrderBy(id => id).ToArray(), resumeIds);
+
+            var resumeConnector = resumeProvider.GetRequiredService<ChromiumConnector>();
+            await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None);
+            await resumeConnector.MapAsync(resumeProvider, CancellationToken.None);
+
+            var documentStore = resumeProvider.GetRequiredService<IDocumentStore>();
+            foreach (var documentId in pendingDocumentIds)
+            {
+                var document = await documentStore.FindAsync(documentId, CancellationToken.None);
+                Assert.NotNull(document);
+                Assert.Equal(DocumentStatuses.Mapped, document!.Status);
+            }
+
+            var stateRepositoryAfter = resumeProvider.GetRequiredService<ISourceStateRepository>();
+            var finalState = await stateRepositoryAfter.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
+            Assert.NotNull(finalState);
+            var finalPending = finalState!.Cursor.TryGetValue("pendingDocuments", out var finalPendingDocs)
+                ? finalPendingDocs.AsBsonArray
+                : new BsonArray();
+            Assert.Empty(finalPending);
+
+            var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var finalPendingMappingsValue)
+                ? finalPendingMappingsValue.AsBsonArray
+                : new BsonArray();
+            Assert.Empty(finalPendingMappings);
+        }
+        finally
+        {
+            await DropDatabaseAsync(databaseName);
+        }
+    }
+
+    [Fact]
+    public async Task Fetch_SkipsUnchangedDocuments()
+    {
+        var databaseName = AllocateDatabaseName();
+        await DropDatabaseAsync(databaseName);
+
+        try
+        {
+            var handler = new CannedHttpMessageHandler();
+            await using var provider = await BuildServiceProviderAsync(handler, databaseName);
+            SeedHttpFixtures(handler);
+
+            var connector = provider.GetRequiredService<ChromiumConnector>();
+            await connector.FetchAsync(provider, CancellationToken.None);
+            await connector.ParseAsync(provider, CancellationToken.None);
+            await connector.MapAsync(provider, CancellationToken.None);
+
+            var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
+            var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
+            Assert.Single(advisories);
+
+            // Re-seed responses and fetch again with unchanged content.
+            SeedHttpFixtures(handler);
+            await connector.FetchAsync(provider, CancellationToken.None);
+
+            var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
+            var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None);
+            Assert.NotNull(state);
+            var cursor = state!.Cursor;
+            var pendingDocuments = cursor.TryGetValue("pendingDocuments", out var pendingDocsValue)
+                ? pendingDocsValue.AsBsonArray
+                : new BsonArray();
+            Assert.Empty(pendingDocuments);
+
+            var pendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappingsValue)
+                ? pendingMappingsValue.AsBsonArray
+                : new BsonArray();
+            Assert.Empty(pendingMappings);
+        }
+        finally
+        {
+            await DropDatabaseAsync(databaseName);
+        }
+    }
+
+    private async Task<ServiceProvider> BuildServiceProviderAsync(CannedHttpMessageHandler handler, string databaseName)
+    {
+        var services = new ServiceCollection();
+        services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
+        services.AddSingleton<TimeProvider>(_timeProvider);
+        services.AddSingleton(handler);
+
+        services.AddMongoStorage(options =>
+        {
+            options.ConnectionString = _fixture.Runner.ConnectionString;
+            options.DatabaseName = databaseName;
+            options.CommandTimeout = TimeSpan.FromSeconds(5);
+        });
+
+        services.AddSourceCommon();
+        services.AddChromiumConnector(opts =>
+        {
+            opts.FeedUri = new Uri("https://chromereleases.googleblog.com/atom.xml");
+            opts.InitialBackfill = TimeSpan.FromDays(30);
+            opts.WindowOverlap = TimeSpan.FromDays(1);
+            opts.MaxFeedPages = 1;
+            opts.MaxEntriesPerPage = 50;
+        });
+
+        services.Configure<HttpClientFactoryOptions>(ChromiumOptions.HttpClientName, builderOptions =>
+        {
+            builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
+            {
+                builder.PrimaryHandler = handler;
+            });
+        });
+
+        var provider = services.BuildServiceProvider();
+
+        var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
+        await bootstrapper.InitializeAsync(CancellationToken.None);
+
+        return provider;
+    }
+
+    private string AllocateDatabaseName()
+    {
+        var name = $"chromium-tests-{Guid.NewGuid():N}";
+        _allocatedDatabases.Add(name);
+        return name;
+    }
+
+    private async Task DropDatabaseAsync(string databaseName)
+    {
+        try
+        {
+            await _fixture.Client.DropDatabaseAsync(databaseName);
+        }
+        catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound")
+        {
+        }
+    }
+
+    private static void SeedHttpFixtures(CannedHttpMessageHandler handler)
+    {
+        var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false");
+        var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html");
+
+        handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml");
+        handler.AddTextResponse(detailUri, ReadFixture("chromium-detail.html"), "text/html");
+    }
+
+    private static string ReadFixture(string filename)
+    {
+        var path = ResolveFixturePath(filename);
+        return File.ReadAllText(path);
+    }
+
+    private static string ResolveFixturePath(string filename)
+    {
+        var baseDirectory = AppContext.BaseDirectory;
+        var primary = Path.Combine(baseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename);
+        if (File.Exists(primary))
+        {
+            return primary;
+        }
+
+        return Path.Combine(baseDirectory, "Chromium", "Fixtures", filename);
+    }
+
+    public Task InitializeAsync() => Task.CompletedTask;
+
+    public async Task DisposeAsync()
+    {
+        foreach (var name in _allocatedDatabases.Distinct(StringComparer.Ordinal))
+        {
+            await DropDatabaseAsync(name);
+        }
+    }
+}

src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Linq;
+using StellaOps.Concelier.Connector.Vndr.Chromium;
+using StellaOps.Concelier.Connector.Vndr.Chromium.Internal;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Vndr.Chromium.Tests;
+
+public sealed class ChromiumMapperTests
+{
+    [Fact]
+    public void Map_DeduplicatesReferencesAndOrdersDeterministically()
+    {
+        var published = new DateTimeOffset(2024, 9, 12, 14, 0, 0, TimeSpan.Zero);
+        var metadata = new ChromiumDocumentMetadata(
+            "post-123",
+            "Stable Channel Update",
+            new Uri("https://chromium.example/stable-update.html"),
+            published,
+            null,
+            "Security fixes");
+
+        var dto = ChromiumDto.From(
+            metadata,
+            new[] { "CVE-2024-0001" },
+            new[] { "windows" },
+            new[] { new ChromiumVersionInfo("windows", "stable", "128.0.6613.88") },
+            new[]
+            {
+                new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1"),
+                new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1 duplicate"),
+                new ChromiumReference("https://chromium.example/ref2", "patch", "Ref 2"),
+            });
+
+        var (advisory, _) = ChromiumMapper.Map(dto, VndrChromiumConnectorPlugin.SourceName, published);
+
+        var referenceUrls = advisory.References.Select(r => r.Url).ToArray();
+        Assert.Equal(
+            new[]
+            {
+                "https://chromium.example/ref1",
+                "https://chromium.example/ref2",
+                "https://chromium.example/stable-update.html",
+            },
+            referenceUrls);
+    }
+}

src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Stable Channel Update for Desktop</title>
+  </head>
+  <body>
+    <div class="post-body" id="post-body-123456789">
+      <p>The Stable channel has been updated to 128.0.6613.138 for Windows and macOS, and 128.0.6613.137 for Linux. A full list of changes in this build is available in the <a href="https://chromium.googlesource.com/chromium/src/+log/128.0.6613.120..128.0.6613.138">log</a>.</p>
+      <p>The Extended Stable channel has been updated to 128.0.6613.138 for Windows and Mac and will roll out over the coming days.</p>
+      <p>The team is also rolling out Chrome 128.0.6613.89 to Android.</p>
+      <h2>Security Fixes and Rewards</h2>
+      <p>We would like to thank all security researchers who worked with us during the development cycle.</p>
+      <ul>
+        <li>CVE-2024-12345: Use after free in Blink.</li>
+        <li>CVE-2024-22222: Heap buffer overflow in GPU.</li>
+      </ul>
+      <p>For details see the <a href="https://issues.chromium.org/issues/123456789">issue tracker</a> and the <a href="https://chromium.org/Home/chromium-security">security page</a>.</p>
+    </div>
+  </body>
+</html>

src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <id>tag:blogger.com,1999:blog-8982037438137564684</id>
+  <updated>2024-09-10T18:00:00Z</updated>
+  <title type="text">Google Chrome Releases</title>
+  <link rel="self" type="application/atom+xml" href="https://chromereleases.googleblog.com/atom.xml" />
+  <entry>
+    <id>tag:blogger.com,1999:blog-8982037438137564684.post-123456789</id>
+    <published>2024-09-10T17:30:00Z</published>
+    <updated>2024-09-10T17:45:00Z</updated>
+    <title type="text">Stable Channel Update for Desktop</title>
+    <summary type="html">Stable channel update rolling out to Windows, macOS, Linux.</summary>
+    <category term="Stable updates" />
+    <link rel="alternate" type="text/html" href="https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html" />
+  </entry>
+</feed>

src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
+    <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
+    <ProjectReference Include="../StellaOps.Concelier.Connector.Vndr.Chromium/StellaOps.Concelier.Connector.Vndr.Chromium.csproj" />
+    <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="Chromium/Fixtures/*.html" CopyToOutputDirectory="Always" />
+    <None Include="Chromium/Fixtures/*.xml" CopyToOutputDirectory="Always" />
+    <None Include="Chromium/Fixtures/*.json" CopyToOutputDirectory="Always" />
+  </ItemGroup>
+</Project>