Refactor code structure for improved readability and maintainability; optimize performance in key functions.

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineConnectorTests.cs

@@ -0,0 +1,88 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using StellaOps.Concelier.Models;
+using StellaOps.Concelier.Connector.Common;
+using StellaOps.Concelier.Connector.Distro.Alpine;
+using StellaOps.Concelier.Connector.Distro.Alpine.Configuration;
+using StellaOps.Concelier.Storage;
+using StellaOps.Concelier.Storage.Advisories;
+using StellaOps.Concelier.Testing;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+[Collection(ConcelierFixtureCollection.Name)]
+public sealed class AlpineConnectorTests
+{
+    private static readonly Uri SecDbUri = new("https://secdb.alpinelinux.org/v3.20/main.json");
+    private readonly ConcelierPostgresFixture _fixture;
+
+    public AlpineConnectorTests(ConcelierPostgresFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    [Fact]
+    public async Task FetchParseMap_StoresAdvisoriesAndUpdatesCursor()
+    {
+        await using var harness = await BuildHarnessAsync();
+
+        harness.Handler.AddJsonResponse(SecDbUri, BuildMinimalSecDb());
+
+        var connector = harness.ServiceProvider.GetRequiredService<AlpineConnector>();
+        await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None);
+        await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None);
+        await connector.MapAsync(harness.ServiceProvider, CancellationToken.None);
+
+        var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>();
+        var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
+        Assert.Equal(2, advisories.Count);
+
+        var advisory = advisories.Single(item => item.AdvisoryKey == "alpine/cve-2021-36159");
+        var package = Assert.Single(advisory.AffectedPackages);
+        Assert.Equal(AffectedPackageTypes.Apk, package.Type);
+        Assert.Equal("apk-tools", package.Identifier);
+        Assert.Equal("v3.20/main", package.Platform);
+
+        var range = Assert.Single(package.VersionRanges);
+        Assert.Equal("apk", range.RangeKind);
+        Assert.Equal("2.12.6-r0", range.FixedVersion);
+
+        var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
+        var state = await stateRepository.TryGetAsync(AlpineConnectorPlugin.SourceName, CancellationToken.None);
+        Assert.NotNull(state);
+        Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)
+                    && pendingDocs.AsDocumentArray.Count == 0);
+        Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)
+                    && pendingMappings.AsDocumentArray.Count == 0);
+
+        harness.Handler.AssertNoPendingResponses();
+    }
+
+    private async Task<ConnectorTestHarness> BuildHarnessAsync()
+    {
+        var initialTime = new DateTimeOffset(2025, 12, 22, 0, 0, 0, TimeSpan.Zero);
+        var harness = new ConnectorTestHarness(_fixture, initialTime, AlpineOptions.HttpClientName);
+        await harness.EnsureServiceProviderAsync(services =>
+        {
+            services.AddAlpineConnector(options =>
+            {
+                options.BaseUri = new Uri("https://secdb.alpinelinux.org/");
+                options.Releases = new[] { "v3.20" };
+                options.Repositories = new[] { "main" };
+                options.MaxDocumentsPerFetch = 1;
+                options.FetchTimeout = TimeSpan.FromSeconds(5);
+                options.RequestDelay = TimeSpan.Zero;
+                options.UserAgent = "StellaOps.Tests.Alpine/1.0";
+            });
+        });
+
+        return harness;
+    }
+
+    private static string BuildMinimalSecDb()
+        => "{\"distroversion\":\"v3.20\",\"reponame\":\"main\",\"urlprefix\":\"https://dl-cdn.alpinelinux.org/alpine\",\"packages\":[{\"pkg\":{\"name\":\"apk-tools\",\"secfixes\":{\"2.12.6-r0\":[\"CVE-2021-36159\"]}}},{\"pkg\":{\"name\":\"busybox\",\"secfixes\":{\"1.36.1-r29\":[\"CVE-2023-42364\"]}}}]}";
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineDependencyInjectionRoutineTests.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using StellaOps.Concelier.Core.Jobs;
+using StellaOps.Concelier.Connector.Common.Http;
+using StellaOps.Concelier.Connector.Distro.Alpine;
+using StellaOps.Concelier.Connector.Distro.Alpine.Configuration;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+public sealed class AlpineDependencyInjectionRoutineTests
+{
+    [Fact]
+    public void Register_ConfiguresOptionsAndScheduler()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        services.AddOptions();
+        services.AddSourceCommon();
+
+        var configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["concelier:sources:alpine:baseUri"] = "https://secdb.alpinelinux.org/",
+                ["concelier:sources:alpine:releases:0"] = "v3.20",
+                ["concelier:sources:alpine:repositories:0"] = "main",
+                ["concelier:sources:alpine:maxDocumentsPerFetch"] = "5",
+                ["concelier:sources:alpine:fetchTimeout"] = "00:00:30",
+                ["concelier:sources:alpine:requestDelay"] = "00:00:00.100",
+                ["concelier:sources:alpine:userAgent"] = "StellaOps.Tests.Alpine/1.0"
+            })
+            .Build();
+
+        var routine = new AlpineDependencyInjectionRoutine();
+        routine.Register(services, configuration);
+        services.Configure<JobSchedulerOptions>(_ => { });
+
+        using var provider = services.BuildServiceProvider(validateScopes: true);
+
+        var options = provider.GetRequiredService<IOptions<AlpineOptions>>().Value;
+        Assert.Equal(new Uri("https://secdb.alpinelinux.org/"), options.BaseUri);
+        Assert.Equal(new[] { "v3.20" }, options.Releases);
+        Assert.Equal(new[] { "main" }, options.Repositories);
+        Assert.Equal(5, options.MaxDocumentsPerFetch);
+        Assert.Equal(TimeSpan.FromSeconds(30), options.FetchTimeout);
+        Assert.Equal(TimeSpan.FromMilliseconds(100), options.RequestDelay);
+        Assert.Equal("StellaOps.Tests.Alpine/1.0", options.UserAgent);
+
+        var schedulerOptions = provider.GetRequiredService<IOptions<JobSchedulerOptions>>().Value;
+        Assert.True(schedulerOptions.Definitions.TryGetValue(AlpineJobKinds.Fetch, out var fetchDefinition));
+        Assert.True(schedulerOptions.Definitions.TryGetValue(AlpineJobKinds.Parse, out var parseDefinition));
+        Assert.True(schedulerOptions.Definitions.TryGetValue(AlpineJobKinds.Map, out var mapDefinition));
+
+        Assert.Equal(typeof(AlpineFetchJob), fetchDefinition.JobType);
+        Assert.Equal(TimeSpan.FromMinutes(5), fetchDefinition.Timeout);
+        Assert.Equal(TimeSpan.FromMinutes(4), fetchDefinition.LeaseDuration);
+        Assert.Equal("*/30 * * * *", fetchDefinition.CronExpression);
+        Assert.True(fetchDefinition.Enabled);
+
+        Assert.Equal(typeof(AlpineParseJob), parseDefinition.JobType);
+        Assert.Equal(TimeSpan.FromMinutes(6), parseDefinition.Timeout);
+        Assert.Equal(TimeSpan.FromMinutes(4), parseDefinition.LeaseDuration);
+        Assert.Equal("7,37 * * * *", parseDefinition.CronExpression);
+        Assert.True(parseDefinition.Enabled);
+
+        Assert.Equal(typeof(AlpineMapJob), mapDefinition.JobType);
+        Assert.Equal(TimeSpan.FromMinutes(8), mapDefinition.Timeout);
+        Assert.Equal(TimeSpan.FromMinutes(4), mapDefinition.LeaseDuration);
+        Assert.Equal("12,42 * * * *", mapDefinition.CronExpression);
+        Assert.True(mapDefinition.Enabled);
+    }
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineFixtureReader.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using StellaOps.Concelier.Connector.Distro.Alpine.Dto;
+using StellaOps.Concelier.Connector.Distro.Alpine.Internal;
+using StellaOps.Concelier.Merge.Comparers;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+internal static class AlpineFixtureReader
+{
+    private static readonly StringComparer NameComparer = StringComparer.OrdinalIgnoreCase;
+
+    public static AlpineSecDbDto LoadDto(string filename)
+        => AlpineSecDbParser.Parse(ReadFixture(filename));
+
+    public static AlpineSecDbDto FilterPackages(
+        AlpineSecDbDto dto,
+        IReadOnlyCollection<string> packageNames,
+        int maxVersionsPerPackage = 0)
+    {
+        if (packageNames is null || packageNames.Count == 0)
+        {
+            return dto;
+        }
+
+        var allowed = new HashSet<string>(
+            packageNames.Where(static name => !string.IsNullOrWhiteSpace(name))
+                .Select(static name => name.Trim()),
+            NameComparer);
+
+        var packages = dto.Packages
+            .Where(pkg => allowed.Contains(pkg.Name))
+            .Select(pkg => pkg with { Secfixes = TrimSecfixes(pkg.Secfixes, maxVersionsPerPackage) })
+            .OrderBy(pkg => pkg.Name, NameComparer)
+            .ToList();
+
+        return dto with { Packages = packages };
+    }
+
+    public static string NormalizeSnapshot(string value)
+        => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
+
+    public static string ReadFixture(string filename)
+    {
+        var path = ResolveFixturePath(filename);
+        return File.ReadAllText(path);
+    }
+
+    public static string GetWritableFixturePath(string filename)
+        => Path.Combine(GetProjectRoot(), "Source", "Distro", "Alpine", "Fixtures", filename);
+
+    private static string ResolveFixturePath(string filename)
+    {
+        var candidates = new[]
+        {
+            Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Alpine", "Fixtures", filename),
+            Path.Combine(AppContext.BaseDirectory, "Fixtures", filename),
+            Path.Combine(GetProjectRoot(), "Source", "Distro", "Alpine", "Fixtures", filename),
+        };
+
+        foreach (var candidate in candidates)
+        {
+            if (File.Exists(candidate))
+            {
+                return candidate;
+            }
+        }
+
+        throw new FileNotFoundException($"Fixture '{filename}' not found.", filename);
+    }
+
+    private static IReadOnlyDictionary<string, string[]> TrimSecfixes(
+        IReadOnlyDictionary<string, string[]> secfixes,
+        int maxVersions)
+    {
+        if (secfixes is null || secfixes.Count == 0)
+        {
+            return new Dictionary<string, string[]>(NameComparer);
+        }
+
+        if (maxVersions <= 0 || secfixes.Count <= maxVersions)
+        {
+            return new Dictionary<string, string[]>(secfixes, NameComparer);
+        }
+
+        var comparer = Comparer<string>.Create((left, right) => ApkVersionComparer.Instance.Compare(left, right));
+        var orderedKeys = secfixes.Keys.OrderBy(static key => key, comparer).ToList();
+        var skip = Math.Max(0, orderedKeys.Count - maxVersions);
+        var trimmed = new Dictionary<string, string[]>(NameComparer);
+        for (var i = skip; i < orderedKeys.Count; i++)
+        {
+            var key = orderedKeys[i];
+            trimmed[key] = secfixes[key];
+        }
+
+        return trimmed;
+    }
+
+    private static string GetProjectRoot()
+        => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", ".."));
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineMapperTests.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using StellaOps.Concelier.Connector.Common;
+using StellaOps.Concelier.Connector.Distro.Alpine;
+using StellaOps.Concelier.Connector.Distro.Alpine.Dto;
+using StellaOps.Concelier.Connector.Distro.Alpine.Internal;
+using StellaOps.Concelier.Models;
+using StellaOps.Concelier.Storage;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+public sealed class AlpineMapperTests
+{
+    [Fact]
+    public void Map_BuildsApkAdvisoriesWithRanges()
+    {
+        var dto = new AlpineSecDbDto(
+            DistroVersion: "v3.20",
+            RepoName: "main",
+            UrlPrefix: "https://dl-cdn.alpinelinux.org/alpine",
+            Packages: new[]
+            {
+                new AlpinePackageDto(
+                    "apk-tools",
+                    new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
+                    {
+                        ["2.12.6-r0"] = new[] { "CVE-2021-36159" }
+                    }),
+                new AlpinePackageDto(
+                    "busybox",
+                    new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
+                    {
+                        ["1.36.1-r29"] = new[] { "CVE-2023-42364" }
+                    })
+            });
+
+        var recordedAt = new DateTimeOffset(2025, 12, 22, 0, 0, 0, TimeSpan.Zero);
+        var document = BuildDocument("https://secdb.alpinelinux.org/v3.20/main.json", recordedAt);
+
+        var advisories = AlpineMapper.Map(dto, document, recordedAt);
+        Assert.Equal(2, advisories.Count);
+
+        var apkToolsAdvisory = advisories.Single(advisory => advisory.AdvisoryKey == "alpine/cve-2021-36159");
+        Assert.Contains("CVE-2021-36159", apkToolsAdvisory.Aliases);
+
+        var apkPackage = Assert.Single(apkToolsAdvisory.AffectedPackages);
+        Assert.Equal(AffectedPackageTypes.Apk, apkPackage.Type);
+        Assert.Equal("apk-tools", apkPackage.Identifier);
+        Assert.Equal("v3.20/main", apkPackage.Platform);
+
+        var range = Assert.Single(apkPackage.VersionRanges);
+        Assert.Equal("apk", range.RangeKind);
+        Assert.Equal("2.12.6-r0", range.FixedVersion);
+        Assert.Equal("fixed:2.12.6-r0", range.RangeExpression);
+        Assert.NotNull(range.Primitives?.VendorExtensions);
+        Assert.Equal("v3.20", range.Primitives!.VendorExtensions["alpine.distroversion"]);
+        Assert.Equal("main", range.Primitives.VendorExtensions["alpine.repo"]);
+    }
+
+    private static DocumentRecord BuildDocument(string uri, DateTimeOffset recordedAt)
+    {
+        return new DocumentRecord(
+            Guid.NewGuid(),
+            AlpineConnectorPlugin.SourceName,
+            uri,
+            recordedAt,
+            new string('0', 64),
+            DocumentStatuses.Mapped,
+            "application/json",
+            Headers: null,
+            Metadata: null,
+            Etag: null,
+            LastModified: recordedAt,
+            PayloadId: null);
+    }
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineSecDbParserTests.cs

@@ -0,0 +1,26 @@
+using System.Linq;
+using StellaOps.Concelier.Connector.Distro.Alpine.Dto;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+public sealed class AlpineSecDbParserTests
+{
+    [Fact]
+    public void Parse_SecDbFixture_ExtractsPackagesAndMetadata()
+    {
+        var dto = AlpineFixtureReader.LoadDto("v3.20-main.json");
+
+        Assert.Equal("v3.20", dto.DistroVersion);
+        Assert.Equal("main", dto.RepoName);
+        Assert.Equal("https://dl-cdn.alpinelinux.org/alpine", dto.UrlPrefix);
+        Assert.NotEmpty(dto.Packages);
+
+        var apkTools = dto.Packages.Single(pkg => pkg.Name == "apk-tools");
+        Assert.True(apkTools.Secfixes.ContainsKey("2.12.6-r0"));
+        Assert.Contains("CVE-2021-36159", apkTools.Secfixes["2.12.6-r0"]);
+
+        var busybox = dto.Packages.Single(pkg => pkg.Name == "busybox");
+        Assert.True(busybox.Secfixes.Keys.Any());
+    }
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/AlpineSnapshotTests.cs

@@ -0,0 +1,78 @@
+using System;
+using System.IO;
+using System.Linq;
+using StellaOps.Concelier.Documents;
+using StellaOps.Concelier.Connector.Common;
+using StellaOps.Concelier.Connector.Distro.Alpine;
+using StellaOps.Concelier.Connector.Distro.Alpine.Dto;
+using StellaOps.Concelier.Connector.Distro.Alpine.Internal;
+using StellaOps.Concelier.Models;
+using StellaOps.Concelier.Storage;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Distro.Alpine.Tests;
+
+public sealed class AlpineSnapshotTests
+{
+    [Theory]
+    [InlineData("v3.18-main.json", "alpine-v3.18-main.snapshot.json", "2025-12-22T00:00:00Z")]
+    [InlineData("v3.19-main.json", "alpine-v3.19-main.snapshot.json", "2025-12-22T00:10:00Z")]
+    [InlineData("v3.20-main.json", "alpine-v3.20-main.snapshot.json", "2025-12-22T00:20:00Z")]
+    public void Snapshot_FixturesMatchGolden(string fixtureFile, string snapshotFile, string recordedAt)
+    {
+        var dto = AlpineFixtureReader.LoadDto(fixtureFile);
+        var filtered = AlpineFixtureReader.FilterPackages(
+            dto,
+            new[] { "apk-tools", "busybox", "zlib" },
+            maxVersionsPerPackage: 2);
+
+        var recorded = DateTimeOffset.Parse(recordedAt);
+        var document = BuildDocument(filtered, recorded);
+
+        var advisories = AlpineMapper.Map(filtered, document, recorded);
+        var ordered = advisories
+            .OrderBy(advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        var snapshot = AlpineFixtureReader.NormalizeSnapshot(SnapshotSerializer.ToSnapshot(ordered));
+        var snapshotPath = AlpineFixtureReader.GetWritableFixturePath(snapshotFile);
+
+        if (ShouldUpdateGoldens() || !File.Exists(snapshotPath))
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(snapshotPath)!);
+            File.WriteAllText(snapshotPath, snapshot);
+            return;
+        }
+
+        var expected = AlpineFixtureReader.NormalizeSnapshot(File.ReadAllText(snapshotPath));
+        Assert.Equal(expected, snapshot);
+    }
+
+    private static DocumentRecord BuildDocument(AlpineSecDbDto dto, DateTimeOffset recordedAt)
+    {
+        var uri = new Uri(new Uri("https://secdb.alpinelinux.org/"), $"{dto.DistroVersion}/{dto.RepoName}.json");
+        return new DocumentRecord(
+            Guid.NewGuid(),
+            AlpineConnectorPlugin.SourceName,
+            uri.ToString(),
+            recordedAt,
+            new string('0', 64),
+            DocumentStatuses.Mapped,
+            "application/json",
+            Headers: null,
+            Metadata: null,
+            Etag: null,
+            LastModified: recordedAt,
+            PayloadId: null);
+    }
+
+    private static bool ShouldUpdateGoldens()
+        => IsTruthy(Environment.GetEnvironmentVariable("UPDATE_GOLDENS"))
+           || IsTruthy(Environment.GetEnvironmentVariable("DOTNET_TEST_UPDATE_GOLDENS"));
+
+    private static bool IsTruthy(string? value)
+        => !string.IsNullOrWhiteSpace(value)
+           && (string.Equals(value, "1", StringComparison.OrdinalIgnoreCase)
+               || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase)
+               || string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase));
+}

src/Concelier/__Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests/StellaOps.Concelier.Connector.Distro.Alpine.Tests.csproj

@@ -0,0 +1,20 @@
+<?xml version='1.0' encoding='utf-8'?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/StellaOps.Concelier.Connector.Distro.Alpine.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="Source\Distro\Alpine\Fixtures\**\*">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>