feat: Implement BerkeleyDB reader for RPM databases

- Added BerkeleyDbReader class to read and extract RPM header blobs from BerkeleyDB hash databases.
- Implemented methods to detect BerkeleyDB format and extract values, including handling of page sizes and magic numbers.
- Added tests for BerkeleyDbReader to ensure correct functionality and header extraction.

feat: Add Yarn PnP data tests

- Created YarnPnpDataTests to validate package resolution and data loading from Yarn PnP cache.
- Implemented tests for resolved keys, package presence, and loading from cache structure.

test: Add egg-info package fixtures for Python tests

- Created egg-info package fixtures for testing Python analyzers.
- Included PKG-INFO, entry_points.txt, and installed-files.txt for comprehensive coverage.

test: Enhance RPM database reader tests

- Added tests for RpmDatabaseReader to validate fallback to legacy packages when SQLite is missing.
- Implemented helper methods to create legacy package files and RPM headers for testing.

test: Implement dual signing tests

- Added DualSignTests to validate secondary signature addition when configured.
- Created stub implementations for crypto providers and key resolvers to facilitate testing.

chore: Update CI script for Playwright Chromium installation

- Modified ci-console-exports.sh to ensure deterministic Chromium binary installation for console exports tests.
- Added checks for Windows compatibility and environment variable setups for Playwright browsers.

src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs

@@ -68,7 +68,7 @@ public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, Date
     {
         return value.BsonType switch
         {
-            BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
+            BsonType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
             _ => null,
         };

src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/RawDocumentStorage.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.IO;
+using StellaOps.Concelier.Storage.Mongo;
 
 namespace StellaOps.Concelier.Connector.Common.Fetch;
 
@@ -9,6 +10,12 @@ namespace StellaOps.Concelier.Connector.Common.Fetch;
 public sealed class RawDocumentStorage
 {
     private readonly ConcurrentDictionary<Guid, byte[]> _blobs = new();
+    private readonly IDocumentStore? _documentStore;
+
+    public RawDocumentStorage(IDocumentStore? documentStore = null)
+    {
+        _documentStore = documentStore;
+    }
 
     public Task<Guid> UploadAsync(
         string sourceName,
@@ -16,7 +23,7 @@ public sealed class RawDocumentStorage
         byte[] content,
         string? contentType,
         CancellationToken cancellationToken)
-        => UploadAsync(sourceName, uri, content, contentType, expiresAt: null, cancellationToken);
+        => UploadAsync(sourceName, uri, content, contentType, ExpiresAt: null, cancellationToken);
 
     public async Task<Guid> UploadAsync(
         string sourceName,
@@ -39,11 +46,21 @@ public sealed class RawDocumentStorage
         return id;
     }
 
-    public Task<byte[]> DownloadAsync(Guid id, CancellationToken cancellationToken)
+    public async Task<byte[]> DownloadAsync(Guid id, CancellationToken cancellationToken)
     {
         if (_blobs.TryGetValue(id, out var bytes))
         {
-            return Task.FromResult(bytes);
+            return bytes;
+        }
+
+        if (_documentStore is not null)
+        {
+            var record = await _documentStore.FindAsync(id, cancellationToken).ConfigureAwait(false);
+            if (record?.Payload is { Length: > 0 })
+            {
+                _blobs[id] = record.Payload;
+                return record.Payload;
+            }
         }
 
         throw new FileNotFoundException($"Blob {id} not found.");

src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/ServiceCollectionExtensions.cs

@@ -9,160 +9,160 @@ using StellaOps.Concelier.Connector.Common.Xml;
 using StellaOps.Concelier.Core.Aoc;
 using StellaOps.Concelier.Core.Linksets;
 using StellaOps.Concelier.Storage.Mongo;
-
-namespace StellaOps.Concelier.Connector.Common.Http;
-
-public static class ServiceCollectionExtensions
-{
-    /// <summary>
-    /// Registers a named HTTP client configured for a source connector with allowlisted hosts and sensible defaults.
-    /// </summary>
-    public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action<SourceHttpClientOptions> configure)
-        => services.AddSourceHttpClient(name, (_, options) => configure(options));
-
-    public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action<IServiceProvider, SourceHttpClientOptions> configure)
-    {
-        ArgumentNullException.ThrowIfNull(services);
-        ArgumentException.ThrowIfNullOrEmpty(name);
-        ArgumentNullException.ThrowIfNull(configure);
-
-        services.AddOptions<SourceHttpClientOptions>(name).Configure<IServiceProvider>((options, sp) =>
-        {
-            configure(sp, options);
-            SourceHttpClientConfigurationBinder.Apply(sp, name, options);
-        });
-
-        return services
-            .AddHttpClient(name)
-            .ConfigureHttpClient((sp, client) =>
-            {
-                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name);
-
-                if (options.BaseAddress is not null)
-                {
-                    client.BaseAddress = options.BaseAddress;
-                }
-
-                client.Timeout = options.Timeout;
-                client.DefaultRequestHeaders.UserAgent.Clear();
-                client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent);
-                client.DefaultRequestVersion = options.RequestVersion;
-                client.DefaultVersionPolicy = options.VersionPolicy;
-
-                foreach (var header in options.DefaultRequestHeaders)
-                {
-                    client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
-                }
-            })
-            .ConfigurePrimaryHttpMessageHandler((sp) =>
-            {
-                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name).Clone();
-                var handler = new SocketsHttpHandler
-                {
-                    AllowAutoRedirect = options.AllowAutoRedirect,
-                    AutomaticDecompression = DecompressionMethods.All,
-                    EnableMultipleHttp2Connections = options.EnableMultipleHttp2Connections,
-                };
-                options.ConfigureHandler?.Invoke(handler);
-                ApplyProxySettings(handler, options);
-
-                if (options.ServerCertificateCustomValidation is not null)
-                {
-                    handler.SslOptions.RemoteCertificateValidationCallback = (_, certificate, chain, sslPolicyErrors) =>
-                    {
-                        X509Certificate2? certToValidate = certificate as X509Certificate2;
-                        X509Certificate2? disposable = null;
-                        if (certToValidate is null && certificate is not null)
-                        {
-                            disposable = X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert));
-                            certToValidate = disposable;
-                        }
-
-                        try
-                        {
-                            return options.ServerCertificateCustomValidation(certToValidate, chain, sslPolicyErrors);
-                        }
-                        finally
-                        {
-                            disposable?.Dispose();
-                        }
-                    };
-                }
-                else if (options.TrustedRootCertificates.Count > 0 && handler.SslOptions.RemoteCertificateValidationCallback is null)
-                {
-                    handler.SslOptions.RemoteCertificateValidationCallback = (_, certificate, chain, errors) =>
-                    {
-                        if (errors == SslPolicyErrors.None)
-                        {
-                            return true;
-                        }
-
-                        if (certificate is null)
-                        {
-                            return false;
-                        }
-
-                        X509Certificate2? certToValidate = certificate as X509Certificate2;
-                        X509Certificate2? disposable = null;
-                        var trustedRootCopies = new X509Certificate2Collection();
-                        try
-                        {
-                            if (certToValidate is null)
-                            {
-                                disposable = X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert));
-                                certToValidate = disposable;
-                            }
-
-                            foreach (var root in options.TrustedRootCertificates)
-                            {
-                                trustedRootCopies.Add(new X509Certificate2(root.RawData));
-                            }
-
-                            using var customChain = new X509Chain();
-                            customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
-                            customChain.ChainPolicy.CustomTrustStore.Clear();
-                            customChain.ChainPolicy.CustomTrustStore.AddRange(trustedRootCopies);
-                            customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
-                            customChain.ChainPolicy.VerificationFlags = X509VerificationFlags.NoFlag;
-
-                            if (chain is not null)
-                            {
-                                foreach (var element in chain.ChainElements)
-                                {
-                                    customChain.ChainPolicy.ExtraStore.Add(element.Certificate);
-                                }
-                            }
-
-                            return certToValidate is not null && customChain.Build(certToValidate);
-                        }
-                        finally
-                        {
-                            foreach (X509Certificate2 root in trustedRootCopies)
-                            {
-                                root.Dispose();
-                            }
-
-                            disposable?.Dispose();
-                        }
-                    };
-                }
-
-                return handler;
-            })
-            .AddHttpMessageHandler(sp =>
-            {
-                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name).Clone();
-                return new AllowlistedHttpMessageHandler(options);
-            });
-    }
-
-    /// <summary>
-    /// Registers shared helpers used by source connectors.
-    /// </summary>
-    public static IServiceCollection AddSourceCommon(this IServiceCollection services)
-    {
-        ArgumentNullException.ThrowIfNull(services);
-
+
+namespace StellaOps.Concelier.Connector.Common.Http;
+
+public static class ServiceCollectionExtensions
+{
+    /// <summary>
+    /// Registers a named HTTP client configured for a source connector with allowlisted hosts and sensible defaults.
+    /// </summary>
+    public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action<SourceHttpClientOptions> configure)
+        => services.AddSourceHttpClient(name, (_, options) => configure(options));
+
+    public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action<IServiceProvider, SourceHttpClientOptions> configure)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentException.ThrowIfNullOrEmpty(name);
+        ArgumentNullException.ThrowIfNull(configure);
+
+        services.AddOptions<SourceHttpClientOptions>(name).Configure<IServiceProvider>((options, sp) =>
+        {
+            configure(sp, options);
+            SourceHttpClientConfigurationBinder.Apply(sp, name, options);
+        });
+
+        return services
+            .AddHttpClient(name)
+            .ConfigureHttpClient((sp, client) =>
+            {
+                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name);
+
+                if (options.BaseAddress is not null)
+                {
+                    client.BaseAddress = options.BaseAddress;
+                }
+
+                client.Timeout = options.Timeout;
+                client.DefaultRequestHeaders.UserAgent.Clear();
+                client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent);
+                client.DefaultRequestVersion = options.RequestVersion;
+                client.DefaultVersionPolicy = options.VersionPolicy;
+
+                foreach (var header in options.DefaultRequestHeaders)
+                {
+                    client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
+                }
+            })
+            .ConfigurePrimaryHttpMessageHandler((sp) =>
+            {
+                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name).Clone();
+                var handler = new SocketsHttpHandler
+                {
+                    AllowAutoRedirect = options.AllowAutoRedirect,
+                    AutomaticDecompression = DecompressionMethods.All,
+                    EnableMultipleHttp2Connections = options.EnableMultipleHttp2Connections,
+                };
+                options.ConfigureHandler?.Invoke(handler);
+                ApplyProxySettings(handler, options);
+
+                if (options.ServerCertificateCustomValidation is not null)
+                {
+                    handler.SslOptions.RemoteCertificateValidationCallback = (_, certificate, chain, sslPolicyErrors) =>
+                    {
+                        X509Certificate2? certToValidate = certificate as X509Certificate2;
+                        X509Certificate2? disposable = null;
+                        if (certToValidate is null && certificate is not null)
+                        {
+                            disposable = X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert));
+                            certToValidate = disposable;
+                        }
+
+                        try
+                        {
+                            return options.ServerCertificateCustomValidation(certToValidate, chain, sslPolicyErrors);
+                        }
+                        finally
+                        {
+                            disposable?.Dispose();
+                        }
+                    };
+                }
+                else if (options.TrustedRootCertificates.Count > 0 && handler.SslOptions.RemoteCertificateValidationCallback is null)
+                {
+                    handler.SslOptions.RemoteCertificateValidationCallback = (_, certificate, chain, errors) =>
+                    {
+                        if (errors == SslPolicyErrors.None)
+                        {
+                            return true;
+                        }
+
+                        if (certificate is null)
+                        {
+                            return false;
+                        }
+
+                        X509Certificate2? certToValidate = certificate as X509Certificate2;
+                        X509Certificate2? disposable = null;
+                        var trustedRootCopies = new X509Certificate2Collection();
+                        try
+                        {
+                            if (certToValidate is null)
+                            {
+                                disposable = X509CertificateLoader.LoadCertificate(certificate.Export(X509ContentType.Cert));
+                                certToValidate = disposable;
+                            }
+
+                            foreach (var root in options.TrustedRootCertificates)
+                            {
+                                trustedRootCopies.Add(new X509Certificate2(root.RawData));
+                            }
+
+                            using var customChain = new X509Chain();
+                            customChain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
+                            customChain.ChainPolicy.CustomTrustStore.Clear();
+                            customChain.ChainPolicy.CustomTrustStore.AddRange(trustedRootCopies);
+                            customChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck;
+                            customChain.ChainPolicy.VerificationFlags = X509VerificationFlags.NoFlag;
+
+                            if (chain is not null)
+                            {
+                                foreach (var element in chain.ChainElements)
+                                {
+                                    customChain.ChainPolicy.ExtraStore.Add(element.Certificate);
+                                }
+                            }
+
+                            return certToValidate is not null && customChain.Build(certToValidate);
+                        }
+                        finally
+                        {
+                            foreach (X509Certificate2 root in trustedRootCopies)
+                            {
+                                root.Dispose();
+                            }
+
+                            disposable?.Dispose();
+                        }
+                    };
+                }
+
+                return handler;
+            })
+            .AddHttpMessageHandler(sp =>
+            {
+                var options = sp.GetRequiredService<IOptionsMonitor<SourceHttpClientOptions>>().Get(name).Clone();
+                return new AllowlistedHttpMessageHandler(options);
+            });
+    }
+
+    /// <summary>
+    /// Registers shared helpers used by source connectors.
+    /// </summary>
+    public static IServiceCollection AddSourceCommon(this IServiceCollection services)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+
         services.AddSingleton<Json.JsonSchemaValidator>();
         services.AddSingleton<Json.IJsonSchemaValidator>(sp => sp.GetRequiredService<Json.JsonSchemaValidator>());
         services.AddSingleton<XmlSchemaValidator>();
@@ -170,40 +170,40 @@ public static class ServiceCollectionExtensions
         services.AddSingleton<Fetch.IJitterSource, Fetch.CryptoJitterSource>();
         services.AddConcelierAocGuards();
         services.AddConcelierLinksetMappers();
-        services.TryAddSingleton<IDocumentStore, InMemoryDocumentStore>();
-        services.AddSingleton<Fetch.RawDocumentStorage>();
-        services.AddSingleton<Fetch.SourceFetchService>();
+        services.TryAddScoped<IDocumentStore, InMemoryDocumentStore>();
+        services.AddScoped<Fetch.RawDocumentStorage>();
+        services.AddScoped<Fetch.SourceFetchService>();
 
         return services;
     }
-
-    private static void ApplyProxySettings(SocketsHttpHandler handler, SourceHttpClientOptions options)
-    {
-        if (options.ProxyAddress is null)
-        {
-            return;
-        }
-
-        var proxy = new WebProxy(options.ProxyAddress)
-        {
-            BypassProxyOnLocal = options.ProxyBypassOnLocal,
-            UseDefaultCredentials = options.ProxyUseDefaultCredentials,
-        };
-
-        if (options.ProxyBypassList.Count > 0)
-        {
-            proxy.BypassList = options.ProxyBypassList.ToArray();
-        }
-
-        if (!options.ProxyUseDefaultCredentials
-            && !string.IsNullOrWhiteSpace(options.ProxyUsername))
-        {
-            proxy.Credentials = new NetworkCredential(
-                options.ProxyUsername,
-                options.ProxyPassword ?? string.Empty);
-        }
-
-        handler.Proxy = proxy;
-        handler.UseProxy = true;
-    }
-}
+
+    private static void ApplyProxySettings(SocketsHttpHandler handler, SourceHttpClientOptions options)
+    {
+        if (options.ProxyAddress is null)
+        {
+            return;
+        }
+
+        var proxy = new WebProxy(options.ProxyAddress)
+        {
+            BypassProxyOnLocal = options.ProxyBypassOnLocal,
+            UseDefaultCredentials = options.ProxyUseDefaultCredentials,
+        };
+
+        if (options.ProxyBypassList.Count > 0)
+        {
+            proxy.BypassList = options.ProxyBypassList.ToArray();
+        }
+
+        if (!options.ProxyUseDefaultCredentials
+            && !string.IsNullOrWhiteSpace(options.ProxyUsername))
+        {
+            proxy.Credentials = new NetworkCredential(
+                options.ProxyUsername,
+                options.ProxyPassword ?? string.Empty);
+        }
+
+        handler.Proxy = proxy;
+        handler.UseProxy = true;
+    }
+}

src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs

@@ -144,18 +144,21 @@ public sealed class SourceStateSeedProcessor
 
         var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false);
 
+        var recordId = document.DocumentId ?? existing?.Id ?? Guid.NewGuid();
+
         if (existing?.PayloadId is { } oldGridId)
         {
             await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false);
         }
 
-        var gridId = await _rawDocumentStorage.UploadAsync(
+        _ = await _rawDocumentStorage.UploadAsync(
                 source,
                 document.Uri,
                 payload,
                 document.ContentType,
                 document.ExpiresAt,
-                cancellationToken)
+                cancellationToken,
+                recordId)
             .ConfigureAwait(false);
 
         var headers = CloneDictionary(document.Headers);
@@ -171,7 +174,7 @@ public sealed class SourceStateSeedProcessor
         var metadata = CloneDictionary(document.Metadata);
 
         var record = new MongoContracts.DocumentRecord(
-            document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
+            recordId,
             source,
             document.Uri,
             document.FetchedAt ?? completedAt,
@@ -182,8 +185,9 @@ public sealed class SourceStateSeedProcessor
             metadata,
             document.Etag,
             document.LastModified,
-            gridId,
-            document.ExpiresAt);
+            recordId,
+            document.ExpiresAt,
+            payload);
 
         var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);