feat: Implement BerkeleyDB reader for RPM databases
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
console-runner-image / build-runner-image (push) Has been cancelled
wine-csp-build / Build Wine CSP Image (push) Has been cancelled
wine-csp-build / Integration Tests (push) Has been cancelled
wine-csp-build / Security Scan (push) Has been cancelled
wine-csp-build / Generate SBOM (push) Has been cancelled
wine-csp-build / Publish Image (push) Has been cancelled
wine-csp-build / Air-Gap Bundle (push) Has been cancelled
wine-csp-build / Test Summary (push) Has been cancelled

- Added BerkeleyDbReader class to read and extract RPM header blobs from BerkeleyDB hash databases.
- Implemented methods to detect BerkeleyDB format and extract values, including handling of page sizes and magic numbers.
- Added tests for BerkeleyDbReader to ensure correct functionality and header extraction.

feat: Add Yarn PnP data tests

- Created YarnPnpDataTests to validate package resolution and data loading from Yarn PnP cache.
- Implemented tests for resolved keys, package presence, and loading from cache structure.

test: Add egg-info package fixtures for Python tests

- Created egg-info package fixtures for testing Python analyzers.
- Included PKG-INFO, entry_points.txt, and installed-files.txt for comprehensive coverage.

test: Enhance RPM database reader tests

- Added tests for RpmDatabaseReader to validate fallback to legacy packages when SQLite is missing.
- Implemented helper methods to create legacy package files and RPM headers for testing.

test: Implement dual signing tests

- Added DualSignTests to validate secondary signature addition when configured.
- Created stub implementations for crypto providers and key resolvers to facilitate testing.

chore: Update CI script for Playwright Chromium installation

- Modified ci-console-exports.sh to ensure deterministic Chromium binary installation for console exports tests.
- Added checks for Windows compatibility and environment variable setups for Playwright browsers.
This commit is contained in:
StellaOps Bot
2025-12-07 16:24:45 +02:00
parent e3f28a21ab
commit 11597679ed
199 changed files with 9809 additions and 4404 deletions

View File

@@ -1,5 +1,4 @@
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Postgres.Advisories;
@@ -7,66 +6,49 @@ using StellaOps.Concelier.Storage.Postgres.Advisories;
namespace StellaOps.Concelier.WebService.DualWrite;
/// <summary>
/// Dual-write advisory store that writes to both MongoDB and PostgreSQL simultaneously.
/// Used during migration to verify parity between backends.
/// Postgres-backed advisory store that implements the legacy Mongo contracts.
/// </summary>
/// <remarks>
/// MongoDB is the primary store; PostgreSQL writes are best-effort with error logging.
/// Read operations are always served from MongoDB.
/// </remarks>
public sealed class DualWriteAdvisoryStore : IAdvisoryStore
{
private readonly AdvisoryStore _mongoStore;
private readonly IPostgresAdvisoryStore _postgresStore;
private readonly ILogger<DualWriteAdvisoryStore> _logger;
public DualWriteAdvisoryStore(
AdvisoryStore mongoStore,
IPostgresAdvisoryStore postgresStore,
ILogger<DualWriteAdvisoryStore> logger)
public DualWriteAdvisoryStore(IPostgresAdvisoryStore postgresStore, ILogger<DualWriteAdvisoryStore> logger)
{
_mongoStore = mongoStore ?? throw new ArgumentNullException(nameof(mongoStore));
_postgresStore = postgresStore ?? throw new ArgumentNullException(nameof(postgresStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc />
public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null)
public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken)
{
// Write to MongoDB (primary)
await _mongoStore.UpsertAsync(advisory, cancellationToken, session).ConfigureAwait(false);
// Write to PostgreSQL (secondary, best-effort)
try
{
await _postgresStore.UpsertAsync(advisory, sourceId: null, cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Dual-write success for advisory {AdvisoryKey}", advisory.AdvisoryKey);
_logger.LogDebug("Stored advisory {AdvisoryKey} in PostgreSQL", advisory.AdvisoryKey);
}
catch (Exception ex)
{
// Log but don't fail - MongoDB is primary during migration
_logger.LogWarning(ex, "Dual-write to PostgreSQL failed for advisory {AdvisoryKey}. MongoDB write succeeded.", advisory.AdvisoryKey);
_logger.LogWarning(ex, "PostgreSQL advisory write failed for {AdvisoryKey}", advisory.AdvisoryKey);
throw;
}
}
/// <inheritdoc />
public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null)
public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
{
// Always read from MongoDB during dual-write mode
return _mongoStore.FindAsync(advisoryKey, cancellationToken, session);
return _postgresStore.FindAsync(advisoryKey, cancellationToken);
}
/// <inheritdoc />
public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
{
// Always read from MongoDB during dual-write mode
return _mongoStore.GetRecentAsync(limit, cancellationToken, session);
return _postgresStore.GetRecentAsync(limit, cancellationToken);
}
/// <inheritdoc />
public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken)
{
// Always read from MongoDB during dual-write mode
return _mongoStore.StreamAsync(cancellationToken, session);
return _postgresStore.StreamAsync(cancellationToken);
}
}