feat: Implement BerkeleyDB reader for RPM databases
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
console-runner-image / build-runner-image (push) Has been cancelled
wine-csp-build / Build Wine CSP Image (push) Has been cancelled
wine-csp-build / Integration Tests (push) Has been cancelled
wine-csp-build / Security Scan (push) Has been cancelled
wine-csp-build / Generate SBOM (push) Has been cancelled
wine-csp-build / Publish Image (push) Has been cancelled
wine-csp-build / Air-Gap Bundle (push) Has been cancelled
wine-csp-build / Test Summary (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
console-runner-image / build-runner-image (push) Has been cancelled
wine-csp-build / Build Wine CSP Image (push) Has been cancelled
wine-csp-build / Integration Tests (push) Has been cancelled
wine-csp-build / Security Scan (push) Has been cancelled
wine-csp-build / Generate SBOM (push) Has been cancelled
wine-csp-build / Publish Image (push) Has been cancelled
wine-csp-build / Air-Gap Bundle (push) Has been cancelled
wine-csp-build / Test Summary (push) Has been cancelled
- Added BerkeleyDbReader class to read and extract RPM header blobs from BerkeleyDB hash databases. - Implemented methods to detect BerkeleyDB format and extract values, including handling of page sizes and magic numbers. - Added tests for BerkeleyDbReader to ensure correct functionality and header extraction. feat: Add Yarn PnP data tests - Created YarnPnpDataTests to validate package resolution and data loading from Yarn PnP cache. - Implemented tests for resolved keys, package presence, and loading from cache structure. test: Add egg-info package fixtures for Python tests - Created egg-info package fixtures for testing Python analyzers. - Included PKG-INFO, entry_points.txt, and installed-files.txt for comprehensive coverage. test: Enhance RPM database reader tests - Added tests for RpmDatabaseReader to validate fallback to legacy packages when SQLite is missing. - Implemented helper methods to create legacy package files and RPM headers for testing. test: Implement dual signing tests - Added DualSignTests to validate secondary signature addition when configured. - Created stub implementations for crypto providers and key resolvers to facilitate testing. chore: Update CI script for Playwright Chromium installation - Modified ci-console-exports.sh to ensure deterministic Chromium binary installation for console exports tests. - Added checks for Windows compatibility and environment variable setups for Playwright browsers.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Buffers.Binary;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using Microsoft.Data.Sqlite;
|
||||
@@ -23,8 +24,8 @@ internal sealed class RpmDatabaseReader : IRpmDatabaseReader
|
||||
var sqlitePath = ResolveSqlitePath(rootPath);
|
||||
if (sqlitePath is null)
|
||||
{
|
||||
_logger.LogWarning("rpmdb.sqlite not found under root {RootPath}; rpm analyzer will skip.", rootPath);
|
||||
return Array.Empty<RpmHeader>();
|
||||
_logger.LogWarning("rpmdb.sqlite not found under root {RootPath}; attempting legacy rpmdb fallback.", rootPath);
|
||||
return ReadLegacyHeaders(rootPath, cancellationToken);
|
||||
}
|
||||
|
||||
var headers = new List<RpmHeader>();
|
||||
@@ -65,7 +66,12 @@ internal sealed class RpmDatabaseReader : IRpmDatabaseReader
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to read rpmdb.sqlite at {Path}.", sqlitePath);
|
||||
return Array.Empty<RpmHeader>();
|
||||
return ReadLegacyHeaders(rootPath, cancellationToken);
|
||||
}
|
||||
|
||||
if (headers.Count == 0)
|
||||
{
|
||||
return ReadLegacyHeaders(rootPath, cancellationToken);
|
||||
}
|
||||
|
||||
return headers;
|
||||
@@ -90,6 +96,230 @@ internal sealed class RpmDatabaseReader : IRpmDatabaseReader
|
||||
return null;
|
||||
}
|
||||
|
||||
private IReadOnlyList<RpmHeader> ReadLegacyHeaders(string rootPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var packagesPath = ResolveLegacyPackagesPath(rootPath);
|
||||
if (packagesPath is null)
|
||||
{
|
||||
_logger.LogWarning("Legacy rpmdb Packages file not found under root {RootPath}; rpm analyzer will skip.", rootPath);
|
||||
return Array.Empty<RpmHeader>();
|
||||
}
|
||||
|
||||
byte[] data;
|
||||
try
|
||||
{
|
||||
data = File.ReadAllBytes(packagesPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to read legacy rpmdb Packages file at {Path}.", packagesPath);
|
||||
return Array.Empty<RpmHeader>();
|
||||
}
|
||||
|
||||
// Detect BerkeleyDB format and use appropriate extraction method
|
||||
if (BerkeleyDbReader.IsBerkeleyDb(data))
|
||||
{
|
||||
_logger.LogDebug("Detected BerkeleyDB format for rpmdb at {Path}; using BDB extraction.", packagesPath);
|
||||
return ReadBerkeleyDbHeaders(data, packagesPath, cancellationToken);
|
||||
}
|
||||
|
||||
// Fall back to raw RPM header scanning for non-BDB files
|
||||
return ReadRawRpmHeaders(data, packagesPath, cancellationToken);
|
||||
}
|
||||
|
||||
private IReadOnlyList<RpmHeader> ReadBerkeleyDbHeaders(byte[] data, string packagesPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<RpmHeader>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Try page-aware extraction first
|
||||
var headerBlobs = BerkeleyDbReader.ExtractValues(data);
|
||||
if (headerBlobs.Count == 0)
|
||||
{
|
||||
// Fall back to overflow-aware extraction for fragmented data
|
||||
headerBlobs = BerkeleyDbReader.ExtractValuesWithOverflow(data);
|
||||
}
|
||||
|
||||
foreach (var blob in headerBlobs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var header = _parser.Parse(blob);
|
||||
var key = $"{header.Name}::{header.Version}::{header.Release}::{header.Architecture}";
|
||||
if (seen.Add(key))
|
||||
{
|
||||
results.Add(header);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to parse RPM header blob from BerkeleyDB.");
|
||||
}
|
||||
}
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("No RPM headers parsed from BerkeleyDB rpmdb at {Path}.", packagesPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Extracted {Count} RPM headers from BerkeleyDB rpmdb at {Path}.", results.Count, packagesPath);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private IReadOnlyList<RpmHeader> ReadRawRpmHeaders(byte[] data, string packagesPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var headerBlobs = new List<byte[]>();
|
||||
|
||||
if (BerkeleyDbReader.IsBerkeleyDb(data))
|
||||
{
|
||||
headerBlobs.AddRange(BerkeleyDbReader.ExtractValues(data));
|
||||
if (headerBlobs.Count == 0)
|
||||
{
|
||||
headerBlobs.AddRange(BerkeleyDbReader.ExtractValuesWithOverflow(data));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
headerBlobs.AddRange(ExtractRpmHeadersFromRaw(data, cancellationToken));
|
||||
}
|
||||
|
||||
if (headerBlobs.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("No RPM headers parsed from legacy rpmdb Packages at {Path}.", packagesPath);
|
||||
return Array.Empty<RpmHeader>();
|
||||
}
|
||||
|
||||
var results = new List<RpmHeader>(headerBlobs.Count);
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var blob in headerBlobs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var header = _parser.Parse(blob);
|
||||
var key = $"{header.Name}::{header.Version}::{header.Release}::{header.Architecture}";
|
||||
if (seen.Add(key))
|
||||
{
|
||||
results.Add(header);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse RPM header from legacy rpmdb blob.");
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static string? ResolveLegacyPackagesPath(string rootPath)
|
||||
{
|
||||
var candidates = new[]
|
||||
{
|
||||
Path.Combine(rootPath, "var", "lib", "rpm", "Packages"),
|
||||
Path.Combine(rootPath, "usr", "lib", "sysimage", "rpm", "Packages"),
|
||||
};
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IEnumerable<byte[]> ExtractRpmHeadersFromRaw(byte[] data, CancellationToken cancellationToken)
|
||||
{
|
||||
var magicBytes = new byte[] { 0x8e, 0xad, 0xe8, 0xab };
|
||||
var seenOffsets = new HashSet<int>();
|
||||
var offset = 0;
|
||||
|
||||
while (offset <= data.Length - magicBytes.Length)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var candidateIndex = FindNextMagic(data, magicBytes, offset);
|
||||
if (candidateIndex < 0)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (!seenOffsets.Add(candidateIndex))
|
||||
{
|
||||
offset = candidateIndex + 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryExtractHeaderSlice(data, candidateIndex, out var slice))
|
||||
{
|
||||
yield return slice;
|
||||
}
|
||||
|
||||
offset = candidateIndex + 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryExtractHeaderSlice(byte[] data, int offset, out byte[] slice)
|
||||
{
|
||||
slice = Array.Empty<byte>();
|
||||
|
||||
if (offset + 16 >= data.Length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var span = data.AsSpan(offset);
|
||||
var indexCount = BinaryPrimitives.ReadInt32BigEndian(span.Slice(8, 4));
|
||||
var storeSize = BinaryPrimitives.ReadInt32BigEndian(span.Slice(12, 4));
|
||||
|
||||
if (indexCount <= 0 || storeSize <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var totalLength = 16 + (indexCount * 16) + storeSize;
|
||||
if (totalLength <= 0 || offset + totalLength > data.Length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
slice = new byte[totalLength];
|
||||
Buffer.BlockCopy(data, offset, slice, 0, totalLength);
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static int FindNextMagic(byte[] data, byte[] magic, int startIndex)
|
||||
{
|
||||
for (var i = startIndex; i <= data.Length - magic.Length; i++)
|
||||
{
|
||||
if (data[i] == magic[0] &&
|
||||
data[i + 1] == magic[1] &&
|
||||
data[i + 2] == magic[2] &&
|
||||
data[i + 3] == magic[3])
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static byte[]? ExtractHeaderBlob(SqliteDataReader reader)
|
||||
{
|
||||
for (var i = 0; i < reader.FieldCount; i++)
|
||||
|
||||
Reference in New Issue
Block a user