save progress

This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions

View File

@@ -97,7 +97,7 @@ public sealed record FingerprintClaimEvidence
public required IReadOnlyList<string> ChangedFunctions { get; init; }
/// <summary>
/// Similarity scores for modified functions (function name score).
/// Similarity scores for modified functions (function name -> score).
/// </summary>
public IReadOnlyDictionary<string, decimal>? FunctionSimilarities { get; init; }

View File

@@ -0,0 +1,17 @@
namespace StellaOps.BinaryIndex.Builders;
/// <summary>
/// Provides GUIDs for deterministic testing.
/// </summary>
public interface IGuidProvider
{
Guid NewGuid();
}
/// <summary>
/// Default GUID provider using <see cref="Guid.NewGuid"/>.
/// </summary>
public sealed class GuidProvider : IGuidProvider
{
public Guid NewGuid() => Guid.NewGuid();
}

View File

@@ -1,4 +1,3 @@
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
namespace StellaOps.BinaryIndex.Builders;
@@ -31,10 +30,13 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
vulnerable.Count, patched.Count);
var changes = new List<FunctionChange>();
var weights = GetEffectiveWeights(options.Weights);
// Index by name for quick lookup
var vulnerableByName = vulnerable.ToDictionary(f => f.Name, f => f);
var patchedByName = patched.ToDictionary(f => f.Name, f => f);
var patchedByNormalizedName = options.FuzzyNameMatching
? BuildNormalizedNameIndex(patched)
: null;
// Track processed functions to find additions
var processedPatched = new HashSet<string>();
@@ -46,7 +48,7 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
{
processedPatched.Add(vulnFunc.Name);
var similarity = ComputeSimilarity(vulnFunc, patchedFunc);
var similarity = ComputeSimilarity(vulnFunc, patchedFunc, weights);
if (similarity >= 1.0m)
{
@@ -86,17 +88,34 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
}
else
{
if (options.FuzzyNameMatching &&
TryGetFuzzyMatch(vulnFunc.Name, patchedByNormalizedName, processedPatched, out var fuzzyMatch))
{
processedPatched.Add(fuzzyMatch.Name);
var similarity = ComputeSimilarity(vulnFunc, fuzzyMatch, weights);
changes.Add(new FunctionChange
{
FunctionName = vulnFunc.Name,
Type = similarity >= options.SimilarityThreshold ? ChangeType.Modified : ChangeType.SignatureChanged,
VulnerableFingerprint = vulnFunc,
PatchedFingerprint = fuzzyMatch,
SimilarityScore = similarity,
DifferingHashes = GetDifferingHashes(vulnFunc, fuzzyMatch)
});
continue;
}
// Not found by name - check if renamed
if (options.DetectRenames)
{
var bestMatch = FindBestMatch(vulnFunc, patched, processedPatched, options.RenameThreshold);
var bestMatch = FindBestMatch(vulnFunc, patched, processedPatched, options.RenameThreshold, weights);
if (bestMatch != null)
{
processedPatched.Add(bestMatch.Name);
var similarity = ComputeSimilarity(vulnFunc, bestMatch);
var similarity = ComputeSimilarity(vulnFunc, bestMatch, weights);
changes.Add(new FunctionChange
{
FunctionName = $"{vulnFunc.Name} {bestMatch.Name}",
FunctionName = $"{vulnFunc.Name} -> {bestMatch.Name}",
Type = ChangeType.Modified,
VulnerableFingerprint = vulnFunc,
PatchedFingerprint = bestMatch,
@@ -156,32 +175,31 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
ArgumentNullException.ThrowIfNull(a);
ArgumentNullException.ThrowIfNull(b);
return ComputeSimilarity(a, b, HashWeights.Default);
}
private static decimal ComputeSimilarity(FunctionFingerprint a, FunctionFingerprint b, HashWeights weights)
{
// Compute weighted similarity based on hash matches
decimal totalWeight = 0m;
decimal matchedWeight = 0m;
// Basic block hash (weight: 0.5)
const decimal bbWeight = 0.5m;
totalWeight += bbWeight;
totalWeight += weights.BasicBlockWeight;
if (HashesEqual(a.BasicBlockHash, b.BasicBlockHash))
{
matchedWeight += bbWeight;
matchedWeight += weights.BasicBlockWeight;
}
// CFG hash (weight: 0.3)
const decimal cfgWeight = 0.3m;
totalWeight += cfgWeight;
totalWeight += weights.CfgWeight;
if (HashesEqual(a.CfgHash, b.CfgHash))
{
matchedWeight += cfgWeight;
matchedWeight += weights.CfgWeight;
}
// String refs hash (weight: 0.2)
const decimal strWeight = 0.2m;
totalWeight += strWeight;
totalWeight += weights.StringRefsWeight;
if (HashesEqual(a.StringRefsHash, b.StringRefsHash))
{
matchedWeight += strWeight;
matchedWeight += weights.StringRefsWeight;
}
// Size similarity bonus (if sizes are within 10%, add small bonus)
@@ -207,7 +225,8 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
ArgumentNullException.ThrowIfNull(vulnerable);
ArgumentNullException.ThrowIfNull(patched);
var mappings = new Dictionary<string, string>();
var mappings = new Dictionary<string, string>(StringComparer.Ordinal);
var patchedByNormalizedName = BuildNormalizedNameIndex(patched);
var usedPatched = new HashSet<string>();
// First pass: exact name matches
@@ -218,6 +237,13 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
{
mappings[vulnFunc.Name] = match.Name;
usedPatched.Add(match.Name);
continue;
}
if (TryGetFuzzyMatch(vulnFunc.Name, patchedByNormalizedName, usedPatched, out var fuzzyMatch))
{
mappings[vulnFunc.Name] = fuzzyMatch.Name;
usedPatched.Add(fuzzyMatch.Name);
}
}
@@ -227,7 +253,7 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
foreach (var vulnFunc in unmatchedVulnerable)
{
var bestMatch = FindBestMatch(vulnFunc, unmatchedPatched, usedPatched, threshold);
var bestMatch = FindBestMatch(vulnFunc, unmatchedPatched, usedPatched, threshold, HashWeights.Default);
if (bestMatch != null)
{
mappings[vulnFunc.Name] = bestMatch.Name;
@@ -242,7 +268,8 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
FunctionFingerprint target,
IReadOnlyList<FunctionFingerprint> candidates,
HashSet<string> excludeNames,
decimal threshold)
decimal threshold,
HashWeights weights)
{
FunctionFingerprint? bestMatch = null;
var bestScore = threshold - 0.001m; // Must exceed threshold
@@ -252,7 +279,7 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
if (excludeNames.Contains(candidate.Name))
continue;
var score = ComputeSimilarity(target, candidate);
var score = ComputeSimilarity(target, candidate, weights);
if (score > bestScore)
{
bestScore = score;
@@ -263,6 +290,88 @@ public sealed class PatchDiffEngine : IPatchDiffEngine
return bestMatch;
}
private HashWeights GetEffectiveWeights(HashWeights weights)
{
if (!weights.IsValid)
{
_logger.LogWarning("Invalid diff weights supplied; using defaults.");
return HashWeights.Default;
}
return weights;
}
private static Dictionary<string, List<FunctionFingerprint>> BuildNormalizedNameIndex(
IReadOnlyList<FunctionFingerprint> fingerprints)
{
var index = new Dictionary<string, List<FunctionFingerprint>>(StringComparer.Ordinal);
foreach (var fingerprint in fingerprints)
{
var key = NormalizeName(fingerprint.Name);
if (!index.TryGetValue(key, out var bucket))
{
bucket = new List<FunctionFingerprint>();
index[key] = bucket;
}
bucket.Add(fingerprint);
}
return index;
}
private static bool TryGetFuzzyMatch(
string name,
Dictionary<string, List<FunctionFingerprint>>? index,
HashSet<string> usedNames,
out FunctionFingerprint match)
{
match = null!;
if (index is null)
{
return false;
}
var normalized = NormalizeName(name);
if (!index.TryGetValue(normalized, out var candidates))
{
return false;
}
foreach (var candidate in candidates)
{
if (usedNames.Contains(candidate.Name))
{
continue;
}
match = candidate;
return true;
}
return false;
}
private static string NormalizeName(string name)
{
if (string.IsNullOrWhiteSpace(name))
{
return string.Empty;
}
var buffer = new char[name.Length];
var index = 0;
foreach (var ch in name)
{
if (char.IsLetterOrDigit(ch))
{
buffer[index++] = char.ToLowerInvariant(ch);
}
}
return new string(buffer, 0, index);
}
private IReadOnlyList<string> GetDifferingHashes(FunctionFingerprint a, FunctionFingerprint b)
{
var differing = new List<string>();

View File

@@ -130,6 +130,8 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
private readonly IPatchDiffEngine _diffEngine;
private readonly IFingerprintClaimRepository _claimRepository;
private readonly IAdvisoryFeedMonitor _advisoryMonitor;
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
/// <summary>
/// Initializes a new instance of <see cref="ReproducibleBuildJob"/>.
@@ -141,7 +143,9 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
IFunctionFingerprintExtractor fingerprintExtractor,
IPatchDiffEngine diffEngine,
IFingerprintClaimRepository claimRepository,
IAdvisoryFeedMonitor advisoryMonitor)
IAdvisoryFeedMonitor advisoryMonitor,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
@@ -150,6 +154,8 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
_diffEngine = diffEngine ?? throw new ArgumentNullException(nameof(diffEngine));
_claimRepository = claimRepository ?? throw new ArgumentNullException(nameof(claimRepository));
_advisoryMonitor = advisoryMonitor ?? throw new ArgumentNullException(nameof(advisoryMonitor));
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new GuidProvider();
}
/// <inheritdoc />
@@ -308,9 +314,17 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
{
var claims = new List<FingerprintClaim>();
var now = _timeProvider.GetUtcNow();
// Create "fixed" claims for patched binaries
foreach (var binary in patchedBuild.Binaries ?? [])
{
if (!TryGetFingerprintId(binary.BuildId, out var fingerprintId))
{
_logger.LogWarning("Skipping patched claim for {CveId}: build id '{BuildId}' is not a GUID.", cve.CveId, binary.BuildId);
continue;
}
var changedFunctions = diff.Changes
.Where(c => c.Type is ChangeType.Modified or ChangeType.Added)
.Select(c => c.FunctionName)
@@ -318,8 +332,8 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
var claim = new FingerprintClaim
{
Id = Guid.NewGuid(),
FingerprintId = Guid.Parse(binary.BuildId), // Assuming BuildId is GUID-like
Id = _guidProvider.NewGuid(),
FingerprintId = fingerprintId,
CveId = cve.CveId,
Verdict = ClaimVerdict.Fixed,
Evidence = new FingerprintClaimEvidence
@@ -332,7 +346,7 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
VulnerableBuildRef = vulnerableBuild.BuildLogRef,
PatchedBuildRef = patchedBuild.BuildLogRef
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
claims.Add(claim);
@@ -341,10 +355,16 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
// Create "vulnerable" claims for vulnerable binaries
foreach (var binary in vulnerableBuild.Binaries ?? [])
{
if (!TryGetFingerprintId(binary.BuildId, out var fingerprintId))
{
_logger.LogWarning("Skipping vulnerable claim for {CveId}: build id '{BuildId}' is not a GUID.", cve.CveId, binary.BuildId);
continue;
}
var claim = new FingerprintClaim
{
Id = Guid.NewGuid(),
FingerprintId = Guid.Parse(binary.BuildId),
Id = _guidProvider.NewGuid(),
FingerprintId = fingerprintId,
CveId = cve.CveId,
Verdict = ClaimVerdict.Vulnerable,
Evidence = new FingerprintClaimEvidence
@@ -356,16 +376,54 @@ public sealed class ReproducibleBuildJob : IReproducibleBuildJob
.ToList(),
VulnerableBuildRef = vulnerableBuild.BuildLogRef
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
claims.Add(claim);
}
if (claims.Count == 0)
{
_logger.LogWarning("No fingerprint claims created for CVE {CveId}; no valid build IDs were available.", cve.CveId);
return;
}
await _claimRepository.CreateClaimsBatchAsync(claims, ct);
_logger.LogDebug(
"Created {Count} fingerprint claims for CVE {CveId}",
claims.Count, cve.CveId);
}
private static bool TryGetFingerprintId(string buildId, out Guid fingerprintId)
{
if (Guid.TryParse(buildId, out fingerprintId))
{
return true;
}
if (buildId.Length == 32 && IsHex(buildId))
{
return Guid.TryParseExact(buildId, "N", out fingerprintId);
}
fingerprintId = Guid.Empty;
return false;
}
private static bool IsHex(string value)
{
foreach (var ch in value)
{
var isHex = ch is >= '0' and <= '9'
or >= 'a' and <= 'f'
or >= 'A' and <= 'F';
if (!isHex)
{
return false;
}
}
return true;
}
}

View File

@@ -23,12 +23,16 @@ public static class ServiceCollectionExtensions
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
// Configuration - register options with defaults (configuration binding happens via host)
services.Configure<BuilderServiceOptions>(options => { });
services.Configure<FunctionExtractionOptions>(options => { });
// Configuration - bind options from configuration
services.AddOptions<BuilderServiceOptions>()
.Bind(configuration.GetSection(BuilderServiceOptions.SectionName));
services.AddOptions<FunctionExtractionOptions>()
.Bind(configuration.GetSection(FunctionExtractionOptions.SectionName));
// Core services
services.TryAddSingleton<IPatchDiffEngine, PatchDiffEngine>();
services.TryAddSingleton<IGuidProvider, GuidProvider>();
services.TryAddSingleton(TimeProvider.System);
// Builders will be added as they are implemented
// services.TryAddSingleton<IReproducibleBuilder, AlpineBuilder>();
@@ -56,6 +60,8 @@ public static class ServiceCollectionExtensions
services.Configure(configureOptions);
services.TryAddSingleton<IPatchDiffEngine, PatchDiffEngine>();
services.TryAddSingleton<IGuidProvider, GuidProvider>();
services.TryAddSingleton(TimeProvider.System);
return services;
}

View File

@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<Description>Reproducible distro builders and function-level fingerprinting for StellaOps BinaryIndex.</Description>
</PropertyGroup>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0112-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Builders. |
| AUDIT-0112-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Builders. |
| AUDIT-0112-A | TODO | Pending approval for changes. |
| AUDIT-0112-A | DONE | Applied audit fixes + tests. |

View File

@@ -35,6 +35,13 @@ public sealed class BinaryCacheOptions
/// </summary>
public TimeSpan FingerprintTtl { get; init; } = TimeSpan.FromMinutes(30);
/// <summary>
/// Optional fingerprint hash length for cache keys.
/// Set to 0 to use the full fingerprint hash.
/// Default: 0 (full hash).
/// </summary>
public int FingerprintHashLength { get; init; } = 0;
/// <summary>
/// Maximum TTL for any cache entry.
/// Default: 24 hours

View File

@@ -7,6 +7,7 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.BinaryIndex.Core.Services;
namespace StellaOps.BinaryIndex.Cache;
@@ -27,9 +28,12 @@ public static class BinaryCacheServiceExtensions
this IServiceCollection services,
IConfiguration configuration)
{
services.TryAddSingleton<IValidateOptions<BinaryCacheOptions>, BinaryCacheOptionsValidator>();
// Bind options
services.Configure<BinaryCacheOptions>(
configuration.GetSection("BinaryIndex:Cache"));
services.AddOptions<BinaryCacheOptions>()
.Bind(configuration.GetSection("BinaryIndex:Cache"))
.ValidateOnStart();
// Decorate the existing service with caching
services.Decorate<IBinaryVulnerabilityService, CachedBinaryVulnerabilityService>();
@@ -44,7 +48,10 @@ public static class BinaryCacheServiceExtensions
this IServiceCollection services,
Action<BinaryCacheOptions> configureOptions)
{
services.Configure(configureOptions);
services.TryAddSingleton<IValidateOptions<BinaryCacheOptions>, BinaryCacheOptionsValidator>();
services.AddOptions<BinaryCacheOptions>()
.Configure(configureOptions)
.ValidateOnStart();
services.Decorate<IBinaryVulnerabilityService, CachedBinaryVulnerabilityService>();
return services;

View File

@@ -0,0 +1,101 @@
using Microsoft.Extensions.Options;
namespace StellaOps.BinaryIndex.Cache;
public sealed class BinaryCacheOptionsValidator : IValidateOptions<BinaryCacheOptions>
{
public ValidateOptionsResult Validate(string? name, BinaryCacheOptions options)
{
if (options is null)
{
return ValidateOptionsResult.Fail("BinaryCacheOptions must be provided.");
}
var failures = new List<string>();
if (string.IsNullOrWhiteSpace(options.KeyPrefix))
{
failures.Add("BinaryCacheOptions.KeyPrefix must be set.");
}
if (options.MaxTtl <= TimeSpan.Zero)
{
failures.Add("BinaryCacheOptions.MaxTtl must be greater than zero.");
}
ValidateTtl(failures, options.IdentityTtl, options.MaxTtl, nameof(options.IdentityTtl));
ValidateTtl(failures, options.FixStatusTtl, options.MaxTtl, nameof(options.FixStatusTtl));
ValidateTtl(failures, options.FingerprintTtl, options.MaxTtl, nameof(options.FingerprintTtl));
if (options.TargetHitRate < 0 || options.TargetHitRate > 1)
{
failures.Add("BinaryCacheOptions.TargetHitRate must be between 0 and 1.");
}
if (options.FingerprintHashLength < 0)
{
failures.Add("BinaryCacheOptions.FingerprintHashLength must be zero or positive.");
}
return failures.Count > 0
? ValidateOptionsResult.Fail(failures)
: ValidateOptionsResult.Success;
}
private static void ValidateTtl(
ICollection<string> failures,
TimeSpan ttl,
TimeSpan maxTtl,
string name)
{
if (ttl <= TimeSpan.Zero)
{
failures.Add($"BinaryCacheOptions.{name} must be greater than zero.");
return;
}
if (maxTtl > TimeSpan.Zero && ttl > maxTtl)
{
failures.Add($"BinaryCacheOptions.{name} must be less than or equal to MaxTtl.");
}
}
}
public sealed class ResolutionCacheOptionsValidator : IValidateOptions<ResolutionCacheOptions>
{
public ValidateOptionsResult Validate(string? name, ResolutionCacheOptions options)
{
if (options is null)
{
return ValidateOptionsResult.Fail("ResolutionCacheOptions must be provided.");
}
var failures = new List<string>();
if (string.IsNullOrWhiteSpace(options.KeyPrefix))
{
failures.Add("ResolutionCacheOptions.KeyPrefix must be set.");
}
ValidateTtl(failures, options.FixedTtl, nameof(options.FixedTtl));
ValidateTtl(failures, options.VulnerableTtl, nameof(options.VulnerableTtl));
ValidateTtl(failures, options.UnknownTtl, nameof(options.UnknownTtl));
if (options.EarlyExpiryFactor < 0 || options.EarlyExpiryFactor > 1)
{
failures.Add("ResolutionCacheOptions.EarlyExpiryFactor must be between 0 and 1.");
}
return failures.Count > 0
? ValidateOptionsResult.Fail(failures)
: ValidateOptionsResult.Success;
}
private static void ValidateTtl(ICollection<string> failures, TimeSpan ttl, string name)
{
if (ttl <= TimeSpan.Zero)
{
failures.Add($"ResolutionCacheOptions.{name} must be greater than zero.");
}
}
}

View File

@@ -97,7 +97,7 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
}
var sw = Stopwatch.StartNew();
var db = await GetDatabaseAsync().ConfigureAwait(false);
var db = await GetDatabaseAsync(ct).ConfigureAwait(false);
// Build cache keys
var cacheKeys = identityList
@@ -106,9 +106,9 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
// Batch get from cache
var redisKeys = cacheKeys.Select(k => (RedisKey)k.Key).ToArray();
var cachedValues = await db.StringGetAsync(redisKeys).ConfigureAwait(false);
var cachedValues = await db.StringGetAsync(redisKeys).WaitAsync(ct).ConfigureAwait(false);
var results = new Dictionary<string, ImmutableArray<BinaryVulnMatch>>();
var results = new Dictionary<string, ImmutableArray<BinaryVulnMatch>>(StringComparer.Ordinal);
var misses = new List<BinaryIdentity>();
for (int i = 0; i < cacheKeys.Count; i++)
@@ -134,9 +134,10 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
misses.Add(identity);
}
var cacheHits = results.Count;
_logger.LogDebug(
"Batch lookup: {Hits} cache hits, {Misses} cache misses",
results.Count,
cacheHits,
misses.Count);
// Fetch misses from inner service
@@ -148,19 +149,33 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
var batch = db.CreateBatch();
var tasks = new List<Task>();
var missLookup = new Dictionary<string, BinaryIdentity>(StringComparer.Ordinal);
foreach (var miss in misses)
{
missLookup[miss.BinaryKey] = miss;
}
foreach (var (binaryKey, matches) in fetchedResults)
{
results[binaryKey] = matches;
var identity = misses.First(i => i.BinaryKey == binaryKey);
var cacheKey = BuildIdentityKey(identity, options);
var value = JsonSerializer.Serialize(matches, _jsonOptions);
if (missLookup.TryGetValue(binaryKey, out var identity))
{
var cacheKey = BuildIdentityKey(identity, options);
var value = JsonSerializer.Serialize(matches, _jsonOptions);
tasks.Add(batch.StringSetAsync(cacheKey, value, _options.IdentityTtl));
tasks.Add(batch.StringSetAsync(cacheKey, value, _options.IdentityTtl));
}
else
{
_logger.LogWarning(
"Lookup batch returned unexpected key {BinaryKey} not requested for cache fill",
binaryKey);
}
}
batch.Execute();
await Task.WhenAll(tasks).ConfigureAwait(false);
await Task.WhenAll(tasks).WaitAsync(ct).ConfigureAwait(false);
}
sw.Stop();
@@ -168,7 +183,7 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
"Batch lookup completed in {ElapsedMs}ms: {Total} total, {Hits} hits, {Misses} misses",
sw.Elapsed.TotalMilliseconds,
identityList.Count,
results.Count - misses.Count,
cacheHits,
misses.Count);
return results.ToImmutableDictionary();
@@ -220,7 +235,7 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
return ImmutableDictionary<string, FixStatusResult>.Empty;
}
var db = await GetDatabaseAsync().ConfigureAwait(false);
var db = await GetDatabaseAsync(ct).ConfigureAwait(false);
// Build cache keys
var cacheKeys = cveList
@@ -229,7 +244,7 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
// Batch get from cache
var redisKeys = cacheKeys.Select(k => (RedisKey)k.Key).ToArray();
var cachedValues = await db.StringGetAsync(redisKeys).ConfigureAwait(false);
var cachedValues = await db.StringGetAsync(redisKeys).WaitAsync(ct).ConfigureAwait(false);
var results = new Dictionary<string, FixStatusResult>();
var misses = new List<string>();
@@ -279,7 +294,7 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
}
batch.Execute();
await Task.WhenAll(tasks).ConfigureAwait(false);
await Task.WhenAll(tasks).WaitAsync(ct).ConfigureAwait(false);
}
return results.ToImmutableDictionary();
@@ -355,20 +370,56 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
{
try
{
var db = await GetDatabaseAsync().ConfigureAwait(false);
var server = _connectionMultiplexer.GetServer(_connectionMultiplexer.GetEndPoints().First());
var db = await GetDatabaseAsync(ct).ConfigureAwait(false);
var endpoints = _connectionMultiplexer.GetEndPoints();
if (endpoints.Length == 0)
{
_logger.LogWarning("No Redis endpoints available for cache invalidation");
return;
}
var pattern = $"{_options.KeyPrefix}fix:{distro}:{release}:*";
var keys = server.Keys(pattern: pattern).ToArray();
const int batchSize = 500;
long totalDeleted = 0;
if (keys.Length > 0)
foreach (var endpoint in endpoints)
{
ct.ThrowIfCancellationRequested();
var server = _connectionMultiplexer.GetServer(endpoint);
if (!server.IsConnected)
{
continue;
}
var buffer = new List<RedisKey>(batchSize);
foreach (var key in server.Keys(pattern: pattern, pageSize: batchSize))
{
ct.ThrowIfCancellationRequested();
buffer.Add(key);
if (buffer.Count >= batchSize)
{
totalDeleted += await db.KeyDeleteAsync(buffer.ToArray()).WaitAsync(ct).ConfigureAwait(false);
buffer.Clear();
}
}
if (buffer.Count > 0)
{
totalDeleted += await db.KeyDeleteAsync(buffer.ToArray()).WaitAsync(ct).ConfigureAwait(false);
}
}
if (totalDeleted > 0)
{
var deleted = await db.KeyDeleteAsync(keys).ConfigureAwait(false);
_logger.LogInformation(
"Invalidated {Count} cache entries for {Distro}:{Release}",
deleted, distro, release);
totalDeleted, distro, release);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error invalidating cache for {Distro}:{Release}", distro, release);
@@ -390,15 +441,20 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
{
var hash = Convert.ToHexString(fingerprint).ToLowerInvariant();
var algo = options?.Algorithm ?? "combined";
return $"{_options.KeyPrefix}fp:{algo}:{hash[..Math.Min(32, hash.Length)]}";
if (_options.FingerprintHashLength > 0 && _options.FingerprintHashLength < hash.Length)
{
hash = hash[.._options.FingerprintHashLength];
}
return $"{_options.KeyPrefix}fp:{algo}:{hash}";
}
private async Task<T?> GetFromCacheAsync<T>(string key, CancellationToken ct)
{
try
{
var db = await GetDatabaseAsync().ConfigureAwait(false);
var value = await db.StringGetAsync(key).ConfigureAwait(false);
var db = await GetDatabaseAsync(ct).ConfigureAwait(false);
var value = await db.StringGetAsync(key).WaitAsync(ct).ConfigureAwait(false);
if (value.IsNullOrEmpty)
{
@@ -407,6 +463,10 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
return JsonSerializer.Deserialize<T>((string)value!, _jsonOptions);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error getting cache entry for key {Key}", key);
@@ -418,10 +478,14 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
{
try
{
var db = await GetDatabaseAsync().ConfigureAwait(false);
var db = await GetDatabaseAsync(ct).ConfigureAwait(false);
var serialized = JsonSerializer.Serialize(value, _jsonOptions);
await db.StringSetAsync(key, serialized, ttl).ConfigureAwait(false);
await db.StringSetAsync(key, serialized, ttl).WaitAsync(ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
@@ -429,12 +493,12 @@ public sealed class CachedBinaryVulnerabilityService : IBinaryVulnerabilityServi
}
}
private async Task<IDatabase> GetDatabaseAsync()
private async Task<IDatabase> GetDatabaseAsync(CancellationToken ct)
{
if (_database is not null)
return _database;
await _connectionLock.WaitAsync().ConfigureAwait(false);
await _connectionLock.WaitAsync(ct).ConfigureAwait(false);
try
{
_database ??= _connectionMultiplexer.GetDatabase();

View File

@@ -0,0 +1,23 @@
namespace StellaOps.BinaryIndex.Cache;
public interface IRandomSource
{
double NextDouble();
}
public sealed class SystemRandomSource : IRandomSource
{
private readonly Random _random;
public SystemRandomSource()
: this(Random.Shared)
{
}
public SystemRandomSource(Random random)
{
_random = random ?? throw new ArgumentNullException(nameof(random));
}
public double NextDouble() => _random.NextDouble();
}

View File

@@ -107,15 +107,18 @@ public sealed class ResolutionCacheService : IResolutionCacheService
private readonly ResolutionCacheOptions _options;
private readonly ILogger<ResolutionCacheService> _logger;
private readonly JsonSerializerOptions _jsonOptions;
private readonly IRandomSource _random;
public ResolutionCacheService(
IConnectionMultiplexer redis,
IOptions<ResolutionCacheOptions> options,
ILogger<ResolutionCacheService> logger)
ILogger<ResolutionCacheService> logger,
IRandomSource random)
{
_redis = redis ?? throw new ArgumentNullException(nameof(redis));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_random = random ?? throw new ArgumentNullException(nameof(random));
_jsonOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
@@ -129,7 +132,7 @@ public sealed class ResolutionCacheService : IResolutionCacheService
try
{
var db = _redis.GetDatabase();
var value = await db.StringGetAsync(cacheKey);
var value = await db.StringGetAsync(cacheKey).WaitAsync(ct).ConfigureAwait(false);
if (value.IsNullOrEmpty)
{
@@ -142,7 +145,7 @@ public sealed class ResolutionCacheService : IResolutionCacheService
// Check for probabilistic early expiry
if (_options.EnableEarlyExpiry && cached is not null)
{
var ttl = await db.KeyTimeToLiveAsync(cacheKey);
var ttl = await db.KeyTimeToLiveAsync(cacheKey).WaitAsync(ct).ConfigureAwait(false);
if (ShouldExpireEarly(ttl))
{
_logger.LogDebug("Early expiry triggered for key {CacheKey}", cacheKey);
@@ -153,6 +156,10 @@ public sealed class ResolutionCacheService : IResolutionCacheService
_logger.LogDebug("Cache hit for key {CacheKey}", cacheKey);
return cached;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to get cache entry for key {CacheKey}", cacheKey);
@@ -168,9 +175,13 @@ public sealed class ResolutionCacheService : IResolutionCacheService
var db = _redis.GetDatabase();
var value = JsonSerializer.Serialize(result, _jsonOptions);
await db.StringSetAsync(cacheKey, value, ttl);
await db.StringSetAsync(cacheKey, value, ttl).WaitAsync(ct).ConfigureAwait(false);
_logger.LogDebug("Cached resolution for key {CacheKey} with TTL {Ttl}", cacheKey, ttl);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to cache resolution for key {CacheKey}", cacheKey);
@@ -182,17 +193,55 @@ public sealed class ResolutionCacheService : IResolutionCacheService
{
try
{
var server = _redis.GetServer(_redis.GetEndPoints().First());
var db = _redis.GetDatabase();
var keys = server.Keys(pattern: pattern).ToArray();
if (keys.Length > 0)
var endpoints = _redis.GetEndPoints();
if (endpoints.Length == 0)
{
await db.KeyDeleteAsync(keys);
_logger.LogInformation("Invalidated {Count} cache entries matching pattern {Pattern}",
keys.Length, pattern);
_logger.LogWarning("No Redis endpoints available for pattern invalidation");
return;
}
const int batchSize = 500;
long totalDeleted = 0;
foreach (var endpoint in endpoints)
{
ct.ThrowIfCancellationRequested();
var server = _redis.GetServer(endpoint);
if (!server.IsConnected)
{
continue;
}
var buffer = new List<RedisKey>(batchSize);
foreach (var key in server.Keys(pattern: pattern, pageSize: batchSize))
{
ct.ThrowIfCancellationRequested();
buffer.Add(key);
if (buffer.Count >= batchSize)
{
totalDeleted += await db.KeyDeleteAsync(buffer.ToArray()).WaitAsync(ct).ConfigureAwait(false);
buffer.Clear();
}
}
if (buffer.Count > 0)
{
totalDeleted += await db.KeyDeleteAsync(buffer.ToArray()).WaitAsync(ct).ConfigureAwait(false);
}
}
if (totalDeleted > 0)
{
_logger.LogInformation(
"Invalidated {Count} cache entries matching pattern {Pattern}",
totalDeleted,
pattern);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
@@ -271,7 +320,7 @@ public sealed class ResolutionCacheService : IResolutionCacheService
return true;
// Probabilistic early expiry using exponential decay
var random = Random.Shared.NextDouble();
var random = _random.NextDouble();
var threshold = _options.EarlyExpiryFactor * Math.Exp(-remainingTtl.Value.TotalSeconds / 3600);
return random < threshold;

View File

@@ -6,7 +6,7 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<RootNamespace>StellaOps.BinaryIndex.Cache</RootNamespace>
<AssemblyName>StellaOps.BinaryIndex.Cache</AssemblyName>
<Description>Valkey/Redis cache layer for BinaryIndex vulnerability lookups</Description>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0114-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Cache. |
| AUDIT-0114-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Cache. |
| AUDIT-0114-A | TODO | Pending approval for changes. |
| AUDIT-0114-A | DONE | Applied cache fixes + tests. |

View File

@@ -5,7 +5,7 @@ namespace StellaOps.BinaryIndex.Contracts.Resolution;
/// <summary>
/// Request to resolve vulnerability status for a binary.
/// </summary>
public sealed record VulnResolutionRequest
public sealed record VulnResolutionRequest : IValidatableObject
{
/// <summary>
/// Package URL (PURL) or CPE identifier.
@@ -47,6 +47,25 @@ public sealed record VulnResolutionRequest
/// Distro hint for fix status lookup (e.g., "debian:bookworm").
/// </summary>
public string? DistroRelease { get; init; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (string.IsNullOrWhiteSpace(BuildId)
&& string.IsNullOrWhiteSpace(Fingerprint)
&& string.IsNullOrWhiteSpace(Hashes?.FileSha256)
&& string.IsNullOrWhiteSpace(Hashes?.TextSha256)
&& string.IsNullOrWhiteSpace(Hashes?.Blake3))
{
yield return new ValidationResult(
"At least one identifier is required (BuildId, Fingerprint, or Hashes).",
new[]
{
nameof(BuildId),
nameof(Fingerprint),
nameof(Hashes)
});
}
}
}
/// <summary>
@@ -67,7 +86,7 @@ public sealed record ResolutionHashes
/// <summary>
/// Response from vulnerability resolution.
/// </summary>
public sealed record VulnResolutionResponse
public sealed record VulnResolutionResponse : IValidatableObject
{
/// <summary>Package identifier from request.</summary>
public required string Package { get; init; }
@@ -92,6 +111,16 @@ public sealed record VulnResolutionResponse
/// <summary>CVE ID if a specific CVE was queried.</summary>
public string? CveId { get; init; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (ResolvedAt == default)
{
yield return new ValidationResult(
"ResolvedAt must be set to a valid timestamp.",
new[] { nameof(ResolvedAt) });
}
}
}
/// <summary>
@@ -142,17 +171,50 @@ public sealed record ResolutionEvidence
public string? FixMethod { get; init; }
}
public static class ResolutionMatchTypes
{
public const string BuildId = "build_id";
public const string Fingerprint = "fingerprint";
public const string HashExact = "hash_exact";
public const string Package = "package";
public const string RangeMatch = "range_match";
public const string DeltaSignature = "delta_signature";
public const string FixStatus = "fix_status";
public const string Unknown = "unknown";
}
public static class ResolutionFixMethods
{
public const string SecurityFeed = "security_feed";
public const string Changelog = "changelog";
public const string PatchHeader = "patch_header";
public const string DeltaSignature = "delta_signature";
public const string UpstreamPatchMatch = "upstream_patch_match";
public const string Unknown = "unknown";
}
/// <summary>
/// Batch request for resolving multiple vulnerabilities.
/// </summary>
public sealed record BatchVulnResolutionRequest
public sealed record BatchVulnResolutionRequest : IValidatableObject
{
/// <summary>List of resolution requests.</summary>
[Required]
[MinLength(1)]
public required IReadOnlyList<VulnResolutionRequest> Items { get; init; }
/// <summary>Resolution options.</summary>
public BatchResolutionOptions? Options { get; init; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (Items is null || Items.Count == 0)
{
yield return new ValidationResult(
"Items must contain at least one request.",
new[] { nameof(Items) });
}
}
}
/// <summary>

View File

@@ -2,6 +2,7 @@
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0115-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Contracts. |
| AUDIT-0115-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Contracts. |
| AUDIT-0115-A | TODO | Pending approval for changes. |
| AUDIT-0115-A | DONE | Applied contract fixes + tests. |

View File

@@ -4,6 +4,8 @@ using Microsoft.Extensions.Options;
using StellaOps.BinaryIndex.Contracts.Resolution;
using StellaOps.BinaryIndex.Core.Models;
using StellaOps.BinaryIndex.Core.Services;
using ResolutionFixMethods = StellaOps.BinaryIndex.Contracts.Resolution.ResolutionFixMethods;
using ResolutionMatchTypes = StellaOps.BinaryIndex.Contracts.Resolution.ResolutionMatchTypes;
namespace StellaOps.BinaryIndex.Core.Resolution;
@@ -76,15 +78,18 @@ public sealed class ResolutionService : IResolutionService
private readonly IBinaryVulnerabilityService _vulnerabilityService;
private readonly ResolutionServiceOptions _options;
private readonly ILogger<ResolutionService> _logger;
private readonly TimeProvider _timeProvider;
public ResolutionService(
IBinaryVulnerabilityService vulnerabilityService,
IOptions<ResolutionServiceOptions> options,
ILogger<ResolutionService> logger)
ILogger<ResolutionService> logger,
TimeProvider timeProvider)
{
_vulnerabilityService = vulnerabilityService ?? throw new ArgumentNullException(nameof(vulnerabilityService));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
/// <inheritdoc />
@@ -95,15 +100,13 @@ public sealed class ResolutionService : IResolutionService
{
ArgumentNullException.ThrowIfNull(request);
var sw = Stopwatch.StartNew();
var effectiveOptions = options ?? new ResolutionOptions();
var resolvedAt = _timeProvider.GetUtcNow();
_logger.LogDebug("Resolving vulnerability for package {Package}", request.Package);
// Build binary identity from request
var identity = BuildBinaryIdentity(request);
EnsureIdentifiersPresent(request);
// Perform lookup
var lookupOptions = new LookupOptions
{
DistroHint = ExtractDistro(request.DistroRelease),
@@ -114,11 +117,18 @@ public sealed class ResolutionService : IResolutionService
// Check if specific CVE requested
if (!string.IsNullOrEmpty(request.CveId))
{
return await ResolveSingleCveAsync(request, identity, lookupOptions, effectiveOptions, sw, ct);
return await ResolveSingleCveAsync(request, resolvedAt, ct);
}
if (HasFingerprintOnly(request))
{
return await ResolveByFingerprintAsync(request, lookupOptions, resolvedAt, ct);
}
var identity = BuildBinaryIdentity(request, resolvedAt);
// Full lookup - all CVEs
return await ResolveAllCvesAsync(request, identity, lookupOptions, effectiveOptions, sw, ct);
return await ResolveAllCvesAsync(request, identity, lookupOptions, resolvedAt, ct);
}
/// <inheritdoc />
@@ -174,7 +184,7 @@ public sealed class ResolutionService : IResolutionService
{
Package = item.Package,
Status = ResolutionStatus.Unknown,
ResolvedAt = DateTimeOffset.UtcNow,
ResolvedAt = _timeProvider.GetUtcNow(),
FromCache = false
});
}
@@ -191,10 +201,7 @@ public sealed class ResolutionService : IResolutionService
private async Task<VulnResolutionResponse> ResolveSingleCveAsync(
VulnResolutionRequest request,
BinaryIdentity identity,
LookupOptions lookupOptions,
ResolutionOptions options,
Stopwatch sw,
DateTimeOffset resolvedAt,
CancellationToken ct)
{
// Check fix status for specific CVE
@@ -214,7 +221,7 @@ public sealed class ResolutionService : IResolutionService
FixedVersion = fixStatus?.FixedVersion,
Evidence = evidence,
CveId = request.CveId,
ResolvedAt = DateTimeOffset.UtcNow,
ResolvedAt = resolvedAt,
FromCache = false
};
}
@@ -223,8 +230,7 @@ public sealed class ResolutionService : IResolutionService
VulnResolutionRequest request,
BinaryIdentity identity,
LookupOptions lookupOptions,
ResolutionOptions options,
Stopwatch sw,
DateTimeOffset resolvedAt,
CancellationToken ct)
{
// Perform full binary lookup
@@ -238,7 +244,7 @@ public sealed class ResolutionService : IResolutionService
{
Package = request.Package,
Status = ResolutionStatus.NotAffected,
ResolvedAt = DateTimeOffset.UtcNow,
ResolvedAt = resolvedAt,
FromCache = false
};
}
@@ -248,7 +254,7 @@ public sealed class ResolutionService : IResolutionService
var evidence = new ResolutionEvidence
{
MatchType = primaryMatch.Method.ToString().ToLowerInvariant(),
MatchType = MapMatchType(primaryMatch.Method),
Confidence = primaryMatch.Confidence,
MatchedFingerprintIds = matches.Select(m => m.CveId).ToList()
};
@@ -267,26 +273,82 @@ public sealed class ResolutionService : IResolutionService
Package = request.Package,
Status = status,
Evidence = evidence,
ResolvedAt = DateTimeOffset.UtcNow,
ResolvedAt = resolvedAt,
FromCache = false
};
}
private static BinaryIdentity BuildBinaryIdentity(VulnResolutionRequest request)
private async Task<VulnResolutionResponse> ResolveByFingerprintAsync(
VulnResolutionRequest request,
LookupOptions lookupOptions,
DateTimeOffset resolvedAt,
CancellationToken ct)
{
var binaryKey = request.BuildId
?? request.Hashes?.FileSha256
?? request.Package;
var fingerprintBytes = Convert.FromBase64String(request.Fingerprint!);
var matches = await _vulnerabilityService.LookupByFingerprintAsync(
fingerprintBytes,
new FingerprintLookupOptions
{
Algorithm = request.FingerprintAlgorithm,
DistroHint = lookupOptions.DistroHint,
ReleaseHint = lookupOptions.ReleaseHint,
CheckFixIndex = true
},
ct);
if (matches.IsEmpty)
{
return new VulnResolutionResponse
{
Package = request.Package,
Status = ResolutionStatus.NotAffected,
ResolvedAt = resolvedAt,
FromCache = false
};
}
var primaryMatch = matches.OrderByDescending(m => m.Confidence).First();
var evidence = new ResolutionEvidence
{
MatchType = ResolutionMatchTypes.Fingerprint,
Confidence = primaryMatch.Confidence,
MatchedFingerprintIds = matches.Select(m => m.CveId).ToList()
};
var status = primaryMatch.Confidence >= _options.MinConfidenceThreshold
? ResolutionStatus.Fixed
: ResolutionStatus.Unknown;
return new VulnResolutionResponse
{
Package = request.Package,
Status = status,
Evidence = evidence,
ResolvedAt = resolvedAt,
FromCache = false
};
}
private BinaryIdentity BuildBinaryIdentity(VulnResolutionRequest request, DateTimeOffset resolvedAt)
{
var binaryKey = request.BuildId
?? request.Hashes?.FileSha256
?? request.Hashes?.TextSha256
?? request.Hashes?.Blake3
?? throw new ArgumentException("Binary identifier is required.");
return new BinaryIdentity
{
BinaryKey = binaryKey,
BuildId = request.BuildId,
FileSha256 = request.Hashes?.FileSha256 ?? "sha256:unknown",
FileSha256 = request.Hashes?.FileSha256 ?? string.Empty,
TextSha256 = request.Hashes?.TextSha256,
Blake3Hash = request.Hashes?.Blake3,
Format = BinaryFormat.Elf,
Architecture = "unknown"
Architecture = string.Empty,
CreatedAt = resolvedAt,
UpdatedAt = resolvedAt
};
}
@@ -309,9 +371,9 @@ public sealed class ResolutionService : IResolutionService
var evidence = new ResolutionEvidence
{
MatchType = "fix_status",
MatchType = ResolutionMatchTypes.FixStatus,
Confidence = fixStatus.Confidence,
FixMethod = fixStatus.Method.ToString().ToLowerInvariant()
FixMethod = MapFixMethod(fixStatus.Method)
};
return (status, evidence);
@@ -357,4 +419,45 @@ public sealed class ResolutionService : IResolutionService
return null;
}
private static string MapMatchType(MatchMethod method) => method switch
{
MatchMethod.BuildIdCatalog => ResolutionMatchTypes.BuildId,
MatchMethod.FingerprintMatch => ResolutionMatchTypes.Fingerprint,
MatchMethod.RangeMatch => ResolutionMatchTypes.RangeMatch,
MatchMethod.DeltaSignature => ResolutionMatchTypes.DeltaSignature,
_ => ResolutionMatchTypes.Unknown
};
private static string MapFixMethod(FixMethod method) => method switch
{
FixMethod.SecurityFeed => ResolutionFixMethods.SecurityFeed,
FixMethod.Changelog => ResolutionFixMethods.Changelog,
FixMethod.PatchHeader => ResolutionFixMethods.PatchHeader,
FixMethod.UpstreamPatchMatch => ResolutionFixMethods.UpstreamPatchMatch,
_ => ResolutionFixMethods.Unknown
};
private static void EnsureIdentifiersPresent(VulnResolutionRequest request)
{
if (!HasBuildIdOrHashes(request) && string.IsNullOrWhiteSpace(request.Fingerprint))
{
throw new ArgumentException(
"At least one identifier is required (BuildId, Fingerprint, or Hashes).",
nameof(request));
}
}
private static bool HasFingerprintOnly(VulnResolutionRequest request)
{
return !HasBuildIdOrHashes(request) && !string.IsNullOrWhiteSpace(request.Fingerprint);
}
private static bool HasBuildIdOrHashes(VulnResolutionRequest request)
{
return !string.IsNullOrWhiteSpace(request.BuildId)
|| !string.IsNullOrWhiteSpace(request.Hashes?.FileSha256)
|| !string.IsNullOrWhiteSpace(request.Hashes?.TextSha256)
|| !string.IsNullOrWhiteSpace(request.Hashes?.Blake3);
}
}

View File

@@ -57,6 +57,8 @@ public sealed class BinaryIdentityService
foreach (var (stream, path) in binaries)
{
ct.ThrowIfCancellationRequested();
try
{
var identity = await IndexBinaryAsync(stream, path, ct);

View File

@@ -10,9 +10,18 @@ namespace StellaOps.BinaryIndex.Core.Services;
public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
{
private static readonly byte[] ElfMagic = [0x7F, 0x45, 0x4C, 0x46]; // \x7fELF
private readonly TimeProvider _timeProvider;
public ElfFeatureExtractor(TimeProvider? timeProvider = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
public bool CanExtract(Stream stream)
{
if (stream is null || !stream.CanSeek || !stream.CanRead)
return false;
if (stream.Length < 4)
return false;
@@ -21,7 +30,7 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
{
Span<byte> magic = stackalloc byte[4];
stream.Position = 0;
var read = stream.Read(magic);
var read = stream.ReadAtLeast(magic, magic.Length, throwOnEndOfStream: false);
return read == 4 && magic.SequenceEqual(ElfMagic);
}
finally
@@ -32,6 +41,7 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
public async Task<BinaryIdentity> ExtractIdentityAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "ELF identity extraction");
var metadata = await ExtractMetadataAsync(stream, ct);
// Compute full file SHA-256
@@ -43,6 +53,7 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
? $"{metadata.BuildId}:{fileSha256}"
: fileSha256;
var now = _timeProvider.GetUtcNow();
return new BinaryIdentity
{
BinaryKey = binaryKey,
@@ -53,15 +64,18 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
Architecture = metadata.Architecture,
OsAbi = metadata.OsAbi,
Type = metadata.Type,
IsStripped = metadata.IsStripped
IsStripped = metadata.IsStripped,
CreatedAt = now,
UpdatedAt = now
};
}
public Task<BinaryMetadata> ExtractMetadataAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "ELF metadata extraction");
stream.Position = 0;
Span<byte> header = stackalloc byte[64];
var read = stream.Read(header);
var read = stream.ReadAtLeast(header, header.Length, throwOnEndOfStream: false);
if (read < 20)
throw new InvalidDataException("Stream too short for ELF header");
@@ -76,7 +90,7 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
var architecture = MapArchitecture(eMachine);
var osAbiStr = MapOsAbi(osAbi);
var type = MapBinaryType(eType);
var buildId = ExtractBuildId(stream);
var buildId = ExtractBuildId(stream, ct);
return Task.FromResult(new BinaryMetadata
{
@@ -90,28 +104,62 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
});
}
private static string? ExtractBuildId(Stream stream)
private static string? ExtractBuildId(Stream stream, CancellationToken ct)
{
StreamGuard.EnsureSeekable(stream, "ELF build-id scan");
// Simplified: scan for .note.gnu.build-id section
// In production, parse program headers properly
stream.Position = 0;
var buffer = new byte[stream.Length];
stream.Read(buffer);
// Look for NT_GNU_BUILD_ID note (type 3)
var buildIdPattern = Encoding.ASCII.GetBytes(".note.gnu.build-id");
for (var i = 0; i < buffer.Length - buildIdPattern.Length; i++)
var buffer = new byte[64 * 1024];
var carry = new byte[buildIdPattern.Length - 1];
var carryCount = 0;
long offset = 0;
while (true)
{
if (buffer.AsSpan(i, buildIdPattern.Length).SequenceEqual(buildIdPattern))
ct.ThrowIfCancellationRequested();
var read = stream.Read(buffer, 0, buffer.Length);
if (read == 0)
break;
var combined = new byte[carryCount + read];
if (carryCount > 0)
{
// Found build-id section, extract it
// This is simplified; real implementation would parse note structure
var noteStart = i + buildIdPattern.Length + 16;
if (noteStart + 20 < buffer.Length)
Buffer.BlockCopy(carry, 0, combined, 0, carryCount);
}
Buffer.BlockCopy(buffer, 0, combined, carryCount, read);
for (var i = 0; i <= combined.Length - buildIdPattern.Length; i++)
{
if (combined.AsSpan(i, buildIdPattern.Length).SequenceEqual(buildIdPattern))
{
return Convert.ToHexString(buffer.AsSpan(noteStart, 20)).ToLowerInvariant();
var matchOffset = offset - carryCount + i;
var noteStart = matchOffset + buildIdPattern.Length + 16;
if (noteStart + 20 <= stream.Length)
{
stream.Position = noteStart;
Span<byte> buildId = stackalloc byte[20];
var buildIdRead = stream.ReadAtLeast(buildId, buildId.Length, throwOnEndOfStream: false);
if (buildIdRead == 20)
{
return Convert.ToHexString(buildId).ToLowerInvariant();
}
}
return null;
}
}
carryCount = Math.Min(carry.Length, combined.Length);
if (carryCount > 0)
{
Buffer.BlockCopy(combined, combined.Length - carryCount, carry, 0, carryCount);
}
offset += read;
}
return null;
@@ -119,11 +167,12 @@ public sealed class ElfFeatureExtractor : IBinaryFeatureExtractor
private static bool HasSymbolTable(Stream stream)
{
StreamGuard.EnsureSeekable(stream, "ELF symbol table scan");
// Simplified: check for .symtab section
stream.Position = 0;
var buffer = new byte[Math.Min(8192, stream.Length)];
stream.Read(buffer);
return Encoding.ASCII.GetString(buffer).Contains(".symtab");
var read = stream.Read(buffer, 0, buffer.Length);
return Encoding.ASCII.GetString(buffer, 0, read).Contains(".symtab");
}
private static string MapArchitecture(ushort eMachine) => eMachine switch

View File

@@ -200,6 +200,9 @@ public sealed record MatchEvidence
/// <summary>Package PURL from the delta signature.</summary>
public string? SignaturePackagePurl { get; init; }
/// <summary>Fingerprint algorithm used for matching when available.</summary>
public string? FingerprintAlgorithm { get; init; }
}
/// <summary>

View File

@@ -6,6 +6,8 @@
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.BinaryIndex.Core.Models;
namespace StellaOps.BinaryIndex.Core.Services;
@@ -27,9 +29,22 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
// Load command types
private const uint LC_UUID = 0x1B; // UUID load command
private const uint LC_ID_DYLIB = 0x0D; // Dylib identification
private readonly TimeProvider _timeProvider;
private readonly ILogger<MachoFeatureExtractor> _logger;
public MachoFeatureExtractor(
TimeProvider? timeProvider = null,
ILogger<MachoFeatureExtractor>? logger = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<MachoFeatureExtractor>.Instance;
}
public bool CanExtract(Stream stream)
{
if (stream is null || !stream.CanSeek || !stream.CanRead)
return false;
if (stream.Length < 4)
return false;
@@ -38,7 +53,7 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
{
Span<byte> magic = stackalloc byte[4];
stream.Position = 0;
var read = stream.Read(magic);
var read = stream.ReadAtLeast(magic, magic.Length, throwOnEndOfStream: false);
if (read < 4)
return false;
@@ -53,6 +68,7 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
public async Task<BinaryIdentity> ExtractIdentityAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "Mach-O identity extraction");
var metadata = await ExtractMetadataAsync(stream, ct);
// Compute full file SHA-256
@@ -64,6 +80,7 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
? $"macho-uuid:{metadata.BuildId}:{fileSha256}"
: fileSha256;
var now = _timeProvider.GetUtcNow();
return new BinaryIdentity
{
BinaryKey = binaryKey,
@@ -73,16 +90,19 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
Format = metadata.Format,
Architecture = metadata.Architecture,
Type = metadata.Type,
IsStripped = metadata.IsStripped
IsStripped = metadata.IsStripped,
CreatedAt = now,
UpdatedAt = now
};
}
public Task<BinaryMetadata> ExtractMetadataAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "Mach-O metadata extraction");
stream.Position = 0;
Span<byte> header = stackalloc byte[32];
var read = stream.Read(header);
var read = stream.ReadAtLeast(header, header.Length, throwOnEndOfStream: false);
if (read < 4)
throw new InvalidDataException("Stream too short for Mach-O header");
@@ -97,7 +117,15 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
var needsSwap = magicValue is MH_CIGAM or MH_CIGAM_64;
var is64Bit = magicValue is MH_MAGIC_64 or MH_CIGAM_64;
return Task.FromResult(ParseMachHeader(stream, header, is64Bit, needsSwap));
try
{
return Task.FromResult(ParseMachHeader(stream, header, is64Bit, needsSwap));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse Mach-O header.");
throw;
}
}
private static BinaryMetadata ParseMachHeader(Stream stream, ReadOnlySpan<byte> header, bool is64Bit, bool needsSwap)
@@ -127,7 +155,11 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
stream.Position = headerSize;
var cmdBuffer = new byte[sizeOfCmds];
stream.Read(cmdBuffer);
var cmdRead = stream.Read(cmdBuffer, 0, cmdBuffer.Length);
if (cmdRead < cmdBuffer.Length)
{
throw new InvalidDataException("Stream too short for Mach-O load commands");
}
var offset = 0;
for (var i = 0; i < ncmds && offset < cmdBuffer.Length - 8; i++)
@@ -170,7 +202,9 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
// 4-8: nfat_arch
stream.Position = 4;
Span<byte> nArchBytes = stackalloc byte[4];
stream.Read(nArchBytes);
var nArchRead = stream.ReadAtLeast(nArchBytes, nArchBytes.Length, throwOnEndOfStream: false);
if (nArchRead < nArchBytes.Length)
throw new InvalidDataException("Stream too short for Mach-O fat header");
var nArch = ReadUInt32(nArchBytes, needsSwap);
if (nArch == 0)
@@ -179,7 +213,9 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
// Read first fat_arch entry to get offset to first slice
// fat_arch: cputype(4), cpusubtype(4), offset(4), size(4), align(4)
Span<byte> fatArch = stackalloc byte[20];
stream.Read(fatArch);
var fatArchRead = stream.ReadAtLeast(fatArch, fatArch.Length, throwOnEndOfStream: false);
if (fatArchRead < fatArch.Length)
throw new InvalidDataException("Stream too short for Mach-O fat arch");
var sliceOffset = ReadUInt32(fatArch[8..12], needsSwap);
var sliceSize = ReadUInt32(fatArch[12..16], needsSwap);
@@ -187,7 +223,9 @@ public sealed class MachoFeatureExtractor : IBinaryFeatureExtractor
// Read the Mach-O header from the first slice
stream.Position = sliceOffset;
Span<byte> sliceHeader = stackalloc byte[32];
stream.Read(sliceHeader);
var sliceHeaderRead = stream.ReadAtLeast(sliceHeader, sliceHeader.Length, throwOnEndOfStream: false);
if (sliceHeaderRead < sliceHeader.Length)
throw new InvalidDataException("Stream too short for Mach-O slice header");
var sliceMagic = BitConverter.ToUInt32(sliceHeader[..4]);
var sliceNeedsSwap = sliceMagic is MH_CIGAM or MH_CIGAM_64;

View File

@@ -6,7 +6,8 @@
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.BinaryIndex.Core.Models;
namespace StellaOps.BinaryIndex.Core.Services;
@@ -22,9 +23,22 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
// PE signature: PE\0\0
private static readonly byte[] PeSignature = [0x50, 0x45, 0x00, 0x00];
private readonly TimeProvider _timeProvider;
private readonly ILogger<PeFeatureExtractor> _logger;
public PeFeatureExtractor(
TimeProvider? timeProvider = null,
ILogger<PeFeatureExtractor>? logger = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<PeFeatureExtractor>.Instance;
}
public bool CanExtract(Stream stream)
{
if (stream is null || !stream.CanSeek || !stream.CanRead)
return false;
if (stream.Length < 64) // Minimum DOS header size
return false;
@@ -33,7 +47,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
{
Span<byte> magic = stackalloc byte[2];
stream.Position = 0;
var read = stream.Read(magic);
var read = stream.ReadAtLeast(magic, magic.Length, throwOnEndOfStream: false);
return read == 2 && magic.SequenceEqual(DosMagic);
}
finally
@@ -44,6 +58,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
public async Task<BinaryIdentity> ExtractIdentityAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "PE identity extraction");
var metadata = await ExtractMetadataAsync(stream, ct);
// Compute full file SHA-256
@@ -55,6 +70,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
? $"pe-cv:{metadata.BuildId}:{fileSha256}"
: fileSha256;
var now = _timeProvider.GetUtcNow();
return new BinaryIdentity
{
BinaryKey = binaryKey,
@@ -64,17 +80,20 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
Format = metadata.Format,
Architecture = metadata.Architecture,
Type = metadata.Type,
IsStripped = metadata.IsStripped
IsStripped = metadata.IsStripped,
CreatedAt = now,
UpdatedAt = now
};
}
public Task<BinaryMetadata> ExtractMetadataAsync(Stream stream, CancellationToken ct = default)
{
StreamGuard.EnsureSeekable(stream, "PE metadata extraction");
stream.Position = 0;
// Read DOS header to get PE header offset
Span<byte> dosHeader = stackalloc byte[64];
var read = stream.Read(dosHeader);
var read = stream.ReadAtLeast(dosHeader, dosHeader.Length, throwOnEndOfStream: false);
if (read < 64)
throw new InvalidDataException("Stream too short for DOS header");
@@ -86,7 +105,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
// Read PE signature and COFF header
stream.Position = peOffset;
Span<byte> peHeader = stackalloc byte[24];
read = stream.Read(peHeader);
read = stream.ReadAtLeast(peHeader, peHeader.Length, throwOnEndOfStream: false);
if (read < 24)
throw new InvalidDataException("Stream too short for PE header");
@@ -102,7 +121,9 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
// Read optional header to determine PE32 vs PE32+
Span<byte> optionalMagic = stackalloc byte[2];
stream.Read(optionalMagic);
var optionalRead = stream.ReadAtLeast(optionalMagic, optionalMagic.Length, throwOnEndOfStream: false);
if (optionalRead < optionalMagic.Length)
throw new InvalidDataException("Stream too short for optional header magic");
var isPe32Plus = BitConverter.ToUInt16(optionalMagic) == 0x20B;
var architecture = MapMachine(machine);
@@ -125,14 +146,16 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
/// <summary>
/// Extract CodeView GUID from PE debug directory.
/// </summary>
private static string? ExtractCodeViewGuid(Stream stream, int peOffset, bool isPe32Plus)
private string? ExtractCodeViewGuid(Stream stream, int peOffset, bool isPe32Plus)
{
try
{
// Calculate optional header size offset
stream.Position = peOffset + 20; // After COFF header
Span<byte> sizeOfOptionalHeader = stackalloc byte[2];
stream.Read(sizeOfOptionalHeader);
var optionalHeaderRead = stream.ReadAtLeast(sizeOfOptionalHeader, sizeOfOptionalHeader.Length, throwOnEndOfStream: false);
if (optionalHeaderRead < sizeOfOptionalHeader.Length)
return null;
var optionalHeaderSize = BitConverter.ToUInt16(sizeOfOptionalHeader);
if (optionalHeaderSize < 128)
@@ -148,7 +171,9 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
stream.Position = debugDirectoryRva;
Span<byte> debugDir = stackalloc byte[8];
stream.Read(debugDir);
var debugDirRead = stream.ReadAtLeast(debugDir, debugDir.Length, throwOnEndOfStream: false);
if (debugDirRead < debugDir.Length)
return null;
var debugRva = BitConverter.ToUInt32(debugDir[..4]);
var debugSize = BitConverter.ToUInt32(debugDir[4..8]);
@@ -163,7 +188,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
stream.Position = debugRva;
Span<byte> debugEntry = stackalloc byte[28];
var read = stream.Read(debugEntry);
var read = stream.ReadAtLeast(debugEntry, debugEntry.Length, throwOnEndOfStream: false);
if (read < 28)
return null;
@@ -178,7 +203,7 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
// Read CodeView header
stream.Position = pointerToRawData;
Span<byte> cvHeader = stackalloc byte[24];
read = stream.Read(cvHeader);
read = stream.ReadAtLeast(cvHeader, cvHeader.Length, throwOnEndOfStream: false);
if (read < 24)
return null;
@@ -196,8 +221,9 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
return null;
}
catch
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to parse CodeView GUID from PE image.");
return null;
}
}
@@ -214,7 +240,9 @@ public sealed class PeFeatureExtractor : IBinaryFeatureExtractor
stream.Position = debugDirectoryRva;
Span<byte> debugDir = stackalloc byte[8];
stream.Read(debugDir);
var debugDirRead = stream.ReadAtLeast(debugDir, debugDir.Length, throwOnEndOfStream: false);
if (debugDirRead < debugDir.Length)
return false;
var debugRva = BitConverter.ToUInt32(debugDir[..4]);
return debugRva != 0;

View File

@@ -0,0 +1,18 @@
namespace StellaOps.BinaryIndex.Core.Services;
internal static class StreamGuard
{
public static void EnsureSeekable(Stream stream, string operation)
{
if (stream is null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanSeek || !stream.CanRead)
{
throw new InvalidOperationException(
$"Stream must be seekable and readable for {operation}.");
}
}
}

View File

@@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0116-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Core. |
| AUDIT-0116-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Core. |
| AUDIT-0116-A | TODO | Pending approval for changes. |
| AUDIT-0116-A | DONE | Applied core fixes + tests. |

View File

@@ -1,13 +1,13 @@
// -----------------------------------------------------------------------------
// AlpineCorpusConnector.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-16 Create AlpineCorpusConnector for Alpine APK
// Task: BACKPORT-16 - Create AlpineCorpusConnector for Alpine APK
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.BinaryIndex.Core.Models;
using StellaOps.BinaryIndex.Core.Services;
using StellaOps.BinaryIndex.Corpus;
namespace StellaOps.BinaryIndex.Corpus.Alpine;
@@ -20,27 +20,28 @@ public sealed class AlpineCorpusConnector : IBinaryCorpusConnector
{
private readonly IAlpinePackageSource _packageSource;
private readonly AlpinePackageExtractor _extractor;
private readonly IBinaryFeatureExtractor _featureExtractor;
private readonly ICorpusSnapshotRepository _snapshotRepo;
private readonly ILogger<AlpineCorpusConnector> _logger;
private const string DefaultMirror = "https://dl-cdn.alpinelinux.org/alpine";
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
public string ConnectorId => "alpine";
public string[] SupportedDistros => ["alpine"];
public ImmutableArray<string> SupportedDistros { get; } = ImmutableArray.Create("alpine");
public AlpineCorpusConnector(
IAlpinePackageSource packageSource,
AlpinePackageExtractor extractor,
IBinaryFeatureExtractor featureExtractor,
ICorpusSnapshotRepository snapshotRepo,
ILogger<AlpineCorpusConnector> logger)
ILogger<AlpineCorpusConnector> logger,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_packageSource = packageSource;
_extractor = extractor;
_featureExtractor = featureExtractor;
_snapshotRepo = snapshotRepo;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new SystemGuidProvider();
}
public async Task<CorpusSnapshot> FetchSnapshotAsync(CorpusQuery query, CancellationToken ct = default)
@@ -71,13 +72,15 @@ public sealed class AlpineCorpusConnector : IBinaryCorpusConnector
var packageList = packages.ToList();
var metadataDigest = ComputeMetadataDigest(packageList);
var snapshot = new CorpusSnapshot(
Id: Guid.NewGuid(),
Distro: "alpine",
Release: query.Release,
Architecture: query.Architecture,
MetadataDigest: metadataDigest,
CapturedAt: DateTimeOffset.UtcNow);
var snapshot = new CorpusSnapshot
{
Id = _guidProvider.NewGuid(),
Distro = query.Distro,
Release = query.Release,
Architecture = query.Architecture,
MetadataDigest = metadataDigest,
CapturedAt = _timeProvider.GetUtcNow()
};
await _snapshotRepo.CreateAsync(snapshot, ct);
@@ -101,14 +104,16 @@ public sealed class AlpineCorpusConnector : IBinaryCorpusConnector
foreach (var pkg in packages)
{
yield return new PackageInfo(
Name: pkg.PackageName,
Version: pkg.Version,
SourcePackage: pkg.Origin ?? pkg.PackageName,
Architecture: pkg.Architecture,
Filename: pkg.Filename,
Size: pkg.Size,
Sha256: pkg.Checksum);
yield return new PackageInfo
{
Name = pkg.PackageName,
Version = pkg.Version,
SourcePackage = pkg.Origin ?? pkg.PackageName,
Architecture = pkg.Architecture,
Filename = pkg.Filename,
Size = pkg.Size,
Sha256 = pkg.Checksum
};
}
}

View File

@@ -1,13 +1,12 @@
// -----------------------------------------------------------------------------
// AlpinePackageExtractor.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-16 Create AlpineCorpusConnector for Alpine APK
// Task: BACKPORT-16 - Create AlpineCorpusConnector for Alpine APK
// -----------------------------------------------------------------------------
using System.IO.Compression;
using Microsoft.Extensions.Logging;
using SharpCompress.Archives;
using SharpCompress.Archives.Tar;
using SharpCompress.Compressors.Deflate;
using StellaOps.BinaryIndex.Core.Models;
using StellaOps.BinaryIndex.Core.Services;
using StellaOps.BinaryIndex.Corpus;
@@ -24,6 +23,8 @@ public sealed class AlpinePackageExtractor
// ELF magic bytes
private static readonly byte[] ElfMagic = [0x7F, 0x45, 0x4C, 0x46];
private const long MaxEntrySizeBytes = 64L * 1024 * 1024;
private const long MaxSegmentSizeBytes = 256L * 1024 * 1024;
public AlpinePackageExtractor(
IBinaryFeatureExtractor featureExtractor,
@@ -46,45 +47,71 @@ public sealed class AlpinePackageExtractor
CancellationToken ct = default)
{
var results = new List<ExtractedBinaryInfo>();
var seekableStream = await EnsureSeekableStreamAsync(apkStream, ct);
var disposeSeekable = !ReferenceEquals(seekableStream, apkStream);
// APK is gzipped tar: signature.tar.gz + control.tar.gz + data.tar.gz
// We need to extract data.tar.gz which contains the actual files
try
{
var dataTar = await ExtractDataTarAsync(apkStream, ct);
if (dataTar == null)
{
_logger.LogWarning("Could not find data.tar in {Package}", pkg.Name);
return results;
}
using var archive = TarArchive.Open(dataTar);
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
while (seekableStream.Position < seekableStream.Length)
{
ct.ThrowIfCancellationRequested();
var startPosition = seekableStream.Position;
// Check if this is an ELF binary
using var entryStream = entry.OpenEntryStream();
using var ms = new MemoryStream();
await entryStream.CopyToAsync(ms, ct);
ms.Position = 0;
if (!IsElfBinary(ms))
using var gzip = new GZipStream(
seekableStream,
CompressionMode.Decompress,
leaveOpen: true);
await using var segmentStream = await ExtractSegmentAsync(gzip, ct);
if (segmentStream is null)
{
continue;
break;
}
ms.Position = 0;
using var archive = TarArchive.Open(segmentStream);
try
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
{
var identity = await _featureExtractor.ExtractIdentityAsync(ms, ct);
results.Add(new ExtractedBinaryInfo(identity, entry.Key ?? ""));
ct.ThrowIfCancellationRequested();
if (entry.Size <= 0 || entry.Size > MaxEntrySizeBytes)
{
_logger.LogWarning(
"Skipping entry {Entry} in {Package} due to size {Size} bytes",
entry.Key,
pkg.Name,
entry.Size);
continue;
}
using var entryStream = entry.OpenEntryStream();
using var ms = new MemoryStream((int)entry.Size);
await entryStream.CopyToAsync(ms, ct);
ms.Position = 0;
if (!IsElfBinary(ms))
{
continue;
}
ms.Position = 0;
try
{
var identity = await _featureExtractor.ExtractIdentityAsync(ms, ct);
results.Add(new ExtractedBinaryInfo(identity, entry.Key ?? ""));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to extract identity from {File} in {Package}",
entry.Key, pkg.Name);
}
}
catch (Exception ex)
if (seekableStream.Position <= startPosition)
{
_logger.LogWarning(ex, "Failed to extract identity from {File} in {Package}",
entry.Key, pkg.Name);
break;
}
}
}
@@ -92,24 +119,93 @@ public sealed class AlpinePackageExtractor
{
_logger.LogError(ex, "Failed to extract binaries from Alpine package {Package}", pkg.Name);
}
finally
{
if (disposeSeekable)
{
await seekableStream.DisposeAsync();
}
}
return results;
}
private static async Task<Stream?> ExtractDataTarAsync(Stream apkStream, CancellationToken ct)
private static async Task<Stream> EnsureSeekableStreamAsync(Stream apkStream, CancellationToken ct)
{
// APK packages contain multiple gzipped tar archives concatenated
// We need to skip to the data.tar.gz portion
// The structure is: signature.tar.gz + control.tar.gz + data.tar.gz
if (apkStream.CanSeek)
{
apkStream.Position = 0;
return apkStream;
}
using var gzip = new GZipStream(apkStream, SharpCompress.Compressors.CompressionMode.Decompress);
using var ms = new MemoryStream();
await gzip.CopyToAsync(ms, ct);
ms.Position = 0;
var tempPath = Path.GetTempFileName();
var tempStream = new FileStream(
tempPath,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 81920,
FileOptions.DeleteOnClose);
// For simplicity, we'll just try to extract from the combined tar
// In a real implementation, we'd need to properly parse the multi-part structure
return ms;
await apkStream.CopyToAsync(tempStream, ct);
tempStream.Position = 0;
return tempStream;
}
private static async Task<Stream?> ExtractSegmentAsync(Stream gzipStream, CancellationToken ct)
{
var tempPath = Path.GetTempFileName();
var tempStream = new FileStream(
tempPath,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 81920,
FileOptions.DeleteOnClose);
var totalCopied = await CopyToWithLimitAsync(
gzipStream,
tempStream,
MaxSegmentSizeBytes,
ct);
if (totalCopied == 0)
{
await tempStream.DisposeAsync();
return null;
}
tempStream.Position = 0;
return tempStream;
}
private static async Task<long> CopyToWithLimitAsync(
Stream source,
Stream destination,
long maxBytes,
CancellationToken ct)
{
var buffer = new byte[81920];
long total = 0;
while (true)
{
var read = await source.ReadAsync(buffer.AsMemory(0, buffer.Length), ct);
if (read == 0)
{
break;
}
total += read;
if (total > maxBytes)
{
throw new InvalidDataException("APK segment exceeds size limit.");
}
await destination.WriteAsync(buffer.AsMemory(0, read), ct);
}
return total;
}
private static bool IsElfBinary(Stream stream)

View File

@@ -1,9 +1,11 @@
// -----------------------------------------------------------------------------
// IAlpinePackageSource.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-16 Create AlpineCorpusConnector for Alpine APK
// Task: BACKPORT-16 - Create AlpineCorpusConnector for Alpine APK
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.BinaryIndex.Corpus.Alpine;
/// <summary>
@@ -76,10 +78,10 @@ public sealed record AlpinePackageMetadata
public string? Maintainer { get; init; }
/// <summary>Dependencies (D:).</summary>
public string[]? Dependencies { get; init; }
public ImmutableArray<string> Dependencies { get; init; } = ImmutableArray<string>.Empty;
/// <summary>Provides (p:).</summary>
public string[]? Provides { get; init; }
public ImmutableArray<string> Provides { get; init; } = ImmutableArray<string>.Empty;
/// <summary>Build timestamp (t:).</summary>
public DateTimeOffset? BuildTime { get; init; }

View File

@@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0119-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Corpus.Alpine. |
| AUDIT-0119-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Corpus.Alpine. |
| AUDIT-0119-A | TODO | Pending approval for changes. |
| AUDIT-0119-A | DOING | Pending approval for changes. |

View File

@@ -1,7 +1,6 @@
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.BinaryIndex.Core.Services;
using StellaOps.BinaryIndex.Corpus;
namespace StellaOps.BinaryIndex.Corpus.Debian;
@@ -13,31 +12,33 @@ public sealed class DebianCorpusConnector : IBinaryCorpusConnector
{
private readonly IDebianPackageSource _packageSource;
private readonly DebianPackageExtractor _extractor;
private readonly IBinaryFeatureExtractor _featureExtractor;
private readonly ICorpusSnapshotRepository _snapshotRepo;
private readonly ILogger<DebianCorpusConnector> _logger;
private const string DefaultMirror = "https://deb.debian.org/debian";
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
public string ConnectorId => "debian";
public string[] SupportedDistros => ["debian", "ubuntu"];
public ImmutableArray<string> SupportedDistros { get; } = ImmutableArray.Create("debian", "ubuntu");
public DebianCorpusConnector(
IDebianPackageSource packageSource,
DebianPackageExtractor extractor,
IBinaryFeatureExtractor featureExtractor,
ICorpusSnapshotRepository snapshotRepo,
ILogger<DebianCorpusConnector> logger)
ILogger<DebianCorpusConnector> logger,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_packageSource = packageSource;
_extractor = extractor;
_featureExtractor = featureExtractor;
_snapshotRepo = snapshotRepo;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new SystemGuidProvider();
}
public async Task<CorpusSnapshot> FetchSnapshotAsync(CorpusQuery query, CancellationToken ct = default)
{
EnsureSupportedDistro(query.Distro);
_logger.LogInformation(
"Fetching corpus snapshot for {Distro} {Release}/{Architecture}",
query.Distro, query.Release, query.Architecture);
@@ -63,22 +64,23 @@ public sealed class DebianCorpusConnector : IBinaryCorpusConnector
ct);
// Compute metadata digest from package list
var packageList = packages.ToList();
var metadataDigest = ComputeMetadataDigest(packageList);
var metadataDigest = ComputeMetadataDigest(packages);
var snapshot = new CorpusSnapshot(
Id: Guid.NewGuid(),
Distro: query.Distro,
Release: query.Release,
Architecture: query.Architecture,
MetadataDigest: metadataDigest,
CapturedAt: DateTimeOffset.UtcNow);
var snapshot = new CorpusSnapshot
{
Id = _guidProvider.NewGuid(),
Distro = query.Distro,
Release = query.Release,
Architecture = query.Architecture,
MetadataDigest = metadataDigest,
CapturedAt = _timeProvider.GetUtcNow()
};
await _snapshotRepo.CreateAsync(snapshot, ct);
_logger.LogInformation(
"Created corpus snapshot {SnapshotId} with {PackageCount} packages",
snapshot.Id, packageList.Count);
snapshot.Id, packages.Length);
return snapshot;
}
@@ -97,14 +99,16 @@ public sealed class DebianCorpusConnector : IBinaryCorpusConnector
foreach (var pkg in packages)
{
yield return new PackageInfo(
Name: pkg.Package,
Version: pkg.Version,
SourcePackage: pkg.Source ?? pkg.Package,
Architecture: pkg.Architecture,
Filename: pkg.Filename,
Size: 0, // We don't have size in current implementation
Sha256: pkg.SHA256);
yield return new PackageInfo
{
Name = pkg.Package,
Version = pkg.Version,
SourcePackage = pkg.Source ?? pkg.Package,
Architecture = pkg.Architecture,
Filename = pkg.Filename,
Size = pkg.Size ?? 0,
Sha256 = pkg.SHA256
};
}
}
@@ -154,11 +158,21 @@ public sealed class DebianCorpusConnector : IBinaryCorpusConnector
{
// Simple digest: SHA256 of concatenated package names and versions
var combined = string.Join("|", packages
.OrderBy(p => p.Package)
.Select(p => $"{p.Package}:{p.Version}:{p.SHA256}"));
.OrderBy(p => p.Package, StringComparer.Ordinal)
.ThenBy(p => p.Version, StringComparer.Ordinal)
.ThenBy(p => p.Architecture, StringComparer.Ordinal)
.Select(p => $"{p.Package}:{p.Version}:{p.Architecture}:{p.SHA256}:{p.Size ?? 0}"));
using var sha256 = System.Security.Cryptography.SHA256.Create();
var hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(combined));
return Convert.ToHexString(hash).ToLowerInvariant();
}
private void EnsureSupportedDistro(string distro)
{
if (!SupportedDistros.Contains(distro, StringComparer.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException(nameof(distro), distro, "Unsupported distro.");
}
}
}

View File

@@ -1,5 +1,6 @@
using System.Collections.Immutable;
using System.Globalization;
using System.IO.Compression;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
namespace StellaOps.BinaryIndex.Corpus.Debian;
@@ -13,6 +14,9 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
private readonly ILogger<DebianMirrorPackageSource> _logger;
private readonly string _mirrorUrl;
private static readonly ImmutableHashSet<string> SupportedDistros =
ImmutableHashSet.Create(StringComparer.OrdinalIgnoreCase, "debian", "ubuntu");
public DebianMirrorPackageSource(
HttpClient httpClient,
ILogger<DebianMirrorPackageSource> logger,
@@ -23,12 +27,13 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
_mirrorUrl = mirrorUrl.TrimEnd('/');
}
public async Task<IEnumerable<DebianPackageMetadata>> FetchPackageIndexAsync(
public async Task<ImmutableArray<DebianPackageMetadata>> FetchPackageIndexAsync(
string distro,
string release,
string architecture,
CancellationToken ct = default)
{
ValidateInputs(distro, release, architecture);
var packagesUrl = $"{_mirrorUrl}/dists/{release}/main/binary-{architecture}/Packages.gz";
_logger.LogInformation("Fetching package index: {Url}", packagesUrl);
@@ -41,8 +46,8 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
using var reader = new StreamReader(decompressed);
var packages = new List<DebianPackageMetadata>();
DebianPackageMetadata? current = null;
var currentFields = new Dictionary<string, string>();
var currentFields = new Dictionary<string, string>(StringComparer.Ordinal);
string? lastKey = null;
while (await reader.ReadLineAsync(ct) is { } line)
{
@@ -57,12 +62,13 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
}
currentFields.Clear();
}
lastKey = null;
continue;
}
if (line.StartsWith(' ') || line.StartsWith('\t'))
{
// Continuation line - ignore for now
AppendContinuation(currentFields, lastKey, line);
continue;
}
@@ -72,6 +78,7 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
var key = line[..colonIndex];
var value = line[(colonIndex + 1)..].Trim();
currentFields[key] = value;
lastKey = key;
}
}
@@ -81,10 +88,13 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
packages.Add(lastPkg);
}
_logger.LogInformation("Fetched {Count} packages for {Release}/{Arch}",
packages.Count, release, architecture);
_logger.LogInformation(
"Fetched {Count} packages for {Release}/{Arch}",
packages.Count,
release,
architecture);
return packages;
return NormalizePackages(packages);
}
public async Task<Stream> DownloadPackageAsync(string poolPath, CancellationToken ct = default)
@@ -96,14 +106,8 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
var response = await _httpClient.GetAsync(packageUrl, HttpCompletionOption.ResponseHeadersRead, ct);
response.EnsureSuccessStatusCode();
var memoryStream = new MemoryStream();
await using (var contentStream = await response.Content.ReadAsStreamAsync(ct))
{
await contentStream.CopyToAsync(memoryStream, ct);
}
memoryStream.Position = 0;
return memoryStream;
var contentStream = await response.Content.ReadAsStreamAsync(ct);
return new HttpResponseStream(response, contentStream);
}
private static bool TryParsePackage(Dictionary<string, string> fields, out DebianPackageMetadata pkg)
@@ -120,6 +124,13 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
}
fields.TryGetValue("Source", out var source);
fields.TryGetValue("Size", out var sizeValue);
long? size = null;
if (!string.IsNullOrWhiteSpace(sizeValue) &&
long.TryParse(sizeValue, NumberStyles.None, CultureInfo.InvariantCulture, out var parsedSize))
{
size = parsedSize;
}
pkg = new DebianPackageMetadata
{
@@ -128,9 +139,137 @@ public sealed partial class DebianMirrorPackageSource : IDebianPackageSource
Architecture = architecture,
Filename = filename,
SHA256 = sha256,
Size = size,
Source = source
};
return true;
}
private static void AppendContinuation(
Dictionary<string, string> fields,
string? lastKey,
string line)
{
if (lastKey is null)
{
return;
}
var continuation = line.TrimStart();
if (continuation.Length == 0)
{
return;
}
if (fields.TryGetValue(lastKey, out var existing))
{
fields[lastKey] = $"{existing}\n{continuation}";
}
}
private static ImmutableArray<DebianPackageMetadata> NormalizePackages(
IEnumerable<DebianPackageMetadata> packages)
{
return packages
.OrderBy(pkg => pkg.Package, StringComparer.Ordinal)
.ThenBy(pkg => pkg.Version, StringComparer.Ordinal)
.ThenBy(pkg => pkg.Architecture, StringComparer.Ordinal)
.ThenBy(pkg => pkg.Filename, StringComparer.Ordinal)
.ToImmutableArray();
}
private static void ValidateInputs(string distro, string release, string architecture)
{
if (string.IsNullOrWhiteSpace(distro))
{
throw new ArgumentException("Distro is required.", nameof(distro));
}
if (!SupportedDistros.Contains(distro))
{
throw new ArgumentOutOfRangeException(
nameof(distro),
distro,
"Unsupported Debian distro.");
}
if (string.IsNullOrWhiteSpace(release))
{
throw new ArgumentException("Release is required.", nameof(release));
}
if (string.IsNullOrWhiteSpace(architecture))
{
throw new ArgumentException("Architecture is required.", nameof(architecture));
}
}
private sealed class HttpResponseStream : Stream
{
private readonly HttpResponseMessage _response;
private readonly Stream _inner;
public HttpResponseStream(HttpResponseMessage response, Stream inner)
{
_response = response;
_inner = inner;
}
public override bool CanRead => _inner.CanRead;
public override bool CanSeek => _inner.CanSeek;
public override bool CanWrite => _inner.CanWrite;
public override long Length => _inner.Length;
public override long Position
{
get => _inner.Position;
set => _inner.Position = value;
}
public override void Flush() => _inner.Flush();
public override Task FlushAsync(CancellationToken cancellationToken) =>
_inner.FlushAsync(cancellationToken);
public override int Read(byte[] buffer, int offset, int count) =>
_inner.Read(buffer, offset, count);
public override ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default) =>
_inner.ReadAsync(buffer, cancellationToken);
public override long Seek(long offset, SeekOrigin origin) =>
_inner.Seek(offset, origin);
public override void SetLength(long value) =>
_inner.SetLength(value);
public override void Write(byte[] buffer, int offset, int count) =>
_inner.Write(buffer, offset, count);
public override ValueTask WriteAsync(
ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default) =>
_inner.WriteAsync(buffer, cancellationToken);
protected override void Dispose(bool disposing)
{
if (disposing)
{
_inner.Dispose();
_response.Dispose();
}
base.Dispose(disposing);
}
public override async ValueTask DisposeAsync()
{
await _inner.DisposeAsync();
_response.Dispose();
await base.DisposeAsync();
}
}
}

View File

@@ -14,6 +14,9 @@ namespace StellaOps.BinaryIndex.Corpus.Debian;
/// </summary>
public sealed class DebianPackageExtractor
{
private const long MaxDataTarSizeBytes = 512L * 1024 * 1024;
private const long MaxEntrySizeBytes = 64L * 1024 * 1024;
private readonly IBinaryFeatureExtractor _featureExtractor;
private readonly ILogger<DebianPackageExtractor> _logger;
@@ -42,16 +45,33 @@ public sealed class DebianPackageExtractor
foreach (var entry in archive.Entries.Where(e => !e.IsDirectory))
{
if (entry.Key == null || !entry.Key.StartsWith("data.tar"))
if (entry.Key == null || !entry.Key.StartsWith("data.tar", StringComparison.Ordinal))
continue;
// Extract data.tar.*
using var dataTarStream = new MemoryStream();
entry.WriteTo(dataTarStream);
dataTarStream.Position = 0;
try
{
if (entry.Size > MaxDataTarSizeBytes)
{
_logger.LogWarning(
"Skipping data archive {EntryKey} in {Package} due to size {SizeBytes}",
entry.Key,
metadata.Package,
entry.Size);
continue;
}
// Now extract from data.tar
await ExtractFromDataTarAsync(dataTarStream, metadata, binaries, ct);
await using var dataTarStream = await ExtractDataTarStreamAsync(entry, ct);
var extracted = await ExtractFromDataTarAsync(dataTarStream, metadata, ct);
binaries.AddRange(extracted);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to extract data archive {EntryKey} from {Package}",
entry.Key,
metadata.Package);
}
}
}
catch (Exception ex)
@@ -63,12 +83,12 @@ public sealed class DebianPackageExtractor
return binaries.ToImmutableArray();
}
private async Task ExtractFromDataTarAsync(
internal async Task<ImmutableArray<ExtractedBinaryInternal>> ExtractFromDataTarAsync(
Stream dataTarStream,
DebianPackageMetadata metadata,
List<ExtractedBinaryInternal> binaries,
CancellationToken ct)
CancellationToken ct = default)
{
var binaries = new List<ExtractedBinaryInternal>();
using var tarArchive = TarArchive.Open(dataTarStream);
foreach (var entry in tarArchive.Entries.Where(e => !e.IsDirectory))
@@ -76,15 +96,24 @@ public sealed class DebianPackageExtractor
if (entry.Key == null)
continue;
if (entry.Size > MaxEntrySizeBytes)
{
_logger.LogDebug(
"Skipping {Path} in {Package} due to size {SizeBytes}",
entry.Key,
metadata.Package,
entry.Size);
continue;
}
// Only process binaries in typical locations
if (!IsPotentialBinary(entry.Key))
continue;
try
{
using var binaryStream = new MemoryStream();
entry.WriteTo(binaryStream);
binaryStream.Position = 0;
await using var entryStream = entry.OpenEntryStream();
await using var binaryStream = await BufferEntryAsync(entryStream, entry.Size, ct);
if (!_featureExtractor.CanExtract(binaryStream))
continue;
@@ -107,19 +136,76 @@ public sealed class DebianPackageExtractor
_logger.LogDebug(ex, "Skipped {Path} in {Package}", entry.Key, metadata.Package);
}
}
return binaries.ToImmutableArray();
}
private static bool IsPotentialBinary(string path)
{
// Typical binary locations in Debian packages
return path.StartsWith("./usr/bin/") ||
path.StartsWith("./usr/sbin/") ||
path.StartsWith("./bin/") ||
path.StartsWith("./sbin/") ||
path.StartsWith("./usr/lib/") ||
path.StartsWith("./lib/") ||
path.Contains(".so") ||
path.EndsWith(".so");
return path.StartsWith("./usr/bin/", StringComparison.Ordinal) ||
path.StartsWith("./usr/sbin/", StringComparison.Ordinal) ||
path.StartsWith("./bin/", StringComparison.Ordinal) ||
path.StartsWith("./sbin/", StringComparison.Ordinal) ||
path.StartsWith("./usr/lib/", StringComparison.Ordinal) ||
path.StartsWith("./lib/", StringComparison.Ordinal) ||
path.Contains(".so", StringComparison.Ordinal) ||
path.EndsWith(".so", StringComparison.Ordinal);
}
private async Task<Stream> ExtractDataTarStreamAsync(IArchiveEntry entry, CancellationToken ct)
{
await using var entryStream = entry.OpenEntryStream();
var tempStream = CreateTempStream();
await CopyToWithLimitAsync(entryStream, tempStream, MaxDataTarSizeBytes, ct);
tempStream.Position = 0;
return tempStream;
}
private static async Task<Stream> BufferEntryAsync(Stream entryStream, long size, CancellationToken ct)
{
var bufferStream = new MemoryStream(
size > 0 && size <= MaxEntrySizeBytes ? (int)size : 0);
await CopyToWithLimitAsync(entryStream, bufferStream, MaxEntrySizeBytes, ct);
bufferStream.Position = 0;
return bufferStream;
}
private static async Task<long> CopyToWithLimitAsync(
Stream source,
Stream destination,
long maxBytes,
CancellationToken ct)
{
var buffer = new byte[16 * 1024];
long total = 0;
int read;
while ((read = await source.ReadAsync(buffer, ct)) > 0)
{
total += read;
if (total > maxBytes)
{
throw new InvalidOperationException(
$"Archive entry exceeded limit of {maxBytes} bytes.");
}
await destination.WriteAsync(buffer.AsMemory(0, read), ct);
}
return total;
}
private static FileStream CreateTempStream()
{
var path = Path.GetTempFileName();
return new FileStream(
path,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
16 * 1024,
FileOptions.DeleteOnClose | FileOptions.SequentialScan);
}
}

View File

@@ -1,3 +1,5 @@
using System.Collections.Immutable;
namespace StellaOps.BinaryIndex.Corpus.Debian;
/// <summary>
@@ -8,7 +10,7 @@ public interface IDebianPackageSource
/// <summary>
/// Fetches package metadata from Packages.gz index.
/// </summary>
Task<IEnumerable<DebianPackageMetadata>> FetchPackageIndexAsync(
Task<ImmutableArray<DebianPackageMetadata>> FetchPackageIndexAsync(
string distro,
string release,
string architecture,
@@ -29,5 +31,6 @@ public sealed record DebianPackageMetadata
public required string Architecture { get; init; }
public required string Filename { get; init; } // Pool path
public required string SHA256 { get; init; }
public long? Size { get; init; }
public string? Source { get; init; }
}

View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
@@ -13,6 +14,10 @@
<PackageReference Include="SharpCompress" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="StellaOps.BinaryIndex.Corpus.Debian.Tests" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.BinaryIndex.Core\StellaOps.BinaryIndex.Core.csproj" />
<ProjectReference Include="..\StellaOps.BinaryIndex.Corpus\StellaOps.BinaryIndex.Corpus.csproj" />

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0120-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Corpus.Debian. |
| AUDIT-0120-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Corpus.Debian. |
| AUDIT-0120-A | TODO | Pending approval for changes. |
| AUDIT-0120-A | DONE | Applied + tests. |

View File

@@ -1,9 +1,11 @@
// -----------------------------------------------------------------------------
// IRpmPackageSource.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-14 Create RpmCorpusConnector for RHEL/Fedora/CentOS
// Task: BACKPORT-14 - Create RpmCorpusConnector for RHEL/Fedora/CentOS
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.BinaryIndex.Corpus.Rpm;
/// <summary>
@@ -19,7 +21,7 @@ public interface IRpmPackageSource
/// <param name="architecture">Target architecture (x86_64, aarch64).</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>Package metadata from primary.xml.</returns>
Task<IReadOnlyList<RpmPackageMetadata>> FetchPackageIndexAsync(
Task<ImmutableArray<RpmPackageMetadata>> FetchPackageIndexAsync(
string distro,
string release,
string architecture,
@@ -89,3 +91,4 @@ public sealed record RpmPackageMetadata
/// <summary>Build timestamp.</summary>
public DateTimeOffset? BuildTime { get; init; }
}

View File

@@ -1,13 +1,12 @@
// -----------------------------------------------------------------------------
// RpmCorpusConnector.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-14 Create RpmCorpusConnector for RHEL/Fedora/CentOS
// Task: BACKPORT-14 - Create RpmCorpusConnector for RHEL/Fedora/CentOS
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.BinaryIndex.Core.Models;
using StellaOps.BinaryIndex.Core.Services;
using StellaOps.BinaryIndex.Corpus;
namespace StellaOps.BinaryIndex.Corpus.Rpm;
@@ -19,29 +18,34 @@ public sealed class RpmCorpusConnector : IBinaryCorpusConnector
{
private readonly IRpmPackageSource _packageSource;
private readonly RpmPackageExtractor _extractor;
private readonly IBinaryFeatureExtractor _featureExtractor;
private readonly ICorpusSnapshotRepository _snapshotRepo;
private readonly ILogger<RpmCorpusConnector> _logger;
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
public string ConnectorId => "rpm";
public string[] SupportedDistros => ["rhel", "fedora", "centos", "rocky", "almalinux"];
public ImmutableArray<string> SupportedDistros { get; } =
ImmutableArray.Create("rhel", "fedora", "centos", "rocky", "almalinux");
public RpmCorpusConnector(
IRpmPackageSource packageSource,
RpmPackageExtractor extractor,
IBinaryFeatureExtractor featureExtractor,
ICorpusSnapshotRepository snapshotRepo,
ILogger<RpmCorpusConnector> logger)
ILogger<RpmCorpusConnector> logger,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_packageSource = packageSource;
_extractor = extractor;
_featureExtractor = featureExtractor;
_snapshotRepo = snapshotRepo;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new SystemGuidProvider();
}
public async Task<CorpusSnapshot> FetchSnapshotAsync(CorpusQuery query, CancellationToken ct = default)
{
EnsureSupportedDistro(query.Distro);
_logger.LogInformation(
"Fetching RPM corpus snapshot for {Distro} {Release}/{Architecture}",
query.Distro, query.Release, query.Architecture);
@@ -66,22 +70,23 @@ public sealed class RpmCorpusConnector : IBinaryCorpusConnector
query.Architecture,
ct);
var packageList = packages.ToList();
var metadataDigest = ComputeMetadataDigest(packageList);
var metadataDigest = ComputeMetadataDigest(packages);
var snapshot = new CorpusSnapshot(
Id: Guid.NewGuid(),
Distro: query.Distro,
Release: query.Release,
Architecture: query.Architecture,
MetadataDigest: metadataDigest,
CapturedAt: DateTimeOffset.UtcNow);
var snapshot = new CorpusSnapshot
{
Id = _guidProvider.NewGuid(),
Distro = query.Distro,
Release = query.Release,
Architecture = query.Architecture,
MetadataDigest = metadataDigest,
CapturedAt = _timeProvider.GetUtcNow()
};
await _snapshotRepo.CreateAsync(snapshot, ct);
_logger.LogInformation(
"Created RPM corpus snapshot {SnapshotId} with {PackageCount} packages",
snapshot.Id, packageList.Count);
snapshot.Id, packages.Length);
return snapshot;
}
@@ -100,14 +105,16 @@ public sealed class RpmCorpusConnector : IBinaryCorpusConnector
foreach (var pkg in packages)
{
yield return new PackageInfo(
Name: pkg.Name,
Version: $"{pkg.Version}-{pkg.Release}",
SourcePackage: pkg.SourceRpm ?? pkg.Name,
Architecture: pkg.Arch,
Filename: pkg.Filename,
Size: pkg.Size,
Sha256: pkg.Checksum);
yield return new PackageInfo
{
Name = pkg.Name,
Version = $"{pkg.Version}-{pkg.Release}",
SourcePackage = pkg.SourceRpm ?? pkg.Name,
Architecture = pkg.Arch,
Filename = pkg.Filename,
Size = pkg.Size,
Sha256 = pkg.Checksum
};
}
}
@@ -146,11 +153,23 @@ public sealed class RpmCorpusConnector : IBinaryCorpusConnector
private static string ComputeMetadataDigest(IEnumerable<RpmPackageMetadata> packages)
{
var combined = string.Join("|", packages
.OrderBy(p => p.Name)
.Select(p => $"{p.Name}:{p.Epoch}:{p.Version}-{p.Release}:{p.Checksum}"));
.OrderBy(p => p.Name, StringComparer.Ordinal)
.ThenBy(p => p.Version, StringComparer.Ordinal)
.ThenBy(p => p.Release, StringComparer.Ordinal)
.ThenBy(p => p.Arch, StringComparer.Ordinal)
.Select(p => $"{p.Name}:{p.Epoch}:{p.Version}-{p.Release}:{p.Arch}:{p.Checksum}:{p.Size}"));
using var sha256 = System.Security.Cryptography.SHA256.Create();
var hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(combined));
return Convert.ToHexString(hash).ToLowerInvariant();
}
private void EnsureSupportedDistro(string distro)
{
if (!SupportedDistros.Contains(distro, StringComparer.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException(nameof(distro), distro, "Unsupported distro.");
}
}
}

View File

@@ -1,11 +1,11 @@
// -----------------------------------------------------------------------------
// RpmPackageExtractor.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-14 Create RpmCorpusConnector for RHEL/Fedora/CentOS
// Task: BACKPORT-14 - Create RpmCorpusConnector for RHEL/Fedora/CentOS
// -----------------------------------------------------------------------------
using System.IO.Compression;
using Microsoft.Extensions.Logging;
using SharpCompress.Archives;
using SharpCompress.Compressors.Xz;
using SharpCompress.Readers;
using StellaOps.BinaryIndex.Core.Models;
@@ -19,6 +19,11 @@ namespace StellaOps.BinaryIndex.Corpus.Rpm;
/// </summary>
public sealed class RpmPackageExtractor
{
private const int RpmLeadSize = 96;
private const long MaxPayloadCompressedBytes = 512L * 1024 * 1024;
private const long MaxPayloadUncompressedBytes = 1024L * 1024 * 1024;
private const long MaxEntrySizeBytes = 64L * 1024 * 1024;
private readonly IBinaryFeatureExtractor _featureExtractor;
private readonly ILogger<RpmPackageExtractor> _logger;
@@ -28,6 +33,18 @@ public sealed class RpmPackageExtractor
// RPM magic bytes
private static readonly byte[] RpmMagic = [0xED, 0xAB, 0xEE, 0xDB];
private static readonly byte[] XzMagic = [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00];
private static readonly byte[] GzipMagic = [0x1F, 0x8B];
private static readonly byte[] ZstdMagic = [0x28, 0xB5, 0x2F, 0xFD];
internal enum PayloadCompression
{
None,
Xz,
Gzip,
Zstd
}
public RpmPackageExtractor(
IBinaryFeatureExtractor featureExtractor,
ILogger<RpmPackageExtractor> logger)
@@ -53,7 +70,7 @@ public sealed class RpmPackageExtractor
try
{
// RPM structure: lead + signature header + header + payload (cpio.xz/cpio.gz/cpio.zstd)
var payloadStream = await ExtractPayloadAsync(rpmStream, ct);
await using var payloadStream = await ExtractPayloadAsync(rpmStream, ct);
if (payloadStream == null)
{
_logger.LogWarning("Could not extract payload from RPM {Package}", pkg.Name);
@@ -68,21 +85,29 @@ public sealed class RpmPackageExtractor
if (reader.Entry.IsDirectory)
continue;
using var entryStream = reader.OpenEntryStream();
using var ms = new MemoryStream();
await entryStream.CopyToAsync(ms, ct);
ms.Position = 0;
if (reader.Entry.Size > MaxEntrySizeBytes)
{
_logger.LogDebug(
"Skipping {File} in RPM {Package} due to size {SizeBytes}",
reader.Entry.Key,
pkg.Name,
reader.Entry.Size);
continue;
}
if (!IsElfBinary(ms))
await using var entryStream = reader.OpenEntryStream();
await using var buffered = await BufferEntryAsync(entryStream, reader.Entry.Size, ct);
if (!IsElfBinary(buffered))
{
continue;
}
ms.Position = 0;
buffered.Position = 0;
try
{
var identity = await _featureExtractor.ExtractIdentityAsync(ms, ct);
var identity = await _featureExtractor.ExtractIdentityAsync(buffered, ct);
results.Add(new ExtractedBinaryInfo(identity, reader.Entry.Key ?? ""));
}
catch (Exception ex)
@@ -103,9 +128,8 @@ public sealed class RpmPackageExtractor
private async Task<Stream?> ExtractPayloadAsync(Stream rpmStream, CancellationToken ct)
{
// Skip RPM lead (96 bytes)
var lead = new byte[96];
var read = await rpmStream.ReadAsync(lead.AsMemory(0, 96), ct);
if (read != 96 || !lead.AsSpan(0, 4).SequenceEqual(RpmMagic))
var lead = new byte[RpmLeadSize];
if (!await ReadExactAsync(rpmStream, lead, ct) || !lead.AsSpan(0, 4).SequenceEqual(RpmMagic))
{
_logger.LogWarning("Invalid RPM lead");
return null;
@@ -128,24 +152,28 @@ public sealed class RpmPackageExtractor
}
// The rest is the payload (compressed cpio)
var payloadMs = new MemoryStream();
await rpmStream.CopyToAsync(payloadMs, ct);
payloadMs.Position = 0;
// Try to decompress (xz is most common for modern RPMs)
var payloadCompressed = CreateTempStream();
try
{
var xzStream = new XZStream(payloadMs);
var decompressed = new MemoryStream();
await xzStream.CopyToAsync(decompressed, ct);
decompressed.Position = 0;
await CopyToWithLimitAsync(rpmStream, payloadCompressed, MaxPayloadCompressedBytes, ct);
payloadCompressed.Position = 0;
var compression = DetectCompression(payloadCompressed);
payloadCompressed.Position = 0;
if (compression == PayloadCompression.None)
{
return payloadCompressed;
}
var decompressed = await DecompressPayloadAsync(payloadCompressed, compression, ct);
payloadCompressed.Dispose();
return decompressed;
}
catch
{
// Try other compression formats or return as-is
payloadMs.Position = 0;
return payloadMs;
payloadCompressed.Dispose();
throw;
}
}
@@ -153,41 +181,45 @@ public sealed class RpmPackageExtractor
{
// RPM header magic: 8D AD E8 01
var headerMagic = new byte[8];
var read = await stream.ReadAsync(headerMagic.AsMemory(0, 8), ct);
if (read != 8)
if (!await ReadExactAsync(stream, headerMagic, ct))
{
return -1;
}
// Header index entries count (4 bytes, big-endian)
var indexCount = (headerMagic[4] << 24) | (headerMagic[5] << 16) | (headerMagic[6] << 8) | headerMagic[7];
// Read data size (4 bytes, big-endian)
var dataSizeBytes = new byte[4];
read = await stream.ReadAsync(dataSizeBytes.AsMemory(0, 4), ct);
if (read != 4)
if (!await ReadExactAsync(stream, dataSizeBytes, ct))
{
return -1;
}
var dataSize = (dataSizeBytes[0] << 24) | (dataSizeBytes[1] << 16) | (dataSizeBytes[2] << 8) | dataSizeBytes[3];
// Skip index entries (16 bytes each) and data
var toSkip = (indexCount * 16) + dataSize;
var toSkip = (indexCount * 16L) + dataSize;
// Align to 8 bytes
var position = stream.Position + toSkip;
var position = 12L + toSkip;
var padding = (8 - (position % 8)) % 8;
toSkip += (int)padding;
toSkip += padding;
var buffer = new byte[toSkip];
read = await stream.ReadAsync(buffer.AsMemory(0, toSkip), ct);
if (read != toSkip)
if (!await SkipBytesAsync(stream, toSkip, ct))
{
return -1;
}
return toSkip;
}
private static bool IsElfBinary(Stream stream)
{
if (stream.Length < 4)
if (!stream.CanRead || !stream.CanSeek)
{
return false;
}
var buffer = new byte[4];
var read = stream.Read(buffer, 0, 4);
@@ -195,9 +227,156 @@ public sealed class RpmPackageExtractor
return read == 4 && buffer.AsSpan().SequenceEqual(ElfMagic);
}
internal static PayloadCompression DetectCompression(Stream stream)
{
if (!stream.CanSeek)
{
return PayloadCompression.None;
}
var originalPosition = stream.Position;
Span<byte> header = stackalloc byte[6];
var read = stream.Read(header);
stream.Position = originalPosition;
if (read >= XzMagic.Length && header[..XzMagic.Length].SequenceEqual(XzMagic))
{
return PayloadCompression.Xz;
}
if (read >= GzipMagic.Length && header[..GzipMagic.Length].SequenceEqual(GzipMagic))
{
return PayloadCompression.Gzip;
}
if (read >= ZstdMagic.Length && header[..ZstdMagic.Length].SequenceEqual(ZstdMagic))
{
return PayloadCompression.Zstd;
}
return PayloadCompression.None;
}
internal static async Task<Stream> DecompressPayloadAsync(
Stream payloadStream,
PayloadCompression compression,
CancellationToken ct)
{
var output = CreateTempStream();
try
{
await using var decompressor = CreateDecompressor(payloadStream, compression);
await CopyToWithLimitAsync(decompressor, output, MaxPayloadUncompressedBytes, ct);
output.Position = 0;
return output;
}
catch
{
output.Dispose();
throw;
}
}
private static Stream CreateDecompressor(Stream payloadStream, PayloadCompression compression)
{
return compression switch
{
PayloadCompression.Xz => new XZStream(payloadStream),
PayloadCompression.Gzip => new GZipStream(payloadStream, CompressionMode.Decompress, leaveOpen: true),
PayloadCompression.Zstd => throw new NotSupportedException("Zstandard payloads are not supported."),
_ => payloadStream
};
}
private static async Task<Stream> BufferEntryAsync(
Stream entryStream,
long size,
CancellationToken ct)
{
var buffered = new MemoryStream(
size > 0 && size <= MaxEntrySizeBytes ? (int)size : 0);
await CopyToWithLimitAsync(entryStream, buffered, MaxEntrySizeBytes, ct);
buffered.Position = 0;
return buffered;
}
private static async Task<long> CopyToWithLimitAsync(
Stream source,
Stream destination,
long maxBytes,
CancellationToken ct)
{
var buffer = new byte[16 * 1024];
long total = 0;
int read;
while ((read = await source.ReadAsync(buffer, ct)) > 0)
{
total += read;
if (total > maxBytes)
{
throw new InvalidOperationException(
$"Payload exceeded limit of {maxBytes} bytes.");
}
await destination.WriteAsync(buffer.AsMemory(0, read), ct);
}
return total;
}
private static async Task<bool> ReadExactAsync(Stream stream, byte[] buffer, CancellationToken ct)
{
var total = 0;
while (total < buffer.Length)
{
var read = await stream.ReadAsync(buffer.AsMemory(total, buffer.Length - total), ct);
if (read == 0)
{
return false;
}
total += read;
}
return true;
}
private static async Task<bool> SkipBytesAsync(Stream stream, long bytes, CancellationToken ct)
{
var buffer = new byte[16 * 1024];
var remaining = bytes;
while (remaining > 0)
{
var toRead = (int)Math.Min(buffer.Length, remaining);
var read = await stream.ReadAsync(buffer.AsMemory(0, toRead), ct);
if (read == 0)
{
return false;
}
remaining -= read;
}
return true;
}
private static FileStream CreateTempStream()
{
var path = Path.GetTempFileName();
return new FileStream(
path,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.None,
16 * 1024,
FileOptions.DeleteOnClose | FileOptions.SequentialScan);
}
}
/// <summary>
/// Information about an extracted binary.
/// </summary>
public sealed record ExtractedBinaryInfo(BinaryIdentity Identity, string FilePath);

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// SrpmChangelogExtractor.cs
// Sprint: SPRINT_20251226_012_BINIDX_backport_handling
// Task: BACKPORT-15 Implement SRPM changelog extraction
// Task: BACKPORT-15 - Implement SRPM changelog extraction
// -----------------------------------------------------------------------------
using Microsoft.Extensions.Logging;
@@ -130,3 +130,4 @@ public sealed class SrpmChangelogExtractor
return _changelogParser.ParseAllEntries(specContent, distro, release, sourcePkg);
}
}

View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
@@ -13,6 +14,10 @@
<PackageReference Include="SharpCompress" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="StellaOps.BinaryIndex.Corpus.Rpm.Tests" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.BinaryIndex.Core\StellaOps.BinaryIndex.Core.csproj" />
<ProjectReference Include="..\StellaOps.BinaryIndex.Corpus\StellaOps.BinaryIndex.Corpus.csproj" />

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0121-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Corpus.Rpm. |
| AUDIT-0121-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Corpus.Rpm. |
| AUDIT-0121-A | TODO | Pending approval for changes. |
| AUDIT-0121-A | DONE | Applied + tests. |

View File

@@ -0,0 +1,11 @@
namespace StellaOps.BinaryIndex.Corpus;
public interface IGuidProvider
{
Guid NewGuid();
}
public sealed class SystemGuidProvider : IGuidProvider
{
public Guid NewGuid() => Guid.NewGuid();
}

View File

@@ -1,4 +1,5 @@
using System.Runtime.CompilerServices;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using StellaOps.BinaryIndex.Core.Models;
namespace StellaOps.BinaryIndex.Corpus;
@@ -17,7 +18,7 @@ public interface IBinaryCorpusConnector
/// <summary>
/// List of supported distro identifiers (e.g., ["debian", "ubuntu"]).
/// </summary>
string[] SupportedDistros { get; }
ImmutableArray<string> SupportedDistros { get; }
/// <summary>
/// Fetches a corpus snapshot for the given query.
@@ -38,34 +39,147 @@ public interface IBinaryCorpusConnector
/// <summary>
/// Query parameters for fetching a corpus snapshot.
/// </summary>
public sealed record CorpusQuery(
string Distro,
string Release,
string Architecture,
string[]? ComponentFilter = null);
public sealed record CorpusQuery : IValidatableObject
{
public CorpusQuery(
string distro,
string release,
string architecture,
IEnumerable<string>? componentFilter = null)
{
Distro = distro;
Release = release;
Architecture = architecture;
ComponentFilter = NormalizeComponentFilter(componentFilter);
}
public string Distro { get; init; }
public string Release { get; init; }
public string Architecture { get; init; }
public ImmutableArray<string> ComponentFilter { get; init; } = ImmutableArray<string>.Empty;
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (string.IsNullOrWhiteSpace(Distro))
{
yield return new ValidationResult(
"Distro must be set.",
new[] { nameof(Distro) });
}
if (string.IsNullOrWhiteSpace(Release))
{
yield return new ValidationResult(
"Release must be set.",
new[] { nameof(Release) });
}
if (string.IsNullOrWhiteSpace(Architecture))
{
yield return new ValidationResult(
"Architecture must be set.",
new[] { nameof(Architecture) });
}
}
private static ImmutableArray<string> NormalizeComponentFilter(IEnumerable<string>? filter)
{
if (filter is null)
{
return ImmutableArray<string>.Empty;
}
var normalized = filter
.Where(component => !string.IsNullOrWhiteSpace(component))
.Select(component => component.Trim())
.Distinct(StringComparer.Ordinal)
.OrderBy(component => component, StringComparer.Ordinal)
.ToImmutableArray();
return normalized;
}
}
/// <summary>
/// Represents a snapshot of a corpus at a specific point in time.
/// </summary>
public sealed record CorpusSnapshot(
Guid Id,
string Distro,
string Release,
string Architecture,
string MetadataDigest,
DateTimeOffset CapturedAt);
public sealed record CorpusSnapshot : IValidatableObject
{
public required Guid Id { get; init; }
public required string Distro { get; init; }
public required string Release { get; init; }
public required string Architecture { get; init; }
public required string MetadataDigest { get; init; }
public required DateTimeOffset CapturedAt { get; init; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (CapturedAt == default)
{
yield return new ValidationResult(
"CapturedAt must be set.",
new[] { nameof(CapturedAt) });
}
if (CapturedAt.Offset != TimeSpan.Zero)
{
yield return new ValidationResult(
"CapturedAt must be in UTC.",
new[] { nameof(CapturedAt) });
}
}
}
/// <summary>
/// Package metadata from repository index.
/// </summary>
public sealed record PackageInfo(
string Name,
string Version,
string SourcePackage,
string Architecture,
string Filename,
long Size,
string Sha256);
public sealed record PackageInfo : IValidatableObject
{
public required string Name { get; init; }
public required string Version { get; init; }
public required string SourcePackage { get; init; }
public required string Architecture { get; init; }
public required string Filename { get; init; }
public long Size { get; init; }
public required string Sha256 { get; init; }
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (!IsValidSha256(Sha256))
{
yield return new ValidationResult(
"Sha256 must be a 64-character hex digest with optional sha256: prefix.",
new[] { nameof(Sha256) });
}
}
private static bool IsValidSha256(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var hex = value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? value["sha256:".Length..]
: value;
if (hex.Length != 64)
{
return false;
}
foreach (var ch in hex)
{
if (!Uri.IsHexDigit(ch))
{
return false;
}
}
return true;
}
}
/// <summary>
/// Binary extracted from a package.

View File

@@ -1,6 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0118-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Corpus. |
| AUDIT-0118-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Corpus. |
| AUDIT-0118-A | TODO | Pending approval for changes. |
| AUDIT-0118-A | DONE | Applied corpus contract fixes + tests. |

View File

@@ -1,8 +1,8 @@
// -----------------------------------------------------------------------------
// BasicBlockFingerprintGenerator.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-06 Implement BasicBlockFingerprintGenerator
// Refactored: DS-033 Use IDisassemblyService for proper disassembly
// Task: FPRINT-06 - Implement BasicBlockFingerprintGenerator
// Refactored: DS-033 - Use IDisassemblyService for proper disassembly
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
@@ -461,3 +461,4 @@ public sealed class BasicBlockFingerprintGenerator : IVulnFingerprintGenerator
return 0.95m;
}
}

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// CombinedFingerprintGenerator.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-09 Implement CombinedFingerprintGenerator (ensemble)
// Task: FPRINT-09 - Implement CombinedFingerprintGenerator (ensemble)
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
@@ -122,7 +122,9 @@ public sealed class CombinedFingerprintGenerator : IVulnFingerprintGenerator
ms.WriteByte(0x00); // Marker: no string refs
}
// Final hash to fixed size (48 bytes)
// Final hash to fixed size (48 bytes). This is not a pure hash of inputs;
// we append the basic block hash for fast lookup and keep the hash prefix
// deterministic for stability across runs.
var combined = SHA256.HashData(ms.ToArray());
var result = new byte[48];
Array.Copy(combined, result, 32);
@@ -180,3 +182,4 @@ public sealed class CombinedFingerprintGenerator : IVulnFingerprintGenerator
return combined;
}
}

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// ControlFlowGraphFingerprintGenerator.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-07 Implement ControlFlowGraphFingerprintGenerator
// Task: FPRINT-07 - Implement ControlFlowGraphFingerprintGenerator
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
@@ -430,3 +430,4 @@ public sealed class ControlFlowGraphFingerprintGenerator : IVulnFingerprintGener
return 0.85m;
}
}

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// IVulnFingerprintGenerator.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-05 Design IVulnFingerprintGenerator interface
// Task: FPRINT-05 - Design IVulnFingerprintGenerator interface
// -----------------------------------------------------------------------------
using StellaOps.BinaryIndex.Fingerprints.Models;
@@ -111,3 +111,4 @@ public interface IVulnFingerprintGenerator
/// <returns>True if the generator can process this input.</returns>
bool CanProcess(FingerprintInput input);
}

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// StringRefsFingerprintGenerator.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-08 Implement StringRefsFingerprintGenerator
// Task: FPRINT-08 - Implement StringRefsFingerprintGenerator
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
@@ -279,3 +279,4 @@ public sealed class StringRefsFingerprintGenerator : IVulnFingerprintGenerator
return 0.6m;
}
}

View File

@@ -0,0 +1,12 @@
namespace StellaOps.BinaryIndex.Fingerprints;
public interface IGuidProvider
{
Guid NewGuid();
}
public sealed class SystemGuidProvider : IGuidProvider
{
public Guid NewGuid() => Guid.NewGuid();
}

View File

@@ -29,7 +29,7 @@ public interface IFingerprintRepository
Task<ImmutableArray<VulnFingerprint>> SearchByHashAsync(
byte[] hash,
FingerprintAlgorithm algorithm,
string architecture,
string? architecture,
CancellationToken ct = default);
/// <summary>
@@ -64,3 +64,4 @@ public interface IFingerprintMatchRepository
ReachabilityStatus status,
CancellationToken ct = default);
}

View File

@@ -1,9 +1,10 @@
// -----------------------------------------------------------------------------
// FingerprintMatcher.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-13 Implement similarity matching with configurable threshold
// Task: FPRINT-13 - Implement similarity matching with configurable threshold
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using StellaOps.BinaryIndex.Fingerprints.Models;
@@ -39,17 +40,43 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
fingerprint.Length,
options.MinSimilarity);
// Determine algorithm from fingerprint size
var algorithm = InferAlgorithm(fingerprint);
var architectureFilter = string.IsNullOrWhiteSpace(options.Architecture)
? null
: options.Architecture;
var allowedAlgorithms = GetAlgorithmsForLength(fingerprint.Length);
var candidateAlgorithms = NormalizeAlgorithms(options.Algorithms, allowedAlgorithms);
// Get candidate fingerprints from repository
var candidates = await _repository.SearchByHashAsync(
fingerprint,
algorithm,
options.Architecture ?? "",
ct);
if (candidateAlgorithms.Length == 0)
{
_logger.LogDebug(
"No matching algorithms for fingerprint length {Length}",
fingerprint.Length);
return new FingerprintMatchResult
{
IsMatch = false,
Similarity = 0,
Confidence = 0,
Details = new MatchDetails
{
MatchingAlgorithm = InferAlgorithm(fingerprint),
CandidatesEvaluated = 0,
MatchTimeMs = sw.ElapsedMilliseconds
}
};
}
if (candidates.Length == 0)
var candidates = new List<VulnFingerprint>();
foreach (var algorithm in candidateAlgorithms)
{
var results = await _repository.SearchByHashAsync(
fingerprint,
algorithm,
architectureFilter,
ct);
candidates.AddRange(results);
}
if (candidates.Count == 0)
{
_logger.LogDebug("No candidates found for fingerprint");
return new FingerprintMatchResult
@@ -59,7 +86,7 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
Confidence = 0,
Details = new MatchDetails
{
MatchingAlgorithm = algorithm,
MatchingAlgorithm = candidateAlgorithms[0],
CandidatesEvaluated = 0,
MatchTimeMs = sw.ElapsedMilliseconds
}
@@ -79,6 +106,7 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
foreach (var candidate in filteredCandidates)
{
var algorithm = candidate.Algorithm;
var similarity = CalculateSimilarity(fingerprint, candidate.FingerprintHash, algorithm);
if (similarity > bestSimilarity)
@@ -101,6 +129,13 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
CfgSimilarity = CalculateCfgSimilarity(fingerprint, candidate.FingerprintHash)
};
}
else if (algorithm == FingerprintAlgorithm.StringRefs)
{
bestDetails = bestDetails with
{
StringRefsSimilarity = CalculateStringRefsSimilarity(fingerprint, candidate.FingerprintHash)
};
}
}
}
@@ -118,7 +153,12 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
Similarity = bestSimilarity,
MatchedFingerprint = isMatch ? bestMatch : null,
Confidence = isMatch ? CalculateMatchConfidence(bestSimilarity, bestMatch) : 0,
Details = bestDetails
Details = bestDetails ?? new MatchDetails
{
MatchingAlgorithm = candidateAlgorithms[0],
CandidatesEvaluated = filteredCandidates.Count,
MatchTimeMs = sw.ElapsedMilliseconds
}
};
}
@@ -171,6 +211,34 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
};
}
private static ImmutableArray<FingerprintAlgorithm> GetAlgorithmsForLength(int length)
{
return length switch
{
16 => ImmutableArray.Create(FingerprintAlgorithm.BasicBlock, FingerprintAlgorithm.StringRefs),
32 => ImmutableArray.Create(FingerprintAlgorithm.ControlFlowGraph),
48 => ImmutableArray.Create(FingerprintAlgorithm.Combined),
_ => ImmutableArray.Create(FingerprintAlgorithm.BasicBlock)
};
}
private static ImmutableArray<FingerprintAlgorithm> NormalizeAlgorithms(
ImmutableArray<FingerprintAlgorithm> requested,
ImmutableArray<FingerprintAlgorithm> allowed)
{
if (requested.IsDefaultOrEmpty)
{
return allowed;
}
var filtered = requested
.Distinct()
.Where(allowed.Contains)
.ToImmutableArray();
return filtered;
}
/// <summary>
/// Calculates similarity using TLSH-like algorithm for basic blocks.
/// </summary>
@@ -306,3 +374,4 @@ public sealed class FingerprintMatcher : IFingerprintMatcher
return baseConfidence;
}
}

View File

@@ -1,9 +1,10 @@
// -----------------------------------------------------------------------------
// IFingerprintMatcher.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-12 Implement IFingerprintMatcher interface
// Task: FPRINT-12 - Implement IFingerprintMatcher interface
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using StellaOps.BinaryIndex.Fingerprints.Models;
namespace StellaOps.BinaryIndex.Fingerprints.Matching;
@@ -64,8 +65,8 @@ public sealed record MatchOptions
/// <summary>Maximum candidates to evaluate. Default 100.</summary>
public int MaxCandidates { get; init; } = 100;
/// <summary>Algorithms to use for matching. Null means all.</summary>
public FingerprintAlgorithm[]? Algorithms { get; init; }
/// <summary>Algorithms to use for matching. Empty means all.</summary>
public ImmutableArray<FingerprintAlgorithm> Algorithms { get; init; } = ImmutableArray<FingerprintAlgorithm>.Empty;
/// <summary>Whether to require validation of matched fingerprint.</summary>
public bool RequireValidated { get; init; }
@@ -104,3 +105,4 @@ public interface IFingerprintMatcher
/// </summary>
decimal CalculateSimilarity(byte[] fingerprint1, byte[] fingerprint2, FingerprintAlgorithm algorithm);
}

View File

@@ -1,3 +1,5 @@
using System.Collections.Immutable;
namespace StellaOps.BinaryIndex.Fingerprints.Models;
/// <summary>
@@ -137,7 +139,7 @@ public sealed record FingerprintMatch
public decimal? Similarity { get; init; }
/// <summary>Associated advisory IDs (CVEs, etc.)</summary>
public string[]? AdvisoryIds { get; init; }
public ImmutableArray<string> AdvisoryIds { get; init; } = ImmutableArray<string>.Empty;
/// <summary>Reachability status</summary>
public ReachabilityStatus? ReachabilityStatus { get; init; }
@@ -178,3 +180,4 @@ public enum ReachabilityStatus
/// <summary>Partial reachability</summary>
Partial
}

View File

@@ -1,8 +1,8 @@
// -----------------------------------------------------------------------------
// ReferenceBuildPipeline.cs
// Sprint: SPRINT_20251226_013_BINIDX_fingerprint_factory
// Task: FPRINT-10 Create reference build generation pipeline
// Task: FPRINT-11 Implement vulnerable/fixed binary pair builder
// Task: FPRINT-10 - Create reference build generation pipeline
// Task: FPRINT-11 - Implement vulnerable/fixed binary pair builder
// -----------------------------------------------------------------------------
using Microsoft.Extensions.Logging;
@@ -117,17 +117,26 @@ public sealed class ReferenceBuildPipeline
private readonly IFingerprintBlobStorage _storage;
private readonly IFingerprintRepository _repository;
private readonly CombinedFingerprintGenerator _fingerprintGenerator;
private readonly IReferenceBuildExecutor _buildExecutor;
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
public ReferenceBuildPipeline(
ILogger<ReferenceBuildPipeline> logger,
IFingerprintBlobStorage storage,
IFingerprintRepository repository,
CombinedFingerprintGenerator fingerprintGenerator)
CombinedFingerprintGenerator fingerprintGenerator,
IReferenceBuildExecutor? buildExecutor = null,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_logger = logger;
_storage = storage;
_repository = repository;
_fingerprintGenerator = fingerprintGenerator;
_buildExecutor = buildExecutor ?? new ReferenceBuildExecutor(logger);
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new SystemGuidProvider();
}
/// <summary>
@@ -145,7 +154,7 @@ public sealed class ReferenceBuildPipeline
try
{
// Step 1: Clone and build vulnerable version
var vulnArtifacts = await BuildVersionAsync(request, isVulnerable: true, ct);
var vulnArtifacts = await _buildExecutor.BuildVersionAsync(request, isVulnerable: true, ct);
if (vulnArtifacts.Count == 0)
{
return new ReferenceBuildResult
@@ -156,7 +165,7 @@ public sealed class ReferenceBuildPipeline
}
// Step 2: Clone and build fixed version
var fixedArtifacts = await BuildVersionAsync(request, isVulnerable: false, ct);
var fixedArtifacts = await _buildExecutor.BuildVersionAsync(request, isVulnerable: false, ct);
if (fixedArtifacts.Count == 0)
{
return new ReferenceBuildResult
@@ -167,8 +176,22 @@ public sealed class ReferenceBuildPipeline
}
// Step 3: Extract functions from both versions
var vulnFunctions = await ExtractFunctionsAsync(vulnArtifacts, request.TargetFunctions, ct);
var fixedFunctions = await ExtractFunctionsAsync(fixedArtifacts, request.TargetFunctions, ct);
var vulnFunctions = await _buildExecutor.ExtractFunctionsAsync(
vulnArtifacts,
request.TargetFunctions,
ct);
var fixedFunctions = await _buildExecutor.ExtractFunctionsAsync(
fixedArtifacts,
request.TargetFunctions,
ct);
if (vulnFunctions.Count == 0 || fixedFunctions.Count == 0)
{
return new ReferenceBuildResult
{
Success = false,
Error = "No functions extracted from reference builds"
};
}
// Step 4: Find differential fingerprints (what changed)
var fingerprints = await GenerateDifferentialFingerprintsAsync(
@@ -211,80 +234,17 @@ public sealed class ReferenceBuildPipeline
}
}
/// <summary>
/// Builds a specific version (vulnerable or fixed).
/// </summary>
private async Task<List<BuildArtifact>> BuildVersionAsync(
ReferenceBuildRequest request,
bool isVulnerable,
CancellationToken ct)
{
var version = isVulnerable ? request.VulnerableRef : request.FixedRef;
_logger.LogDebug(
"Building {Type} version at {Ref}",
isVulnerable ? "vulnerable" : "fixed",
version);
// NOTE: Actual implementation would:
// 1. Clone repo to sandboxed environment
// 2. Checkout the specific ref
// 3. Run build command
// 4. Extract built binaries
//
// This is a placeholder that returns empty for now.
// Production implementation would use containers or VMs for sandboxing.
await Task.CompletedTask;
// Placeholder: return empty list
// Real impl would return built artifacts
return [];
}
/// <summary>
/// Extracts functions from build artifacts.
/// </summary>
private async Task<List<ExtractedFunction>> ExtractFunctionsAsync(
List<BuildArtifact> artifacts,
string[]? targetFunctions,
CancellationToken ct)
{
var functions = new List<ExtractedFunction>();
foreach (var artifact in artifacts)
{
ct.ThrowIfCancellationRequested();
// NOTE: Real implementation would:
// 1. Parse ELF/PE headers
// 2. Find symbol table
// 3. Extract function boundaries
// 4. Extract code bytes for each function
//
// This is a placeholder.
_logger.LogDebug(
"Extracting functions from {Path} ({Size} bytes)",
artifact.Path,
artifact.Content.Length);
// Placeholder: would use ELF parser
}
await Task.CompletedTask;
return functions;
}
/// <summary>
/// Generates differential fingerprints by comparing vulnerable and fixed versions.
/// </summary>
private async Task<VulnFingerprint[]> GenerateDifferentialFingerprintsAsync(
ReferenceBuildRequest request,
List<ExtractedFunction> vulnFunctions,
List<ExtractedFunction> fixedFunctions,
IReadOnlyList<ExtractedFunction> vulnFunctions,
IReadOnlyList<ExtractedFunction> fixedFunctions,
CancellationToken ct)
{
var fingerprints = new List<VulnFingerprint>();
var now = _timeProvider.GetUtcNow();
// Find functions that changed between versions
var changedFunctions = FindChangedFunctions(vulnFunctions, fixedFunctions);
@@ -319,7 +279,7 @@ public sealed class ReferenceBuildPipeline
fingerprints.Add(new VulnFingerprint
{
Id = Guid.NewGuid(),
Id = _guidProvider.NewGuid(),
CveId = request.CveId,
Component = request.Component,
Algorithm = output.Algorithm,
@@ -332,7 +292,7 @@ public sealed class ReferenceBuildPipeline
Confidence = output.Confidence,
VulnBuildRef = request.VulnerableRef,
FixedBuildRef = request.FixedRef,
IndexedAt = DateTimeOffset.UtcNow
IndexedAt = now
});
}
@@ -343,8 +303,8 @@ public sealed class ReferenceBuildPipeline
/// Finds functions that changed between vulnerable and fixed versions.
/// </summary>
private static List<(ExtractedFunction vuln, ExtractedFunction? fix)> FindChangedFunctions(
List<ExtractedFunction> vulnFunctions,
List<ExtractedFunction> fixedFunctions)
IReadOnlyList<ExtractedFunction> vulnFunctions,
IReadOnlyList<ExtractedFunction> fixedFunctions)
{
var results = new List<(ExtractedFunction, ExtractedFunction?)>();
@@ -368,7 +328,7 @@ public sealed class ReferenceBuildPipeline
/// </summary>
private async Task<string> StoreReferenceBuildAsync(
string cveId,
List<BuildArtifact> artifacts,
IReadOnlyList<BuildArtifact> artifacts,
string buildType,
CancellationToken ct)
{
@@ -388,3 +348,80 @@ public sealed class ReferenceBuildPipeline
return storagePath;
}
}
public interface IReferenceBuildExecutor
{
Task<IReadOnlyList<BuildArtifact>> BuildVersionAsync(
ReferenceBuildRequest request,
bool isVulnerable,
CancellationToken ct = default);
Task<IReadOnlyList<ExtractedFunction>> ExtractFunctionsAsync(
IReadOnlyList<BuildArtifact> artifacts,
string[]? targetFunctions,
CancellationToken ct = default);
}
public sealed class ReferenceBuildExecutor : IReferenceBuildExecutor
{
private readonly ILogger _logger;
public ReferenceBuildExecutor(ILogger logger)
{
_logger = logger;
}
public async Task<IReadOnlyList<BuildArtifact>> BuildVersionAsync(
ReferenceBuildRequest request,
bool isVulnerable,
CancellationToken ct = default)
{
var version = isVulnerable ? request.VulnerableRef : request.FixedRef;
_logger.LogDebug(
"Building {Type} version at {Ref}",
isVulnerable ? "vulnerable" : "fixed",
version);
// NOTE: Actual implementation would:
// 1. Clone repo to sandboxed environment
// 2. Checkout the specific ref
// 3. Run build command
// 4. Extract built binaries
//
// This is a placeholder that returns empty for now.
// Production implementation would use containers or VMs for sandboxing.
await Task.CompletedTask;
return [];
}
public async Task<IReadOnlyList<ExtractedFunction>> ExtractFunctionsAsync(
IReadOnlyList<BuildArtifact> artifacts,
string[]? targetFunctions,
CancellationToken ct = default)
{
var functions = new List<ExtractedFunction>();
foreach (var artifact in artifacts)
{
ct.ThrowIfCancellationRequested();
// NOTE: Real implementation would:
// 1. Parse ELF/PE headers
// 2. Find symbol table
// 3. Extract function boundaries
// 4. Extract code bytes for each function
//
// This is a placeholder.
_logger.LogDebug(
"Extracting functions from {Path} ({Size} bytes)",
artifact.Path,
artifact.Content.Length);
}
await Task.CompletedTask;
return functions;
}
}

View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

View File

@@ -5,8 +5,8 @@ namespace StellaOps.BinaryIndex.Fingerprints.Storage;
/// <summary>
/// Blob storage implementation for fingerprints.
/// NOTE: This is a placeholder implementation showing the structure.
/// Production implementation would use RustFS or S3-compatible storage.
/// NOTE: This is a placeholder implementation showing deterministic storage paths.
/// Production implementation would use RustFS or S3-compatible storage with atomic writes.
/// </summary>
public sealed class FingerprintBlobStorage : IFingerprintBlobStorage
{
@@ -101,3 +101,4 @@ public sealed class FingerprintBlobStorage : IFingerprintBlobStorage
return null;
}
}

View File

@@ -47,3 +47,4 @@ public interface IFingerprintBlobStorage
string storagePath,
CancellationToken ct = default);
}

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0122-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Fingerprints. |
| AUDIT-0122-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Fingerprints. |
| AUDIT-0122-A | TODO | Pending approval for changes. |
| AUDIT-0122-A | DOING | Pending approval for changes. |

View File

@@ -16,15 +16,26 @@ namespace StellaOps.BinaryIndex.FixIndex.Parsers;
/// </remarks>
public sealed partial class AlpineSecfixesParser : ISecfixesParser
{
private readonly FixIndexParserOptions _options;
private readonly TimeProvider _timeProvider;
[GeneratedRegex(@"^#\s*secfixes:\s*$", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex SecfixesPatternRegex();
[GeneratedRegex(@"^#\s+(\d+\.\d+[^:]*):$", RegexOptions.Compiled)]
[GeneratedRegex(@"^#\s+([vV]?\d[^\s:]*):\s*$", RegexOptions.Compiled)]
private static partial Regex VersionPatternRegex();
[GeneratedRegex(@"^#\s+-\s+(CVE-\d{4}-\d{4,7})$", RegexOptions.Compiled)]
[GeneratedRegex(@"^#\s+-\s+(CVE-\d{4}-\d{4,7})(?:\s|$)", RegexOptions.Compiled)]
private static partial Regex CvePatternRegex();
public AlpineSecfixesParser(
FixIndexParserOptions? options = null,
TimeProvider? timeProvider = null)
{
_options = options ?? FixIndexParserOptions.Default;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Parses APKBUILD secfixes section for version-to-CVE mappings.
/// </summary>
@@ -37,6 +48,10 @@ public sealed partial class AlpineSecfixesParser : ISecfixesParser
if (string.IsNullOrWhiteSpace(apkbuild))
yield break;
var normalizedDistro = FixIndexParserHelpers.NormalizeKey(distro, _options.NormalizeDistroRelease);
var normalizedRelease = FixIndexParserHelpers.NormalizeKey(release, _options.NormalizeDistroRelease);
var now = _timeProvider.GetUtcNow();
// Normalize line endings to handle both Unix and Windows formats
var lines = apkbuild.ReplaceLineEndings("\n").Split('\n');
var inSecfixes = false;
@@ -72,21 +87,21 @@ public sealed partial class AlpineSecfixesParser : ISecfixesParser
{
yield return new FixEvidence
{
Distro = distro,
Release = release,
Distro = normalizedDistro,
Release = normalizedRelease,
SourcePkg = sourcePkg,
CveId = cveMatch.Groups[1].Value,
State = FixState.Fixed,
FixedVersion = currentVersion,
Method = FixMethod.SecurityFeed, // APKBUILD is authoritative
Confidence = 0.95m,
Confidence = _options.SecurityFeedConfidence,
Evidence = new SecurityFeedEvidence
{
FeedId = "alpine-secfixes",
EntryId = $"{sourcePkg}/{currentVersion}",
PublishedAt = DateTimeOffset.UtcNow
PublishedAt = now
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
}
}

View File

@@ -9,6 +9,9 @@ namespace StellaOps.BinaryIndex.FixIndex.Parsers;
/// </summary>
public sealed partial class DebianChangelogParser : IChangelogParser
{
private readonly FixIndexParserOptions _options;
private readonly TimeProvider _timeProvider;
[GeneratedRegex(@"\bCVE-\d{4}-\d{4,7}\b", RegexOptions.Compiled)]
private static partial Regex CvePatternRegex();
@@ -18,6 +21,14 @@ public sealed partial class DebianChangelogParser : IChangelogParser
[GeneratedRegex(@"^\s+--\s+", RegexOptions.Compiled)]
private static partial Regex TrailerPatternRegex();
public DebianChangelogParser(
FixIndexParserOptions? options = null,
TimeProvider? timeProvider = null)
{
_options = options ?? FixIndexParserOptions.Default;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Parses the top entry of a Debian changelog for CVE mentions.
/// </summary>
@@ -30,6 +41,10 @@ public sealed partial class DebianChangelogParser : IChangelogParser
if (string.IsNullOrWhiteSpace(changelog))
yield break;
var normalizedDistro = FixIndexParserHelpers.NormalizeKey(distro, _options.NormalizeDistroRelease);
var normalizedRelease = FixIndexParserHelpers.NormalizeKey(release, _options.NormalizeDistroRelease);
var now = _timeProvider.GetUtcNow();
// Normalize line endings to handle both Unix and Windows formats
var lines = changelog.ReplaceLineEndings("\n").Split('\n');
if (lines.Length == 0)
@@ -61,22 +76,22 @@ public sealed partial class DebianChangelogParser : IChangelogParser
{
yield return new FixEvidence
{
Distro = distro,
Release = release,
Distro = normalizedDistro,
Release = normalizedRelease,
SourcePkg = sourcePkg,
CveId = cve,
State = FixState.Fixed,
FixedVersion = version,
Method = FixMethod.Changelog,
Confidence = 0.80m,
Confidence = _options.DebianChangelogConfidence,
Evidence = new ChangelogEvidence
{
File = "debian/changelog",
Version = version,
Excerpt = entryText.Length > 2000 ? entryText[..2000] : entryText,
Excerpt = FixIndexParserHelpers.TruncateToWholeLines(entryText, _options.ChangelogExcerptMaxLength),
LineNumber = null // Could be enhanced to track line number
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
}
}

View File

@@ -0,0 +1,99 @@
using System.Text;
namespace StellaOps.BinaryIndex.FixIndex.Parsers;
public sealed record FixIndexParserOptions
{
public static FixIndexParserOptions Default { get; } = new();
public int ChangelogExcerptMaxLength { get; init; } = 2000;
public int PatchHeaderExcerptMaxLength { get; init; } = 1200;
public int PatchHeaderMaxLines { get; init; } = 80;
public int PatchHeaderMaxChars { get; init; } = 16_384;
public decimal DebianChangelogConfidence { get; init; } = 0.80m;
public decimal RpmChangelogConfidence { get; init; } = 0.75m;
public decimal PatchHeaderConfidence { get; init; } = 0.87m;
public decimal SecurityFeedConfidence { get; init; } = 0.95m;
public bool NormalizeDistroRelease { get; init; } = true;
}
internal static class FixIndexParserHelpers
{
public static string NormalizeKey(string value, bool normalize)
{
if (!normalize)
return value;
return value.Trim().ToLowerInvariant();
}
public static string ReadHeader(string content, int maxLines, int maxChars)
{
if (string.IsNullOrEmpty(content))
return string.Empty;
var lines = content.ReplaceLineEndings("\n").Split('\n');
var builder = new StringBuilder();
var lineCount = 0;
foreach (var line in lines)
{
if (lineCount >= maxLines)
break;
var projectedLength = builder.Length + line.Length + (builder.Length > 0 ? 1 : 0);
if (projectedLength > maxChars)
break;
if (builder.Length > 0)
builder.Append('\n');
builder.Append(line);
lineCount++;
}
return builder.ToString();
}
public static string TruncateToWholeLines(string text, int maxLength)
{
if (text.Length <= maxLength)
return text;
var lines = text.ReplaceLineEndings("\n").Split('\n');
var builder = new StringBuilder();
foreach (var line in lines)
{
var projectedLength = builder.Length + line.Length + (builder.Length > 0 ? 1 : 0);
if (projectedLength > maxLength)
break;
if (builder.Length > 0)
builder.Append('\n');
builder.Append(line);
}
if (builder.Length > 0)
return builder.ToString();
return text[..maxLength];
}
public static bool IsTextSafe(string text)
{
foreach (var ch in text)
{
if (ch == '\0' || ch == '\uFFFD')
return false;
if (char.IsControl(ch) && ch != '\n' && ch != '\r' && ch != '\t')
return false;
}
return true;
}
}

View File

@@ -9,9 +9,20 @@ namespace StellaOps.BinaryIndex.FixIndex.Parsers;
/// </summary>
public sealed partial class PatchHeaderParser : IPatchParser
{
private readonly FixIndexParserOptions _options;
private readonly TimeProvider _timeProvider;
[GeneratedRegex(@"\bCVE-\d{4}-\d{4,7}\b", RegexOptions.Compiled)]
private static partial Regex CvePatternRegex();
public PatchHeaderParser(
FixIndexParserOptions? options = null,
TimeProvider? timeProvider = null)
{
_options = options ?? FixIndexParserOptions.Default;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Parses patches for CVE mentions in headers.
/// </summary>
@@ -22,12 +33,19 @@ public sealed partial class PatchHeaderParser : IPatchParser
string sourcePkg,
string version)
{
var normalizedDistro = FixIndexParserHelpers.NormalizeKey(distro, _options.NormalizeDistroRelease);
var normalizedRelease = FixIndexParserHelpers.NormalizeKey(release, _options.NormalizeDistroRelease);
var now = _timeProvider.GetUtcNow();
foreach (var (path, content, sha256) in patches)
{
// Read first 80 lines as header (typical patch header size)
// Normalize line endings to handle both Unix and Windows formats
var headerLines = content.ReplaceLineEndings("\n").Split('\n').Take(80);
var header = string.Join('\n', headerLines);
var header = FixIndexParserHelpers.ReadHeader(
content,
_options.PatchHeaderMaxLines,
_options.PatchHeaderMaxChars);
if (!FixIndexParserHelpers.IsTextSafe(header))
continue;
// Also check filename for CVE (e.g., "CVE-2024-1234.patch")
var searchText = header + "\n" + Path.GetFileName(path);
@@ -40,21 +58,23 @@ public sealed partial class PatchHeaderParser : IPatchParser
{
yield return new FixEvidence
{
Distro = distro,
Release = release,
Distro = normalizedDistro,
Release = normalizedRelease,
SourcePkg = sourcePkg,
CveId = cve,
State = FixState.Fixed,
FixedVersion = version,
Method = FixMethod.PatchHeader,
Confidence = 0.87m,
Confidence = _options.PatchHeaderConfidence,
Evidence = new PatchHeaderEvidence
{
PatchPath = path,
PatchSha256 = sha256,
HeaderExcerpt = header.Length > 1200 ? header[..1200] : header
HeaderExcerpt = FixIndexParserHelpers.TruncateToWholeLines(
header,
_options.PatchHeaderExcerptMaxLength)
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
}
}

View File

@@ -15,6 +15,9 @@ namespace StellaOps.BinaryIndex.FixIndex.Parsers;
/// </remarks>
public sealed partial class RpmChangelogParser : IChangelogParser
{
private readonly FixIndexParserOptions _options;
private readonly TimeProvider _timeProvider;
[GeneratedRegex(@"\bCVE-\d{4}-\d{4,7}\b", RegexOptions.Compiled)]
private static partial Regex CvePatternRegex();
@@ -27,6 +30,14 @@ public sealed partial class RpmChangelogParser : IChangelogParser
[GeneratedRegex(@"^%\w+", RegexOptions.Compiled)]
private static partial Regex SectionStartPatternRegex();
public RpmChangelogParser(
FixIndexParserOptions? options = null,
TimeProvider? timeProvider = null)
{
_options = options ?? FixIndexParserOptions.Default;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <summary>
/// Parses the top entry of an RPM spec changelog for CVE mentions.
/// </summary>
@@ -39,6 +50,10 @@ public sealed partial class RpmChangelogParser : IChangelogParser
if (string.IsNullOrWhiteSpace(specContent))
yield break;
var normalizedDistro = FixIndexParserHelpers.NormalizeKey(distro, _options.NormalizeDistroRelease);
var normalizedRelease = FixIndexParserHelpers.NormalizeKey(release, _options.NormalizeDistroRelease);
var now = _timeProvider.GetUtcNow();
// Normalize line endings to handle both Unix and Windows formats
var lines = specContent.ReplaceLineEndings("\n").Split('\n');
var inChangelog = false;
@@ -97,22 +112,22 @@ public sealed partial class RpmChangelogParser : IChangelogParser
{
yield return new FixEvidence
{
Distro = distro,
Release = release,
Distro = normalizedDistro,
Release = normalizedRelease,
SourcePkg = sourcePkg,
CveId = cve,
State = FixState.Fixed,
FixedVersion = currentVersion,
Method = FixMethod.Changelog,
Confidence = 0.75m, // RPM changelogs are less structured than Debian
Confidence = _options.RpmChangelogConfidence, // RPM changelogs are less structured than Debian
Evidence = new ChangelogEvidence
{
File = "*.spec",
Version = currentVersion,
Excerpt = entryText.Length > 2000 ? entryText[..2000] : entryText,
Excerpt = FixIndexParserHelpers.TruncateToWholeLines(entryText, _options.ChangelogExcerptMaxLength),
LineNumber = null
},
CreatedAt = DateTimeOffset.UtcNow
CreatedAt = now
};
}
}
@@ -129,6 +144,10 @@ public sealed partial class RpmChangelogParser : IChangelogParser
if (string.IsNullOrWhiteSpace(specContent))
yield break;
var normalizedDistro = FixIndexParserHelpers.NormalizeKey(distro, _options.NormalizeDistroRelease);
var normalizedRelease = FixIndexParserHelpers.NormalizeKey(release, _options.NormalizeDistroRelease);
var now = _timeProvider.GetUtcNow();
// Normalize line endings to handle both Unix and Windows formats
var lines = specContent.ReplaceLineEndings("\n").Split('\n');
var inChangelog = false;
@@ -153,7 +172,7 @@ public sealed partial class RpmChangelogParser : IChangelogParser
// Process last entry
if (currentVersion != null && currentEntry.Count > 0)
{
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, sourcePkg))
yield return fix;
}
break;
@@ -166,7 +185,7 @@ public sealed partial class RpmChangelogParser : IChangelogParser
// Process previous entry
if (currentVersion != null && currentEntry.Count > 0)
{
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, sourcePkg))
yield return fix;
}
@@ -184,44 +203,42 @@ public sealed partial class RpmChangelogParser : IChangelogParser
// Process final entry if exists
if (currentVersion != null && currentEntry.Count > 0)
{
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, distro, release, sourcePkg))
foreach (var fix in ExtractCvesFromEntry(currentEntry, currentVersion, sourcePkg))
yield return fix;
}
}
private IEnumerable<FixEvidence> ExtractCvesFromEntry(
List<string> entryLines,
string version,
string distro,
string release,
string sourcePkg)
{
var entryText = string.Join('\n', entryLines);
var cves = CvePatternRegex().Matches(entryText)
.Select(m => m.Value)
.Distinct();
foreach (var cve in cves)
IEnumerable<FixEvidence> ExtractCvesFromEntry(
List<string> entryLines,
string version,
string sourcePkgValue)
{
yield return new FixEvidence
var entryText = string.Join('\n', entryLines);
var cves = CvePatternRegex().Matches(entryText)
.Select(m => m.Value)
.Distinct();
foreach (var cve in cves)
{
Distro = distro,
Release = release,
SourcePkg = sourcePkg,
CveId = cve,
State = FixState.Fixed,
FixedVersion = version,
Method = FixMethod.Changelog,
Confidence = 0.75m,
Evidence = new ChangelogEvidence
yield return new FixEvidence
{
File = "*.spec",
Version = version,
Excerpt = entryText.Length > 2000 ? entryText[..2000] : entryText,
LineNumber = null
},
CreatedAt = DateTimeOffset.UtcNow
};
Distro = normalizedDistro,
Release = normalizedRelease,
SourcePkg = sourcePkgValue,
CveId = cve,
State = FixState.Fixed,
FixedVersion = version,
Method = FixMethod.Changelog,
Confidence = _options.RpmChangelogConfidence,
Evidence = new ChangelogEvidence
{
File = "*.spec",
Version = version,
Excerpt = FixIndexParserHelpers.TruncateToWholeLines(entryText, _options.ChangelogExcerptMaxLength),
LineNumber = null
},
CreatedAt = now
};
}
}
}
}

View File

@@ -17,12 +17,26 @@ public sealed class FixIndexBuilder : IFixIndexBuilder
private readonly RpmChangelogParser _rpmParser;
public FixIndexBuilder(ILogger<FixIndexBuilder> logger)
: this(logger, null, null, null, null, null, null)
{
}
public FixIndexBuilder(
ILogger<FixIndexBuilder> logger,
FixIndexParserOptions? options,
TimeProvider? timeProvider,
DebianChangelogParser? debianParser,
PatchHeaderParser? patchParser,
AlpineSecfixesParser? alpineParser,
RpmChangelogParser? rpmParser)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_debianParser = new DebianChangelogParser();
_patchParser = new PatchHeaderParser();
_alpineParser = new AlpineSecfixesParser();
_rpmParser = new RpmChangelogParser();
var resolvedOptions = options ?? FixIndexParserOptions.Default;
var resolvedTimeProvider = timeProvider ?? TimeProvider.System;
_debianParser = debianParser ?? new DebianChangelogParser(resolvedOptions, resolvedTimeProvider);
_patchParser = patchParser ?? new PatchHeaderParser(resolvedOptions, resolvedTimeProvider);
_alpineParser = alpineParser ?? new AlpineSecfixesParser(resolvedOptions, resolvedTimeProvider);
_rpmParser = rpmParser ?? new RpmChangelogParser(resolvedOptions, resolvedTimeProvider);
}
/// <inheritdoc />

View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0124-M | DONE | Maintainability audit for StellaOps.BinaryIndex.FixIndex. |
| AUDIT-0124-T | DONE | Test coverage audit for StellaOps.BinaryIndex.FixIndex. |
| AUDIT-0124-A | TODO | Pending approval for changes. |
| AUDIT-0124-A | DONE | Pending approval for changes. |

View File

@@ -1,4 +1,5 @@
using Npgsql;
using NpgsqlTypes;
using StellaOps.BinaryIndex.Core.Services;
namespace StellaOps.BinaryIndex.Persistence;
@@ -26,11 +27,24 @@ public sealed class BinaryIndexDbContext
{
var connection = await _dataSource.OpenConnectionAsync(ct);
var tenantId = GetTenantId();
// Set tenant context for RLS
await using var cmd = connection.CreateCommand();
cmd.CommandText = $"SET app.tenant_id = '{_tenantContext.TenantId}'";
cmd.CommandText = "SELECT set_config('app.tenant_id', @tenantId, false)";
cmd.Parameters.AddWithValue("tenantId", NpgsqlDbType.Text, tenantId.ToString());
await cmd.ExecuteNonQueryAsync(ct);
return connection;
}
private Guid GetTenantId()
{
if (!Guid.TryParse(_tenantContext.TenantId, out var tenantId))
{
throw new InvalidOperationException($"Invalid tenant ID format: '{_tenantContext.TenantId}'.");
}
return tenantId;
}
}

View File

@@ -1,5 +1,8 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Npgsql;
using NpgsqlTypes;
namespace StellaOps.BinaryIndex.Persistence;
@@ -24,14 +27,13 @@ public sealed class BinaryIndexMigrationRunner
/// </summary>
public async Task MigrateAsync(CancellationToken ct = default)
{
const string lockKey = "binaries_schema_migration";
var lockHash = unchecked((int)lockKey.GetHashCode());
await using var connection = await _dataSource.OpenConnectionAsync(ct);
var lockHash = ComputeAdvisoryLockKey("binaries_schema_migration");
// Acquire advisory lock to prevent concurrent migrations
await using var lockCmd = connection.CreateCommand();
lockCmd.CommandText = $"SELECT pg_try_advisory_lock({lockHash})";
lockCmd.CommandText = "SELECT pg_try_advisory_lock(@lockKey)";
lockCmd.Parameters.AddWithValue("lockKey", NpgsqlDbType.Bigint, lockHash);
var acquired = (bool)(await lockCmd.ExecuteScalarAsync(ct))!;
if (!acquired)
@@ -42,25 +44,92 @@ public sealed class BinaryIndexMigrationRunner
try
{
var migrations = GetEmbeddedMigrations();
foreach (var (name, sql) in migrations.OrderBy(m => m.name))
await EnsureHistoryTableAsync(connection, ct);
var applied = await GetAppliedMigrationsAsync(connection, ct);
var migrations = GetEmbeddedMigrations()
.Where(m => !applied.Contains(m.name))
.OrderBy(m => m.name)
.ToList();
if (migrations.Count == 0)
{
_logger.LogInformation("No pending migrations to apply");
return;
}
await using var tx = await connection.BeginTransactionAsync(ct);
foreach (var (name, sql) in migrations)
{
_logger.LogInformation("Applying migration: {Name}", name);
await using var cmd = connection.CreateCommand();
cmd.Transaction = tx;
cmd.CommandText = sql;
await cmd.ExecuteNonQueryAsync(ct);
await using var insert = connection.CreateCommand();
insert.Transaction = tx;
insert.CommandText = """
INSERT INTO binaries.schema_migrations (name)
VALUES (@name)
""";
insert.Parameters.AddWithValue("name", NpgsqlDbType.Text, name);
await insert.ExecuteNonQueryAsync(ct);
_logger.LogInformation("Migration {Name} applied successfully", name);
}
await tx.CommitAsync(ct);
}
finally
{
// Release advisory lock
await using var unlockCmd = connection.CreateCommand();
unlockCmd.CommandText = $"SELECT pg_advisory_unlock({lockHash})";
unlockCmd.CommandText = "SELECT pg_advisory_unlock(@lockKey)";
unlockCmd.Parameters.AddWithValue("lockKey", NpgsqlDbType.Bigint, lockHash);
await unlockCmd.ExecuteScalarAsync(ct);
}
}
private static async Task EnsureHistoryTableAsync(NpgsqlConnection connection, CancellationToken ct)
{
const string sql = """
CREATE SCHEMA IF NOT EXISTS binaries;
CREATE TABLE IF NOT EXISTS binaries.schema_migrations (
name TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
""";
await using var cmd = connection.CreateCommand();
cmd.CommandText = sql;
await cmd.ExecuteNonQueryAsync(ct);
}
private static async Task<HashSet<string>> GetAppliedMigrationsAsync(
NpgsqlConnection connection,
CancellationToken ct)
{
const string sql = "SELECT name FROM binaries.schema_migrations";
await using var cmd = connection.CreateCommand();
cmd.CommandText = sql;
var applied = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
await using var reader = await cmd.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
{
applied.Add(reader.GetString(0));
}
return applied;
}
private static long ComputeAdvisoryLockKey(string lockKey)
{
var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(lockKey));
return BinaryPrimitives.ReadInt64LittleEndian(bytes);
}
private static IEnumerable<(string name, string sql)> GetEmbeddedMigrations()
{
var assembly = typeof(BinaryIndexMigrationRunner).Assembly;

View File

@@ -75,7 +75,7 @@ ALTER TABLE binaries.delta_signature ENABLE ROW LEVEL SECURITY;
-- RLS policy for tenant isolation
DROP POLICY IF EXISTS delta_signature_tenant_isolation ON binaries.delta_signature;
CREATE POLICY delta_signature_tenant_isolation ON binaries.delta_signature
USING (tenant_id = binaries_app.current_tenant()::uuid);
USING (tenant_id = binaries_app.require_current_tenant()::uuid);
-- =============================================================================
-- SIGNATURE PACKS (for offline distribution)
@@ -101,7 +101,7 @@ ALTER TABLE binaries.signature_pack ENABLE ROW LEVEL SECURITY;
-- RLS policy for tenant isolation
DROP POLICY IF EXISTS signature_pack_tenant_isolation ON binaries.signature_pack;
CREATE POLICY signature_pack_tenant_isolation ON binaries.signature_pack
USING (tenant_id = binaries_app.current_tenant()::uuid);
USING (tenant_id = binaries_app.require_current_tenant()::uuid);
-- Index
CREATE INDEX IF NOT EXISTS idx_sig_pack_tenant ON binaries.signature_pack(tenant_id);
@@ -169,7 +169,7 @@ ALTER TABLE binaries.delta_sig_match ENABLE ROW LEVEL SECURITY;
-- RLS policy for tenant isolation
DROP POLICY IF EXISTS delta_sig_match_tenant_isolation ON binaries.delta_sig_match;
CREATE POLICY delta_sig_match_tenant_isolation ON binaries.delta_sig_match
USING (tenant_id = binaries_app.current_tenant()::uuid);
USING (tenant_id = binaries_app.require_current_tenant()::uuid);
-- =============================================================================
-- COMMENTS

View File

@@ -43,7 +43,11 @@ public sealed class BinaryIdentityRepository : IBinaryIdentityRepository
LIMIT 1
""";
var row = await conn.QuerySingleOrDefaultAsync<BinaryIdentityRow>(sql, new { BuildId = buildId, BuildIdType = buildIdType });
var command = new CommandDefinition(
sql,
new { BuildId = buildId, BuildIdType = buildIdType },
cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<BinaryIdentityRow>(command);
return row?.ToModel();
}
@@ -74,7 +78,11 @@ public sealed class BinaryIdentityRepository : IBinaryIdentityRepository
LIMIT 1
""";
var row = await conn.QuerySingleOrDefaultAsync<BinaryIdentityRow>(sql, new { BinaryKey = binaryKey });
var command = new CommandDefinition(
sql,
new { BinaryKey = binaryKey },
cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<BinaryIdentityRow>(command);
return row?.ToModel();
}
@@ -114,24 +122,28 @@ public sealed class BinaryIdentityRepository : IBinaryIdentityRepository
updated_at AS "UpdatedAt"
""";
var row = await conn.QuerySingleAsync<BinaryIdentityRow>(sql, new
{
identity.BinaryKey,
identity.BuildId,
identity.BuildIdType,
identity.FileSha256,
identity.TextSha256,
identity.Blake3Hash,
Format = identity.Format.ToString().ToLowerInvariant(),
identity.Architecture,
identity.OsAbi,
BinaryType = ToDbBinaryType(identity.Type),
identity.IsStripped,
identity.FirstSeenSnapshotId,
identity.LastSeenSnapshotId,
identity.CreatedAt,
identity.UpdatedAt
});
var command = new CommandDefinition(
sql,
new
{
identity.BinaryKey,
identity.BuildId,
identity.BuildIdType,
identity.FileSha256,
identity.TextSha256,
identity.Blake3Hash,
Format = identity.Format.ToString().ToLowerInvariant(),
identity.Architecture,
identity.OsAbi,
BinaryType = ToDbBinaryType(identity.Type),
identity.IsStripped,
identity.FirstSeenSnapshotId,
identity.LastSeenSnapshotId,
identity.CreatedAt,
identity.UpdatedAt
},
cancellationToken: ct);
var row = await conn.QuerySingleAsync<BinaryIdentityRow>(command);
return row.ToModel();
}
@@ -162,7 +174,11 @@ public sealed class BinaryIdentityRepository : IBinaryIdentityRepository
WHERE binary_key = ANY(@BinaryKeys)
""";
var rows = await conn.QueryAsync<BinaryIdentityRow>(sql, new { BinaryKeys = binaryKeys.ToArray() });
var command = new CommandDefinition(
sql,
new { BinaryKeys = binaryKeys.ToArray() },
cancellationToken: ct);
var rows = await conn.QueryAsync<BinaryIdentityRow>(command);
return rows.Select(r => r.ToModel()).ToImmutableArray();
}

View File

@@ -23,7 +23,8 @@ public sealed class BinaryVulnAssertionRepository : IBinaryVulnAssertionReposito
WHERE binary_key = @BinaryKey
""";
var rows = await conn.QueryAsync<BinaryVulnAssertion>(sql, new { BinaryKey = binaryKey });
var command = new CommandDefinition(sql, new { BinaryKey = binaryKey }, cancellationToken: ct);
var rows = await conn.QueryAsync<BinaryVulnAssertion>(command);
return rows.ToImmutableArray();
}
}

View File

@@ -53,15 +53,19 @@ public sealed class CorpusSnapshotRepository : ICorpusSnapshotRepository
created_at AS "CapturedAt"
""";
var row = await conn.QuerySingleAsync<CorpusSnapshotRow>(sql, new
{
snapshot.Id,
snapshot.Distro,
snapshot.Release,
snapshot.Architecture,
SnapshotId = $"{snapshot.Distro}_{snapshot.Release}_{snapshot.Architecture}_{snapshot.CapturedAt:yyyyMMddHHmmss}",
snapshot.MetadataDigest
});
var command = new CommandDefinition(
sql,
new
{
snapshot.Id,
snapshot.Distro,
snapshot.Release,
snapshot.Architecture,
SnapshotId = $"{snapshot.Distro}_{snapshot.Release}_{snapshot.Architecture}_{snapshot.CapturedAt:yyyyMMddHHmmss}",
snapshot.MetadataDigest
},
cancellationToken: ct);
var row = await conn.QuerySingleAsync<CorpusSnapshotRow>(command);
_logger.LogInformation(
"Created corpus snapshot {Id} for {Distro} {Release}/{Architecture}",
@@ -93,12 +97,16 @@ public sealed class CorpusSnapshotRepository : ICorpusSnapshotRepository
LIMIT 1
""";
var row = await conn.QuerySingleOrDefaultAsync<CorpusSnapshotRow>(sql, new
{
Distro = distro,
Release = release,
Architecture = architecture
});
var command = new CommandDefinition(
sql,
new
{
Distro = distro,
Release = release,
Architecture = architecture
},
cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<CorpusSnapshotRow>(command);
return row?.ToModel();
}
@@ -118,7 +126,8 @@ public sealed class CorpusSnapshotRepository : ICorpusSnapshotRepository
WHERE id = @Id
""";
var row = await conn.QuerySingleOrDefaultAsync<CorpusSnapshotRow>(sql, new { Id = id });
var command = new CommandDefinition(sql, new { Id = id }, cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<CorpusSnapshotRow>(command);
return row?.ToModel();
}
@@ -132,12 +141,14 @@ public sealed class CorpusSnapshotRepository : ICorpusSnapshotRepository
public string MetadataDigest { get; set; } = string.Empty;
public DateTimeOffset CapturedAt { get; set; }
public CorpusSnapshot ToModel() => new(
Id: Id,
Distro: Distro,
Release: Release,
Architecture: Architecture,
MetadataDigest: MetadataDigest,
CapturedAt: CapturedAt);
public CorpusSnapshot ToModel() => new()
{
Id = Id,
Distro = Distro,
Release = Release,
Architecture = Architecture,
MetadataDigest = MetadataDigest,
CapturedAt = CapturedAt
};
}
}

View File

@@ -49,7 +49,7 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
attestation_dsse, metadata
)
VALUES (
@Id, binaries_app.current_tenant()::uuid, @CveId, @PackageName, @Soname, @Arch, @Abi,
@Id, binaries_app.require_current_tenant()::uuid, @CveId, @PackageName, @Soname, @Arch, @Abi,
@RecipeId, @RecipeVersion, @SymbolName, @Scope,
@HashAlg, @HashHex, @SizeBytes,
@CfgBbCount, @CfgEdgeHash, @ChunkHashes::jsonb,
@@ -62,7 +62,7 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
var now = DateTimeOffset.UtcNow;
var id = entity.Id != Guid.Empty ? entity.Id : Guid.NewGuid();
var result = await conn.QuerySingleAsync<(Guid Id, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt)>(
var command = new CommandDefinition(
sql,
new
{
@@ -91,7 +91,9 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
Metadata = entity.Metadata != null
? JsonSerializer.Serialize(entity.Metadata, s_jsonOptions)
: null
});
},
cancellationToken: ct);
var result = await conn.QuerySingleAsync<(Guid Id, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt)>(command);
_logger.LogDebug(
"Created delta signature {Id} for {CveId}/{SymbolName} ({State})",
@@ -141,7 +143,8 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
WHERE id = @Id
""";
var row = await conn.QuerySingleOrDefaultAsync<DeltaSignatureRow>(sql, new { Id = id });
var command = new CommandDefinition(sql, new { Id = id }, cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<DeltaSignatureRow>(command);
return row?.ToEntity();
}
@@ -165,7 +168,8 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
ORDER BY package_name, symbol_name, signature_state
""";
var rows = await conn.QueryAsync<DeltaSignatureRow>(sql, new { CveId = cveId });
var command = new CommandDefinition(sql, new { CveId = cveId }, cancellationToken: ct);
var rows = await conn.QueryAsync<DeltaSignatureRow>(command);
return rows.Select(r => r.ToEntity()).ToList();
}
@@ -196,9 +200,11 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
sql += " ORDER BY cve_id, symbol_name, signature_state";
var rows = await conn.QueryAsync<DeltaSignatureRow>(
var command = new CommandDefinition(
sql,
new { PackageName = packageName, Soname = soname });
new { PackageName = packageName, Soname = soname },
cancellationToken: ct);
var rows = await conn.QueryAsync<DeltaSignatureRow>(command);
return rows.Select(r => r.ToEntity()).ToList();
}
@@ -222,9 +228,11 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
WHERE hash_hex = @HashHex
""";
var rows = await conn.QueryAsync<DeltaSignatureRow>(
var command = new CommandDefinition(
sql,
new { HashHex = hashHex.ToLowerInvariant() });
new { HashHex = hashHex.ToLowerInvariant() },
cancellationToken: ct);
var rows = await conn.QueryAsync<DeltaSignatureRow>(command);
return rows.Select(r => r.ToEntity()).ToList();
}
@@ -259,9 +267,11 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
ORDER BY cve_id, symbol_name, signature_state
""";
var rows = await conn.QueryAsync<DeltaSignatureRow>(
var command = new CommandDefinition(
sql,
new { Arch = arch, Abi = abi, SymbolNames = symbolList.ToArray() });
new { Arch = arch, Abi = abi, SymbolNames = symbolList.ToArray() },
cancellationToken: ct);
var rows = await conn.QueryAsync<DeltaSignatureRow>(command);
return rows.Select(r => r.ToEntity()).ToList();
}
@@ -313,7 +323,8 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
ORDER BY cve_id, symbol_name, signature_state
""";
var rows = await conn.QueryAsync<DeltaSignatureRow>(sql, parameters);
var command = new CommandDefinition(sql, parameters, cancellationToken: ct);
var rows = await conn.QueryAsync<DeltaSignatureRow>(command);
_logger.LogDebug("GetAllMatchingAsync returned {Count} signatures", rows.Count());
return rows.Select(r => r.ToEntity()).ToList();
@@ -353,7 +364,7 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
var now = DateTimeOffset.UtcNow;
var updatedAt = await conn.ExecuteScalarAsync<DateTimeOffset>(
var command = new CommandDefinition(
sql,
new
{
@@ -381,7 +392,9 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
Metadata = entity.Metadata != null
? JsonSerializer.Serialize(entity.Metadata, s_jsonOptions)
: null
});
},
cancellationToken: ct);
var updatedAt = await conn.ExecuteScalarAsync<DateTimeOffset>(command);
_logger.LogDebug("Updated delta signature {Id}", entity.Id);
return entity with { UpdatedAt = updatedAt };
@@ -395,7 +408,8 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
await using var conn = await _dbContext.OpenConnectionAsync(ct);
const string sql = "DELETE FROM binaries.delta_signature WHERE id = @Id";
var rows = await conn.ExecuteAsync(sql, new { Id = id });
var command = new CommandDefinition(sql, new { Id = id }, cancellationToken: ct);
var rows = await conn.ExecuteAsync(command);
if (rows > 0)
{
@@ -417,7 +431,8 @@ public sealed class DeltaSignatureRepository : IDeltaSignatureRepository
GROUP BY signature_state
""";
var rows = await conn.QueryAsync<(string State, int Count)>(sql);
var command = new CommandDefinition(sql, cancellationToken: ct);
var rows = await conn.QueryAsync<(string State, int Count)>(command);
return rows.ToDictionary(r => r.State, r => r.Count);
}

View File

@@ -2,6 +2,7 @@ using System.Collections.Immutable;
using Dapper;
using StellaOps.BinaryIndex.Fingerprints;
using StellaOps.BinaryIndex.Fingerprints.Models;
using System.Text.Json;
namespace StellaOps.BinaryIndex.Persistence.Repositories;
@@ -11,6 +12,7 @@ namespace StellaOps.BinaryIndex.Persistence.Repositories;
public sealed class FingerprintRepository : IFingerprintRepository
{
private readonly BinaryIndexDbContext _dbContext;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public FingerprintRepository(BinaryIndexDbContext dbContext)
{
@@ -28,7 +30,7 @@ public sealed class FingerprintRepository : IFingerprintRepository
confidence, validated, validation_stats, vuln_build_ref, fixed_build_ref, indexed_at
)
VALUES (
@Id, binaries_app.current_tenant()::uuid, @CveId, @Component, @Purl, @Algorithm,
@Id, binaries_app.require_current_tenant()::uuid, @CveId, @Component, @Purl, @Algorithm,
@FingerprintId, @FingerprintHash, @Architecture, @FunctionName, @SourceFile,
@SourceLine, @SimilarityThreshold, @Confidence, @Validated, @ValidationStats::jsonb,
@VulnBuildRef, @FixedBuildRef, @IndexedAt
@@ -36,29 +38,33 @@ public sealed class FingerprintRepository : IFingerprintRepository
RETURNING id
""";
var id = await conn.ExecuteScalarAsync<Guid>(sql, new
{
Id = fingerprint.Id != Guid.Empty ? fingerprint.Id : Guid.NewGuid(),
fingerprint.CveId,
fingerprint.Component,
fingerprint.Purl,
Algorithm = fingerprint.Algorithm.ToString().ToLowerInvariant().Replace("_", ""),
fingerprint.FingerprintId,
fingerprint.FingerprintHash,
fingerprint.Architecture,
fingerprint.FunctionName,
fingerprint.SourceFile,
fingerprint.SourceLine,
fingerprint.SimilarityThreshold,
fingerprint.Confidence,
fingerprint.Validated,
ValidationStats = fingerprint.ValidationStats != null
? System.Text.Json.JsonSerializer.Serialize(fingerprint.ValidationStats)
: "{}",
fingerprint.VulnBuildRef,
fingerprint.FixedBuildRef,
fingerprint.IndexedAt
});
var command = new CommandDefinition(
sql,
new
{
Id = fingerprint.Id != Guid.Empty ? fingerprint.Id : Guid.NewGuid(),
fingerprint.CveId,
fingerprint.Component,
fingerprint.Purl,
Algorithm = ToDbAlgorithm(fingerprint.Algorithm),
fingerprint.FingerprintId,
fingerprint.FingerprintHash,
fingerprint.Architecture,
fingerprint.FunctionName,
fingerprint.SourceFile,
fingerprint.SourceLine,
fingerprint.SimilarityThreshold,
fingerprint.Confidence,
fingerprint.Validated,
ValidationStats = fingerprint.ValidationStats != null
? JsonSerializer.Serialize(fingerprint.ValidationStats, JsonOptions)
: "{}",
fingerprint.VulnBuildRef,
fingerprint.FixedBuildRef,
fingerprint.IndexedAt
},
cancellationToken: ct);
var id = await conn.ExecuteScalarAsync<Guid>(command);
return fingerprint with { Id = id };
}
@@ -78,21 +84,36 @@ public sealed class FingerprintRepository : IFingerprintRepository
WHERE id = @Id
""";
// Simplified: Would need proper mapping from DB row to model
// Including JSONB deserialization for validation_stats
return null; // Placeholder for brevity
var command = new CommandDefinition(sql, new { Id = id }, cancellationToken: ct);
var row = await conn.QuerySingleOrDefaultAsync<VulnFingerprintRow>(command);
return row?.ToModel();
}
public async Task<ImmutableArray<VulnFingerprint>> GetByCveAsync(string cveId, CancellationToken ct = default)
{
// Similar implementation to GetByIdAsync but for multiple records
return ImmutableArray<VulnFingerprint>.Empty;
await using var conn = await _dbContext.OpenConnectionAsync(ct);
const string sql = """
SELECT id, cve_id as CveId, component, purl, algorithm, fingerprint_id as FingerprintId,
fingerprint_hash as FingerprintHash, architecture, function_name as FunctionName,
source_file as SourceFile, source_line as SourceLine,
similarity_threshold as SimilarityThreshold, confidence, validated,
validation_stats as ValidationStats, vuln_build_ref as VulnBuildRef,
fixed_build_ref as FixedBuildRef, indexed_at as IndexedAt
FROM binaries.vulnerable_fingerprints
WHERE cve_id = @CveId
ORDER BY component, fingerprint_id
""";
var command = new CommandDefinition(sql, new { CveId = cveId }, cancellationToken: ct);
var rows = await conn.QueryAsync<VulnFingerprintRow>(command);
return rows.Select(r => r.ToModel()).ToImmutableArray();
}
public async Task<ImmutableArray<VulnFingerprint>> SearchByHashAsync(
byte[] hash,
FingerprintAlgorithm algorithm,
string architecture,
string? architecture,
CancellationToken ct = default)
{
await using var conn = await _dbContext.OpenConnectionAsync(ct);
@@ -107,11 +128,21 @@ public sealed class FingerprintRepository : IFingerprintRepository
FROM binaries.vulnerable_fingerprints
WHERE fingerprint_hash = @Hash
AND algorithm = @Algorithm
AND architecture = @Architecture
AND (@Architecture IS NULL OR architecture = @Architecture)
""";
// Simplified: Would need proper mapping
return ImmutableArray<VulnFingerprint>.Empty;
var command = new CommandDefinition(
sql,
new
{
Hash = hash,
Algorithm = ToDbAlgorithm(algorithm),
Architecture = architecture
},
cancellationToken: ct);
var rows = await conn.QueryAsync<VulnFingerprintRow>(command);
return rows.Select(r => r.ToModel()).ToImmutableArray();
}
public async Task UpdateValidationStatsAsync(
@@ -128,11 +159,94 @@ public sealed class FingerprintRepository : IFingerprintRepository
WHERE id = @Id
""";
await conn.ExecuteAsync(sql, new
var command = new CommandDefinition(
sql,
new
{
Id = id,
Stats = JsonSerializer.Serialize(stats, JsonOptions)
},
cancellationToken: ct);
await conn.ExecuteAsync(command);
}
private static string ToDbAlgorithm(FingerprintAlgorithm algorithm)
{
return algorithm switch
{
Id = id,
Stats = System.Text.Json.JsonSerializer.Serialize(stats)
});
FingerprintAlgorithm.BasicBlock => "basic_block",
FingerprintAlgorithm.ControlFlowGraph => "control_flow_graph",
FingerprintAlgorithm.StringRefs => "string_refs",
FingerprintAlgorithm.Combined => "combined",
_ => algorithm.ToString().ToLowerInvariant()
};
}
private static FingerprintAlgorithm ParseAlgorithm(string value)
{
return value.ToLowerInvariant() switch
{
"basic_block" => FingerprintAlgorithm.BasicBlock,
"cfg" => FingerprintAlgorithm.ControlFlowGraph,
"control_flow_graph" => FingerprintAlgorithm.ControlFlowGraph,
"string_refs" => FingerprintAlgorithm.StringRefs,
"combined" => FingerprintAlgorithm.Combined,
_ => Enum.Parse<FingerprintAlgorithm>(value, ignoreCase: true)
};
}
private sealed class VulnFingerprintRow
{
public Guid Id { get; init; }
public string CveId { get; init; } = string.Empty;
public string Component { get; init; } = string.Empty;
public string? Purl { get; init; }
public string Algorithm { get; init; } = string.Empty;
public string FingerprintId { get; init; } = string.Empty;
public byte[] FingerprintHash { get; init; } = Array.Empty<byte>();
public string Architecture { get; init; } = string.Empty;
public string? FunctionName { get; init; }
public string? SourceFile { get; init; }
public int? SourceLine { get; init; }
public decimal SimilarityThreshold { get; init; }
public decimal? Confidence { get; init; }
public bool Validated { get; init; }
public string? ValidationStats { get; init; }
public string? VulnBuildRef { get; init; }
public string? FixedBuildRef { get; init; }
public DateTimeOffset IndexedAt { get; init; }
public VulnFingerprint ToModel()
{
FingerprintValidationStats? stats = null;
if (!string.IsNullOrWhiteSpace(ValidationStats))
{
stats = JsonSerializer.Deserialize<FingerprintValidationStats>(ValidationStats, JsonOptions);
}
return new VulnFingerprint
{
Id = Id,
CveId = CveId,
Component = Component,
Purl = Purl,
Algorithm = ParseAlgorithm(Algorithm),
FingerprintId = FingerprintId,
FingerprintHash = FingerprintHash,
Architecture = Architecture,
FunctionName = FunctionName,
SourceFile = SourceFile,
SourceLine = SourceLine,
SimilarityThreshold = SimilarityThreshold,
Confidence = Confidence,
Validated = Validated,
ValidationStats = stats,
VulnBuildRef = VulnBuildRef,
FixedBuildRef = FixedBuildRef,
IndexedAt = IndexedAt
};
}
}
}
@@ -159,29 +273,33 @@ public sealed class FingerprintMatchRepository : IFingerprintMatchRepository
similarity, advisory_ids, reachability_status, matched_at
)
VALUES (
@Id, binaries_app.current_tenant()::uuid, @ScanId, @MatchType, @BinaryKey,
@Id, binaries_app.require_current_tenant()::uuid, @ScanId, @MatchType, @BinaryKey,
@BinaryIdentityId, @VulnerablePurl, @VulnerableVersion, @MatchedFingerprintId,
@MatchedFunction, @Similarity, @AdvisoryIds, @ReachabilityStatus, @MatchedAt
)
RETURNING id
""";
var id = await conn.ExecuteScalarAsync<Guid>(sql, new
{
Id = match.Id != Guid.Empty ? match.Id : Guid.NewGuid(),
match.ScanId,
MatchType = match.Type.ToString().ToLowerInvariant(),
match.BinaryKey,
BinaryIdentityId = (Guid?)null,
match.VulnerablePurl,
match.VulnerableVersion,
match.MatchedFingerprintId,
match.MatchedFunction,
match.Similarity,
match.AdvisoryIds,
ReachabilityStatus = match.ReachabilityStatus?.ToString().ToLowerInvariant(),
match.MatchedAt
});
var command = new CommandDefinition(
sql,
new
{
Id = match.Id != Guid.Empty ? match.Id : Guid.NewGuid(),
match.ScanId,
MatchType = match.Type.ToString().ToLowerInvariant(),
match.BinaryKey,
BinaryIdentityId = (Guid?)null,
match.VulnerablePurl,
match.VulnerableVersion,
match.MatchedFingerprintId,
match.MatchedFunction,
match.Similarity,
AdvisoryIds = match.AdvisoryIds.IsDefaultOrEmpty ? null : match.AdvisoryIds.ToArray(),
ReachabilityStatus = match.ReachabilityStatus?.ToString().ToLowerInvariant(),
match.MatchedAt
},
cancellationToken: ct);
var id = await conn.ExecuteScalarAsync<Guid>(command);
return match with { Id = id };
}
@@ -202,10 +320,14 @@ public sealed class FingerprintMatchRepository : IFingerprintMatchRepository
WHERE id = @Id
""";
await conn.ExecuteAsync(sql, new
{
Id = id,
Status = status.ToString().ToLowerInvariant()
});
var command = new CommandDefinition(
sql,
new
{
Id = id,
Status = status.ToString().ToLowerInvariant()
},
cancellationToken: ct);
await conn.ExecuteAsync(command);
}
}

View File

@@ -124,10 +124,10 @@ public sealed class FixIndexRepository : IFixIndexRepository
const string sql = """
INSERT INTO binaries.cve_fix_index
(distro, release, source_pkg, cve_id, state, fixed_version, method, confidence, evidence_id, snapshot_id)
(distro, release, source_pkg, cve_id, architecture, state, fixed_version, method, confidence, evidence_id, snapshot_id)
VALUES
(@distro, @release, @sourcePkg, @cveId, @state, @fixedVersion, @method, @confidence, @evidenceId, @snapshotId)
ON CONFLICT (tenant_id, distro, release, source_pkg, cve_id)
(@distro, @release, @sourcePkg, @cveId, @architecture, @state, @fixedVersion, @method, @confidence, @evidenceId, @snapshotId)
ON CONFLICT (tenant_id, distro, release, source_pkg, cve_id, architecture)
DO UPDATE SET
state = EXCLUDED.state,
fixed_version = EXCLUDED.fixed_version,
@@ -152,9 +152,10 @@ public sealed class FixIndexRepository : IFixIndexRepository
cmd.Parameters.AddWithValue("release", evidence.Release);
cmd.Parameters.AddWithValue("sourcePkg", evidence.SourcePkg);
cmd.Parameters.AddWithValue("cveId", evidence.CveId);
cmd.Parameters.AddWithValue("architecture", DBNull.Value);
cmd.Parameters.AddWithValue("state", evidence.State.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("fixedVersion", (object?)evidence.FixedVersion ?? DBNull.Value);
cmd.Parameters.AddWithValue("method", evidence.Method.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("method", ToDbFixMethod(evidence.Method));
cmd.Parameters.AddWithValue("confidence", evidence.Confidence);
cmd.Parameters.AddWithValue("evidenceId", evidenceId);
cmd.Parameters.AddWithValue("snapshotId", (object?)evidence.SnapshotId ?? DBNull.Value);
@@ -232,7 +233,7 @@ public sealed class FixIndexRepository : IFixIndexRepository
Excerpt = reader.IsDBNull(4) ? null : reader.GetString(4),
MetadataJson = reader.GetString(5),
SnapshotId = reader.IsDBNull(6) ? null : reader.GetGuid(6),
CreatedAt = reader.GetDateTime(7)
CreatedAt = reader.GetFieldValue<DateTimeOffset>(7)
};
}
@@ -277,8 +278,8 @@ public sealed class FixIndexRepository : IFixIndexRepository
Confidence = reader.GetDecimal(8),
EvidenceId = reader.IsDBNull(9) ? null : reader.GetGuid(9),
SnapshotId = reader.IsDBNull(10) ? null : reader.GetGuid(10),
IndexedAt = reader.GetDateTime(11),
UpdatedAt = reader.GetDateTime(12)
IndexedAt = reader.GetFieldValue<DateTimeOffset>(11),
UpdatedAt = reader.GetFieldValue<DateTimeOffset>(12)
};
}
@@ -290,10 +291,23 @@ public sealed class FixIndexRepository : IFixIndexRepository
"changelog" => FixMethod.Changelog,
"patch_header" => FixMethod.PatchHeader,
"upstream_match" => FixMethod.UpstreamPatchMatch,
"upstreampatchmatch" => FixMethod.UpstreamPatchMatch,
_ => FixMethod.Changelog
};
}
private static string ToDbFixMethod(FixMethod method)
{
return method switch
{
FixMethod.SecurityFeed => "security_feed",
FixMethod.Changelog => "changelog",
FixMethod.PatchHeader => "patch_header",
FixMethod.UpstreamPatchMatch => "upstream_match",
_ => "changelog"
};
}
private static (string Type, string? File, string? Excerpt, string Metadata) MapEvidencePayload(FixEvidencePayload payload)
{
return payload switch

View File

@@ -69,11 +69,22 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
CancellationToken ct = default)
{
var results = new Dictionary<string, ImmutableArray<BinaryVulnMatch>>();
var identityList = identities.ToList();
const int batchSize = 16;
foreach (var identity in identities)
for (var i = 0; i < identityList.Count; i += batchSize)
{
var matches = await LookupByIdentityAsync(identity, options, ct);
results[identity.BinaryKey] = matches;
var batch = identityList.Skip(i).Take(batchSize).ToList();
var tasks = batch.Select(async identity =>
{
var matches = await LookupByIdentityAsync(identity, options, ct).ConfigureAwait(false);
return (identity.BinaryKey, matches);
});
foreach (var (key, matches) in await Task.WhenAll(tasks).ConfigureAwait(false))
{
results[key] = matches;
}
}
return results.ToImmutableDictionary();
@@ -125,12 +136,24 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
return results.ToImmutableDictionary();
}
foreach (var cveId in cveIds)
var cveList = cveIds.ToList();
const int batchSize = 32;
for (var i = 0; i < cveList.Count; i += batchSize)
{
var status = await GetFixStatusAsync(distro, release, sourcePkg, cveId, ct);
if (status is not null)
var batch = cveList.Skip(i).Take(batchSize).ToList();
var tasks = batch.Select(async cveId =>
{
results[cveId] = status;
var status = await GetFixStatusAsync(distro, release, sourcePkg, cveId, ct).ConfigureAwait(false);
return (cveId, status);
});
foreach (var (cveId, status) in await Task.WhenAll(tasks).ConfigureAwait(false))
{
if (status is not null)
{
results[cveId] = status;
}
}
}
@@ -181,7 +204,8 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
Evidence = new MatchEvidence
{
Similarity = result.Similarity,
MatchedFunction = fp.FunctionName
MatchedFunction = fp.FunctionName,
FingerprintAlgorithm = fp.Algorithm.ToString().ToLowerInvariant()
}
});
}
@@ -196,11 +220,22 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
CancellationToken ct = default)
{
var results = new Dictionary<string, ImmutableArray<BinaryVulnMatch>>();
var fingerprintList = fingerprints.ToList();
const int batchSize = 16;
foreach (var (key, fingerprint) in fingerprints)
for (var i = 0; i < fingerprintList.Count; i += batchSize)
{
var matches = await LookupByFingerprintAsync(fingerprint, options, ct).ConfigureAwait(false);
results[key] = matches;
var batch = fingerprintList.Skip(i).Take(batchSize).ToList();
var tasks = batch.Select(async item =>
{
var matches = await LookupByFingerprintAsync(item.Fingerprint, options, ct).ConfigureAwait(false);
return (item.Key, matches);
});
foreach (var (key, matches) in await Task.WhenAll(tasks).ConfigureAwait(false))
{
results[key] = matches;
}
}
return results.ToImmutableDictionary();
@@ -240,9 +275,16 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
continue;
var firstMatch = result.SymbolMatches.FirstOrDefault();
var cveId = result.Cve;
if (string.IsNullOrWhiteSpace(cveId))
{
_logger.LogWarning("Delta signature match missing CVE id for {Symbol}", firstMatch?.SymbolName ?? "unknown");
continue;
}
matches.Add(new BinaryVulnMatch
{
CveId = result.Cve,
CveId = cveId,
VulnerablePurl = "pkg:generic/unknown", // Will be enriched from signature
Method = MatchMethod.DeltaSignature,
Confidence = (decimal)result.Confidence,
@@ -291,9 +333,22 @@ public sealed class BinaryVulnerabilityService : IBinaryVulnerabilityService
if (!ShouldIncludeResult(result, options))
continue;
if (string.IsNullOrWhiteSpace(result.Cve))
{
_logger.LogWarning("Delta signature match missing CVE id for {Symbol}", symbolName);
continue;
}
var cveId = result.Cve;
if (string.IsNullOrWhiteSpace(cveId))
{
_logger.LogWarning("Delta signature match missing CVE id for {Symbol}", symbolName);
continue;
}
matches.Add(new BinaryVulnMatch
{
CveId = result.Cve,
CveId = cveId,
VulnerablePurl = "pkg:generic/unknown", // Will be enriched from signature
Method = MatchMethod.DeltaSignature,
Confidence = (decimal)result.Confidence,

View File

@@ -5,6 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0125-M | DONE | Maintainability audit for StellaOps.BinaryIndex.Persistence. |
| AUDIT-0125-T | DONE | Test coverage audit for StellaOps.BinaryIndex.Persistence. |
| AUDIT-0125-A | TODO | Pending approval for changes. |
| AUDIT-0125-A | DONE | Pending approval for changes. |

View File

@@ -1,3 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json.Nodes;
namespace StellaOps.BinaryIndex.VexBridge;
@@ -44,6 +45,52 @@ public static class BinaryMatchEvidenceSchema
public const string HashExact = "hash_exact";
}
private static readonly HashSet<string> s_validMatchTypes = new(StringComparer.Ordinal)
{
MatchTypes.BuildId,
MatchTypes.Fingerprint,
MatchTypes.HashExact
};
/// <summary>
/// Validates an evidence payload against the expected schema.
/// </summary>
public static bool ValidateEvidence(JsonObject evidence, out string? error)
{
error = null;
if (evidence is null)
{
error = "Evidence payload is null.";
return false;
}
if (!TryGetString(evidence, Fields.Type, out var type) || type != EvidenceType)
{
error = $"Evidence type must be '{EvidenceType}'.";
return false;
}
if (!TryGetString(evidence, Fields.SchemaVersion, out var version) || version != SchemaVersion)
{
error = $"Schema version must be '{SchemaVersion}'.";
return false;
}
if (!TryGetString(evidence, Fields.MatchType, out var matchType))
{
error = "Match type is missing or invalid.";
return false;
}
if (!s_validMatchTypes.Contains(matchType))
{
error = "Match type is missing or invalid.";
return false;
}
return true;
}
/// <summary>
/// Creates an evidence JSON object from the provided parameters.
/// </summary>
@@ -119,4 +166,19 @@ public static class BinaryMatchEvidenceSchema
return evidence;
}
private static bool TryGetString(
JsonObject evidence,
string field,
[NotNullWhen(true)] out string? value)
{
value = null;
if (!evidence.TryGetPropertyValue(field, out var node))
{
return false;
}
value = node?.GetValue<string>();
return !string.IsNullOrWhiteSpace(value);
}
}

View File

@@ -1,7 +1,7 @@
// -----------------------------------------------------------------------------
// IDsseSigningAdapter.cs
// Sprint: SPRINT_1227_0001_0001_LB_binary_vex_generator
// Task: T5 DSSE signing integration
// Task: T5 - DSSE signing integration
// -----------------------------------------------------------------------------
namespace StellaOps.BinaryIndex.VexBridge;

View File

@@ -5,7 +5,7 @@
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<Description>Bridges binary fingerprint matching to VEX observation generation for StellaOps.</Description>
</PropertyGroup>

View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0127-M | DONE | Maintainability audit for StellaOps.BinaryIndex.VexBridge. |
| AUDIT-0127-T | DONE | Test coverage audit for StellaOps.BinaryIndex.VexBridge. |
| AUDIT-0127-A | TODO | Pending approval for changes. |
| AUDIT-0127-A | DONE | Applied TimeProvider, link control, DSSE metadata, schema validation, algorithm propagation, deterministic tests. |

View File

@@ -51,4 +51,14 @@ public sealed class VexBridgeOptions
/// Default: StellaOps BinaryIndex namespace.
/// </summary>
public Guid ObservationIdNamespace { get; set; } = new("d9e0a5f3-7b2c-4e8d-9a1f-6c3b5d8e2f0a");
/// <summary>
/// Whether to include external reference links (e.g., NVD) in linksets.
/// </summary>
public bool IncludeExternalLinks { get; set; } = true;
/// <summary>
/// Base URL for CVE references when external links are enabled.
/// </summary>
public string NvdCveBaseUrl { get; set; } = "https://nvd.nist.gov/vuln/detail/";
}

View File

@@ -22,15 +22,18 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
private readonly ILogger<VexEvidenceGenerator> _logger;
private readonly VexBridgeOptions _options;
private readonly IDsseSigningAdapter? _dsseSigner;
private readonly TimeProvider _timeProvider;
public VexEvidenceGenerator(
ILogger<VexEvidenceGenerator> logger,
IOptions<VexBridgeOptions> options,
IDsseSigningAdapter? dsseSigner = null)
IDsseSigningAdapter? dsseSigner = null,
TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_dsseSigner = dsseSigner;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
@@ -47,17 +50,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
ct.ThrowIfCancellationRequested();
// Check confidence threshold
var effectiveConfidence = fixStatus?.Confidence ?? match.Confidence;
if (effectiveConfidence < _options.MinConfidenceThreshold)
{
_logger.LogDebug(
"Skipping observation for {CveId}: confidence {Confidence} below threshold {Threshold}",
match.CveId, effectiveConfidence, _options.MinConfidenceThreshold);
throw new InvalidOperationException(
$"Match confidence {effectiveConfidence} is below minimum threshold {_options.MinConfidenceThreshold}");
}
EnsureAboveThreshold(match, fixStatus);
var observation = await CreateObservationAsync(match, identity, fixStatus, context, ct);
return observation;
@@ -87,6 +80,14 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
try
{
if (IsBelowThreshold(item.Match, item.FixStatus))
{
_logger.LogDebug(
"Skipping observation for {CveId}: confidence below threshold {Threshold}",
item.Match.CveId, _options.MinConfidenceThreshold);
continue;
}
var observation = await GenerateFromBinaryMatchAsync(
item.Match,
item.Identity,
@@ -98,7 +99,6 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
}
catch (InvalidOperationException ex) when (ex.Message.Contains("below minimum threshold"))
{
// Skip items below threshold, continue with batch
_logger.LogDebug("Skipping batch item: {Message}", ex.Message);
}
}
@@ -133,7 +133,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
context.ProductKey,
context.ScanId);
var now = DateTimeOffset.UtcNow;
var now = _timeProvider.GetUtcNow();
// Map fix status to VEX status and justification
var (vexStatus, justification) = MapToVexStatus(fixStatus);
@@ -145,7 +145,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
var upstream = await CreateUpstreamAsync(observationId, evidence, now, context.SignWithDsse, ct);
// Create statement
var statement = CreateStatement(match, context, vexStatus, justification, fixStatus);
var statement = CreateStatement(match, context, vexStatus, justification, fixStatus, now);
// Create content
var content = CreateContent(evidence);
@@ -217,7 +217,9 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
buildId: identity.BuildId,
fileSha256: identity.FileSha256,
textSha256: identity.TextSha256,
fingerprintAlgorithm: matchType == BinaryMatchEvidenceSchema.MatchTypes.Fingerprint ? "combined" : null,
fingerprintAlgorithm: matchType == BinaryMatchEvidenceSchema.MatchTypes.Fingerprint
? match.Evidence?.FingerprintAlgorithm
: null,
similarity: match.Evidence?.Similarity ?? match.Confidence,
distroRelease: context.DistroRelease,
sourcePackage: ExtractSourcePackage(match.VulnerablePurl),
@@ -243,6 +245,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
var contentHash = ComputeSha256(evidenceJson);
VexObservationSignature signature;
var metadata = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
// Sign with DSSE if requested and signer is available
if (signWithDsse && _dsseSigner is { IsAvailable: true })
@@ -263,6 +266,8 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
format: "dsse",
keyId: _dsseSigner.SigningKeyId,
signature: envelopeBase64);
metadata["dsse_status"] = "signed";
metadata["dsse_envelope_hash"] = envelopeHash;
_logger.LogDebug(
"DSSE signature generated for observation {ObservationId} with key {KeyId}",
@@ -279,6 +284,8 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
format: null,
keyId: null,
signature: null);
metadata["dsse_status"] = "failed";
metadata["dsse_error"] = ex.GetType().Name;
}
}
else
@@ -288,6 +295,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
_logger.LogDebug(
"DSSE signing requested but no signer configured for observation {ObservationId}",
observationId);
metadata["dsse_status"] = "unavailable";
}
signature = new VexObservationSignature(
@@ -304,7 +312,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
receivedAt: now,
contentHash: contentHash,
signature: signature,
metadata: ImmutableDictionary<string, string>.Empty
metadata: metadata.ToImmutable()
.Add("source", "binary_fingerprint_analysis"));
}
@@ -313,7 +321,8 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
VexGenerationContext context,
VexClaimStatus status,
VexJustification? justification,
FixStatusResult? fixStatus)
FixStatusResult? fixStatus,
DateTimeOffset now)
{
var detail = BuildStatementDetail(match, fixStatus);
@@ -321,7 +330,7 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
vulnerabilityId: match.CveId,
productKey: context.ProductKey,
status: status,
lastObserved: DateTimeOffset.UtcNow,
lastObserved: now,
locator: null,
justification: justification,
introducedVersion: null,
@@ -365,16 +374,24 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
raw: evidence);
}
private static VexObservationLinkset CreateLinkset(
private VexObservationLinkset CreateLinkset(
BinaryVulnMatch match,
BinaryIdentity identity)
{
var refs = new List<VexObservationReference>
{
new(type: "vulnerability", url: $"https://nvd.nist.gov/vuln/detail/{match.CveId}"),
new(type: "package", url: match.VulnerablePurl)
};
if (_options.IncludeExternalLinks)
{
var baseUrl = string.IsNullOrWhiteSpace(_options.NvdCveBaseUrl)
? "https://nvd.nist.gov/vuln/detail/"
: _options.NvdCveBaseUrl;
var separator = baseUrl.EndsWith("/", StringComparison.Ordinal) ? string.Empty : "/";
refs.Insert(0, new(type: "vulnerability", url: $"{baseUrl}{separator}{match.CveId}"));
}
if (!string.IsNullOrEmpty(identity.BuildId))
{
refs.Add(new(type: "build_id", url: $"urn:build-id:{identity.BuildId}"));
@@ -389,19 +406,39 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
private static string? ExtractSourcePackage(string purl)
{
// Simple extraction from PURL: pkg:deb/debian/openssl@3.0.7 openssl
// Simple extraction from PURL: pkg:deb/debian/openssl@3.0.7 -> openssl
if (string.IsNullOrEmpty(purl))
return null;
try
{
var parts = purl.Split('/');
if (parts.Length >= 3)
var trimmed = purl;
if (trimmed.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase))
{
var nameVersion = parts[^1];
var atIndex = nameVersion.IndexOf('@');
return atIndex > 0 ? nameVersion[..atIndex] : nameVersion;
trimmed = trimmed[4..];
}
var qualifierIndex = trimmed.IndexOf('?');
if (qualifierIndex >= 0)
{
trimmed = trimmed[..qualifierIndex];
}
var subpathIndex = trimmed.IndexOf('#');
if (subpathIndex >= 0)
{
trimmed = trimmed[..subpathIndex];
}
var segments = trimmed.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length == 0)
{
return null;
}
var nameVersion = segments[^1];
var atIndex = nameVersion.IndexOf('@');
return atIndex > 0 ? nameVersion[..atIndex] : nameVersion;
}
catch
{
@@ -411,6 +448,23 @@ public sealed class VexEvidenceGenerator : IVexEvidenceGenerator
return null;
}
private void EnsureAboveThreshold(BinaryVulnMatch match, FixStatusResult? fixStatus)
{
var effectiveConfidence = fixStatus?.Confidence ?? match.Confidence;
if (effectiveConfidence < _options.MinConfidenceThreshold)
{
_logger.LogDebug(
"Skipping observation for {CveId}: confidence {Confidence} below threshold {Threshold}",
match.CveId, effectiveConfidence, _options.MinConfidenceThreshold);
throw new InvalidOperationException(
$"Match confidence {effectiveConfidence} is below minimum threshold {_options.MinConfidenceThreshold}");
}
}
private bool IsBelowThreshold(BinaryVulnMatch match, FixStatusResult? fixStatus)
=> (fixStatus?.Confidence ?? match.Confidence) < _options.MinConfidenceThreshold;
private static string ComputeSha256(string content)
{
var bytes = Encoding.UTF8.GetBytes(content);