Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,308 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ReproducibleBuildJob.cs
|
||||
// Sprint: SPRINT_1227_0002_0001_LB_reproducible_builders
|
||||
// Task: T10 — Implement ReproducibleBuildJob
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.BinaryIndex.Builders;
|
||||
using StellaOps.BinaryIndex.Persistence.Repositories;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Worker.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Background job that orchestrates reproducible builds for binary CVE attribution.
|
||||
/// Monitors advisory feeds, triggers builds, extracts fingerprints, and creates claims.
|
||||
/// </summary>
|
||||
public sealed class ReproducibleBuildJob : IReproducibleBuildJob
|
||||
{
|
||||
private readonly ILogger<ReproducibleBuildJob> _logger;
|
||||
private readonly ReproducibleBuildOptions _options;
|
||||
private readonly IEnumerable<IReproducibleBuilder> _builders;
|
||||
private readonly IFunctionFingerprintExtractor _fingerprintExtractor;
|
||||
private readonly IPatchDiffEngine _diffEngine;
|
||||
private readonly IFingerprintClaimRepository _claimRepository;
|
||||
private readonly IAdvisoryFeedMonitor _advisoryMonitor;
|
||||
|
||||
public ReproducibleBuildJob(
|
||||
ILogger<ReproducibleBuildJob> logger,
|
||||
IOptions<ReproducibleBuildOptions> options,
|
||||
IEnumerable<IReproducibleBuilder> builders,
|
||||
IFunctionFingerprintExtractor fingerprintExtractor,
|
||||
IPatchDiffEngine diffEngine,
|
||||
IFingerprintClaimRepository claimRepository,
|
||||
IAdvisoryFeedMonitor advisoryMonitor)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_builders = builders ?? throw new ArgumentNullException(nameof(builders));
|
||||
_fingerprintExtractor = fingerprintExtractor ?? throw new ArgumentNullException(nameof(fingerprintExtractor));
|
||||
_diffEngine = diffEngine ?? throw new ArgumentNullException(nameof(diffEngine));
|
||||
_claimRepository = claimRepository ?? throw new ArgumentNullException(nameof(claimRepository));
|
||||
_advisoryMonitor = advisoryMonitor ?? throw new ArgumentNullException(nameof(advisoryMonitor));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
_logger.LogInformation("Starting reproducible build job");
|
||||
|
||||
try
|
||||
{
|
||||
// Step 1: Get pending CVEs that need binary attribution
|
||||
var pendingCves = await _advisoryMonitor.GetPendingCvesAsync(ct);
|
||||
|
||||
_logger.LogInformation("Found {Count} CVEs pending binary attribution", pendingCves.Count);
|
||||
|
||||
foreach (var cve in pendingCves)
|
||||
{
|
||||
if (ct.IsCancellationRequested) break;
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessCveAsync(cve, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to process CVE {CveId}", cve.CveId);
|
||||
// Continue with next CVE
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Reproducible build job completed");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_logger.LogInformation("Reproducible build job cancelled");
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Reproducible build job failed");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ProcessCveAsync(CveAttribution cve, CancellationToken ct)
|
||||
{
|
||||
_logger.LogDebug("Processing CVE {CveId} for package {Package}", cve.CveId, cve.SourcePackage);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
// Find appropriate builder for distro
|
||||
var builder = _builders.FirstOrDefault(b =>
|
||||
b.Distro.Equals(cve.Distro, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (builder == null)
|
||||
{
|
||||
_logger.LogWarning("No builder available for distro {Distro}", cve.Distro);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build vulnerable version
|
||||
var vulnerableBuild = await BuildVersionAsync(builder, cve, cve.VulnerableVersion, ct);
|
||||
if (!vulnerableBuild.Success)
|
||||
{
|
||||
_logger.LogWarning("Failed to build vulnerable version {Version}", cve.VulnerableVersion);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build patched version
|
||||
var patchedBuild = await BuildVersionAsync(builder, cve, cve.FixedVersion, ct);
|
||||
if (!patchedBuild.Success)
|
||||
{
|
||||
_logger.LogWarning("Failed to build patched version {Version}", cve.FixedVersion);
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract function fingerprints from both builds
|
||||
var vulnerableFunctions = await ExtractFunctionsAsync(vulnerableBuild, ct);
|
||||
var patchedFunctions = await ExtractFunctionsAsync(patchedBuild, ct);
|
||||
|
||||
// Compute diff to identify changed functions
|
||||
var diff = _diffEngine.ComputeDiff(vulnerableFunctions, patchedFunctions);
|
||||
|
||||
_logger.LogDebug(
|
||||
"CVE {CveId}: {Modified} modified, {Added} added, {Removed} removed functions",
|
||||
cve.CveId, diff.ModifiedCount, diff.AddedCount, diff.RemovedCount);
|
||||
|
||||
// Create fingerprint claims
|
||||
await CreateClaimsAsync(cve, diff, vulnerableBuild, patchedBuild, ct);
|
||||
|
||||
stopwatch.Stop();
|
||||
_logger.LogInformation(
|
||||
"Processed CVE {CveId} in {Duration}ms",
|
||||
cve.CveId, stopwatch.ElapsedMilliseconds);
|
||||
}
|
||||
|
||||
private async Task<BuildResult> BuildVersionAsync(
|
||||
IReproducibleBuilder builder,
|
||||
CveAttribution cve,
|
||||
string version,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var request = new BuildRequest
|
||||
{
|
||||
SourcePackage = cve.SourcePackage,
|
||||
Version = version,
|
||||
Release = cve.Release,
|
||||
Architecture = _options.DefaultArchitecture,
|
||||
Options = new BuildOptions
|
||||
{
|
||||
Timeout = _options.BuildTimeout,
|
||||
CacheIntermediates = true
|
||||
}
|
||||
};
|
||||
|
||||
return await builder.BuildAsync(request, ct);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<FunctionFingerprint>> ExtractFunctionsAsync(
|
||||
BuildResult build,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var allFunctions = new List<FunctionFingerprint>();
|
||||
|
||||
foreach (var binary in build.Binaries ?? [])
|
||||
{
|
||||
if (binary.Functions != null)
|
||||
{
|
||||
allFunctions.AddRange(binary.Functions);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Extract if not already done during build
|
||||
var functions = await _fingerprintExtractor.ExtractAsync(
|
||||
binary.Path,
|
||||
new ExtractionOptions
|
||||
{
|
||||
IncludeInternalFunctions = false,
|
||||
IncludeCallGraph = true,
|
||||
MinFunctionSize = _options.MinFunctionSize
|
||||
},
|
||||
ct);
|
||||
|
||||
allFunctions.AddRange(functions);
|
||||
}
|
||||
}
|
||||
|
||||
return allFunctions;
|
||||
}
|
||||
|
||||
private async Task CreateClaimsAsync(
|
||||
CveAttribution cve,
|
||||
PatchDiffResult diff,
|
||||
BuildResult vulnerableBuild,
|
||||
BuildResult patchedBuild,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var claims = new List<FingerprintClaim>();
|
||||
|
||||
// Create "fixed" claims for patched binaries
|
||||
foreach (var binary in patchedBuild.Binaries ?? [])
|
||||
{
|
||||
var changedFunctions = diff.Changes
|
||||
.Where(c => c.Type is ChangeType.Modified or ChangeType.Added)
|
||||
.Select(c => c.FunctionName)
|
||||
.ToList();
|
||||
|
||||
var claim = new FingerprintClaim
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FingerprintId = Guid.Parse(binary.BuildId), // Assuming BuildId is GUID-like
|
||||
CveId = cve.CveId,
|
||||
Verdict = ClaimVerdict.Fixed,
|
||||
Evidence = new FingerprintClaimEvidence
|
||||
{
|
||||
PatchCommit = cve.PatchCommit ?? "unknown",
|
||||
ChangedFunctions = changedFunctions,
|
||||
FunctionSimilarities = diff.Changes
|
||||
.Where(c => c.SimilarityScore.HasValue)
|
||||
.ToDictionary(c => c.FunctionName, c => c.SimilarityScore!.Value),
|
||||
VulnerableBuildRef = vulnerableBuild.BuildLogRef,
|
||||
PatchedBuildRef = patchedBuild.BuildLogRef
|
||||
},
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
claims.Add(claim);
|
||||
}
|
||||
|
||||
// Create "vulnerable" claims for vulnerable binaries
|
||||
foreach (var binary in vulnerableBuild.Binaries ?? [])
|
||||
{
|
||||
var claim = new FingerprintClaim
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FingerprintId = Guid.Parse(binary.BuildId),
|
||||
CveId = cve.CveId,
|
||||
Verdict = ClaimVerdict.Vulnerable,
|
||||
Evidence = new FingerprintClaimEvidence
|
||||
{
|
||||
PatchCommit = cve.PatchCommit ?? "unknown",
|
||||
ChangedFunctions = diff.Changes
|
||||
.Where(c => c.Type == ChangeType.Modified)
|
||||
.Select(c => c.FunctionName)
|
||||
.ToList(),
|
||||
VulnerableBuildRef = vulnerableBuild.BuildLogRef
|
||||
},
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
claims.Add(claim);
|
||||
}
|
||||
|
||||
await _claimRepository.CreateClaimsBatchAsync(claims, ct);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Created {Count} fingerprint claims for CVE {CveId}",
|
||||
claims.Count, cve.CveId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface for the reproducible build job.
|
||||
/// </summary>
|
||||
public interface IReproducibleBuildJob
|
||||
{
|
||||
Task ExecuteAsync(CancellationToken ct);
|
||||
Task ProcessCveAsync(CveAttribution cve, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// CVE attribution request.
|
||||
/// </summary>
|
||||
public sealed record CveAttribution
|
||||
{
|
||||
public required string CveId { get; init; }
|
||||
public required string SourcePackage { get; init; }
|
||||
public required string Distro { get; init; }
|
||||
public required string Release { get; init; }
|
||||
public required string VulnerableVersion { get; init; }
|
||||
public required string FixedVersion { get; init; }
|
||||
public string? PatchCommit { get; init; }
|
||||
public string? AdvisoryId { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Advisory feed monitor interface.
|
||||
/// </summary>
|
||||
public interface IAdvisoryFeedMonitor
|
||||
{
|
||||
Task<IReadOnlyList<CveAttribution>> GetPendingCvesAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for reproducible builds.
|
||||
/// </summary>
|
||||
public sealed class ReproducibleBuildOptions
|
||||
{
|
||||
public TimeSpan BuildTimeout { get; set; } = TimeSpan.FromMinutes(30);
|
||||
public string DefaultArchitecture { get; set; } = "amd64";
|
||||
public int MinFunctionSize { get; set; } = 16;
|
||||
public int MaxConcurrentBuilds { get; set; } = 2;
|
||||
public string BuildCacheDirectory { get; set; } = "/var/cache/stellaops/builds";
|
||||
}
|
||||
Reference in New Issue
Block a user