using System; using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Formats.Tar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Concelier.Exporter.Json; namespace StellaOps.Concelier.Exporter.TrivyDb; public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder { private readonly TrivyDbExportOptions _options; private readonly ILogger _logger; public TrivyDbBoltBuilder(IOptions options, ILogger logger) { _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task BuildAsync( JsonExportResult jsonTree, DateTimeOffset exportedAt, string exportId, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(jsonTree); ArgumentException.ThrowIfNullOrEmpty(exportId); var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId); var outputDir = Path.Combine(builderRoot, "out"); Directory.CreateDirectory(outputDir); try { await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false); } catch { TryDeleteDirectory(builderRoot); throw; } var metadataPath = Path.Combine(outputDir, "metadata.json"); var dbPath = Path.Combine(outputDir, "trivy.db"); if (!File.Exists(metadataPath)) { TryDeleteDirectory(builderRoot); throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'."); } if (!File.Exists(dbPath)) { TryDeleteDirectory(builderRoot); throw new InvalidOperationException($"trivy.db not found at '{dbPath}'."); } var archivePath = Path.Combine(builderRoot, "db.tar.gz"); await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false); var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false); var length = new FileInfo(archivePath).Length; var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false); return new TrivyDbBuilderResult( archivePath, digest, length, builderMetadata, builderRoot); } private string PrepareBuilderRoot(string exportDirectory, string exportId) { var root = Path.Combine(exportDirectory, $".builder-{exportId}"); if (Directory.Exists(root)) { Directory.Delete(root, recursive: true); } Directory.CreateDirectory(root); return root; } private static void TryDeleteDirectory(string directory) { try { if (Directory.Exists(directory)) { Directory.Delete(directory, recursive: true); } } catch { // ignore cleanup failures } } private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken) { var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions(); var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath) ? "trivy-db" : builderOptions.ExecutablePath; var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List(); var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); var startInfo = new ProcessStartInfo { FileName = executable, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, }; startInfo.ArgumentList.Add("build"); startInfo.ArgumentList.Add("--cache-dir"); startInfo.ArgumentList.Add(cacheDir); startInfo.ArgumentList.Add("--output-dir"); startInfo.ArgumentList.Add(outputDir); if (builderOptions.UpdateInterval != default) { startInfo.ArgumentList.Add("--update-interval"); startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval)); } if (targets.Count > 0) { foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t))) { startInfo.ArgumentList.Add("--only-update"); startInfo.ArgumentList.Add(target); } } if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory)) { startInfo.WorkingDirectory = builderOptions.WorkingDirectory; } if (!builderOptions.InheritEnvironment) { startInfo.Environment.Clear(); } foreach (var kvp in environment) { startInfo.Environment[kvp.Key] = kvp.Value; } using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false }; var stdOut = new StringBuilder(); var stdErr = new StringBuilder(); var stdoutCompletion = new TaskCompletionSource(); var stderrCompletion = new TaskCompletionSource(); process.OutputDataReceived += (_, e) => { if (e.Data is null) { stdoutCompletion.TrySetResult(null); } else { stdOut.AppendLine(e.Data); } }; process.ErrorDataReceived += (_, e) => { if (e.Data is null) { stderrCompletion.TrySetResult(null); } else { stdErr.AppendLine(e.Data); } }; _logger.LogInformation("Running {Executable} to build Trivy DB", executable); try { if (!process.Start()) { throw new InvalidOperationException($"Failed to start '{executable}'."); } } catch (Exception ex) { throw new InvalidOperationException($"Failed to start '{executable}'.", ex); } process.BeginOutputReadLine(); process.BeginErrorReadLine(); using var registration = cancellationToken.Register(() => { try { if (!process.HasExited) { process.Kill(entireProcessTree: true); } } catch { // Ignore kill failures. } }); #if NET8_0_OR_GREATER await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); #else await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); #endif await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); if (process.ExitCode != 0) { _logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString()); throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); } if (stdOut.Length > 0) { _logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString()); } if (stdErr.Length > 0) { _logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString()); } } private static async Task CreateArchiveAsync( string archivePath, DateTimeOffset exportedAt, string metadataPath, string dbPath, CancellationToken cancellationToken) { await using var archiveStream = new FileStream( archivePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan); await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true); await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false); var timestamp = exportedAt.UtcDateTime; foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath)) { cancellationToken.ThrowIfCancellationRequested(); var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name) { ModificationTime = timestamp, Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, }; await using var source = new FileStream( file.Path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan); entry.DataStream = source; writer.WriteEntry(entry); } await writer.DisposeAsync().ConfigureAwait(false); await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false); } private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath) { yield return ("metadata.json", metadataPath); yield return ("trivy.db", dbPath); } private static async Task ComputeDigestAsync(string archivePath, CancellationToken cancellationToken) { await using var stream = new FileStream( archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan); var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; } private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) { await using var stream = new FileStream( archivePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None, bufferSize: 8, options: FileOptions.Asynchronous); if (stream.Length < 10) { return; } stream.Position = 4; var zeros = new byte[4]; await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); } private static string ToGoDuration(TimeSpan span) { if (span <= TimeSpan.Zero) { return "0s"; } span = span.Duration(); var builder = new StringBuilder(); var totalHours = (int)span.TotalHours; if (totalHours > 0) { builder.Append(totalHours); builder.Append('h'); } var minutes = span.Minutes; if (minutes > 0) { builder.Append(minutes); builder.Append('m'); } var seconds = span.Seconds + span.Milliseconds / 1000.0; if (seconds > 0 || builder.Length == 0) { if (span.Milliseconds == 0) { builder.Append(span.Seconds); } else { builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture)); } builder.Append('s'); } return builder.ToString(); } }