377 lines
12 KiB
C#
377 lines
12 KiB
C#
using System;
|
|
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using System.Formats.Tar;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using StellaOps.Concelier.Exporter.Json;
|
|
|
|
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
|
|
|
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
|
|
{
|
|
private readonly TrivyDbExportOptions _options;
|
|
private readonly ILogger<TrivyDbBoltBuilder> _logger;
|
|
|
|
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
|
|
{
|
|
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
|
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
|
}
|
|
|
|
public async Task<TrivyDbBuilderResult> BuildAsync(
|
|
JsonExportResult jsonTree,
|
|
DateTimeOffset exportedAt,
|
|
string exportId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(jsonTree);
|
|
ArgumentException.ThrowIfNullOrEmpty(exportId);
|
|
|
|
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
|
|
var outputDir = Path.Combine(builderRoot, "out");
|
|
Directory.CreateDirectory(outputDir);
|
|
|
|
try
|
|
{
|
|
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch
|
|
{
|
|
TryDeleteDirectory(builderRoot);
|
|
throw;
|
|
}
|
|
|
|
var metadataPath = Path.Combine(outputDir, "metadata.json");
|
|
var dbPath = Path.Combine(outputDir, "trivy.db");
|
|
|
|
if (!File.Exists(metadataPath))
|
|
{
|
|
TryDeleteDirectory(builderRoot);
|
|
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
|
|
}
|
|
|
|
if (!File.Exists(dbPath))
|
|
{
|
|
TryDeleteDirectory(builderRoot);
|
|
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
|
|
}
|
|
|
|
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
|
|
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
|
|
|
|
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
|
var length = new FileInfo(archivePath).Length;
|
|
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
|
|
|
return new TrivyDbBuilderResult(
|
|
archivePath,
|
|
digest,
|
|
length,
|
|
builderMetadata,
|
|
builderRoot);
|
|
}
|
|
|
|
private string PrepareBuilderRoot(string exportDirectory, string exportId)
|
|
{
|
|
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
|
|
if (Directory.Exists(root))
|
|
{
|
|
Directory.Delete(root, recursive: true);
|
|
}
|
|
|
|
Directory.CreateDirectory(root);
|
|
return root;
|
|
}
|
|
|
|
private static void TryDeleteDirectory(string directory)
|
|
{
|
|
try
|
|
{
|
|
if (Directory.Exists(directory))
|
|
{
|
|
Directory.Delete(directory, recursive: true);
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
// ignore cleanup failures
|
|
}
|
|
}
|
|
|
|
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
|
|
{
|
|
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
|
|
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
|
|
? "trivy-db"
|
|
: builderOptions.ExecutablePath;
|
|
|
|
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
|
|
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var startInfo = new ProcessStartInfo
|
|
{
|
|
FileName = executable,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
UseShellExecute = false,
|
|
};
|
|
|
|
startInfo.ArgumentList.Add("build");
|
|
startInfo.ArgumentList.Add("--cache-dir");
|
|
startInfo.ArgumentList.Add(cacheDir);
|
|
startInfo.ArgumentList.Add("--output-dir");
|
|
startInfo.ArgumentList.Add(outputDir);
|
|
|
|
if (builderOptions.UpdateInterval != default)
|
|
{
|
|
startInfo.ArgumentList.Add("--update-interval");
|
|
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
|
|
}
|
|
|
|
if (targets.Count > 0)
|
|
{
|
|
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
|
|
{
|
|
startInfo.ArgumentList.Add("--only-update");
|
|
startInfo.ArgumentList.Add(target);
|
|
}
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
|
|
{
|
|
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
|
|
}
|
|
|
|
if (!builderOptions.InheritEnvironment)
|
|
{
|
|
startInfo.Environment.Clear();
|
|
}
|
|
|
|
foreach (var kvp in environment)
|
|
{
|
|
startInfo.Environment[kvp.Key] = kvp.Value;
|
|
}
|
|
|
|
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
|
|
|
|
var stdOut = new StringBuilder();
|
|
var stdErr = new StringBuilder();
|
|
|
|
var stdoutCompletion = new TaskCompletionSource<object?>();
|
|
var stderrCompletion = new TaskCompletionSource<object?>();
|
|
|
|
process.OutputDataReceived += (_, e) =>
|
|
{
|
|
if (e.Data is null)
|
|
{
|
|
stdoutCompletion.TrySetResult(null);
|
|
}
|
|
else
|
|
{
|
|
stdOut.AppendLine(e.Data);
|
|
}
|
|
};
|
|
|
|
process.ErrorDataReceived += (_, e) =>
|
|
{
|
|
if (e.Data is null)
|
|
{
|
|
stderrCompletion.TrySetResult(null);
|
|
}
|
|
else
|
|
{
|
|
stdErr.AppendLine(e.Data);
|
|
}
|
|
};
|
|
|
|
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
|
|
|
|
try
|
|
{
|
|
if (!process.Start())
|
|
{
|
|
throw new InvalidOperationException($"Failed to start '{executable}'.");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
|
|
}
|
|
|
|
process.BeginOutputReadLine();
|
|
process.BeginErrorReadLine();
|
|
|
|
using var registration = cancellationToken.Register(() =>
|
|
{
|
|
try
|
|
{
|
|
if (!process.HasExited)
|
|
{
|
|
process.Kill(entireProcessTree: true);
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
// Ignore kill failures.
|
|
}
|
|
});
|
|
|
|
#if NET8_0_OR_GREATER
|
|
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
|
|
#else
|
|
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
|
|
#endif
|
|
|
|
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
|
|
|
|
if (process.ExitCode != 0)
|
|
{
|
|
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
|
|
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
|
|
}
|
|
|
|
if (stdOut.Length > 0)
|
|
{
|
|
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
|
|
}
|
|
|
|
if (stdErr.Length > 0)
|
|
{
|
|
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
|
|
}
|
|
}
|
|
|
|
private static async Task CreateArchiveAsync(
|
|
string archivePath,
|
|
DateTimeOffset exportedAt,
|
|
string metadataPath,
|
|
string dbPath,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var archiveStream = new FileStream(
|
|
archivePath,
|
|
FileMode.Create,
|
|
FileAccess.Write,
|
|
FileShare.None,
|
|
bufferSize: 81920,
|
|
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
|
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
|
|
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
|
|
|
|
var timestamp = exportedAt.UtcDateTime;
|
|
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
|
|
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
|
|
{
|
|
ModificationTime = timestamp,
|
|
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
|
|
};
|
|
|
|
await using var source = new FileStream(
|
|
file.Path,
|
|
FileMode.Open,
|
|
FileAccess.Read,
|
|
FileShare.Read,
|
|
bufferSize: 81920,
|
|
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
|
entry.DataStream = source;
|
|
writer.WriteEntry(entry);
|
|
}
|
|
|
|
await writer.DisposeAsync().ConfigureAwait(false);
|
|
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
|
|
{
|
|
yield return ("metadata.json", metadataPath);
|
|
yield return ("trivy.db", dbPath);
|
|
}
|
|
|
|
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
|
|
{
|
|
await using var stream = new FileStream(
|
|
archivePath,
|
|
FileMode.Open,
|
|
FileAccess.Read,
|
|
FileShare.Read,
|
|
bufferSize: 81920,
|
|
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
|
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
|
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
|
}
|
|
|
|
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
|
{
|
|
await using var stream = new FileStream(
|
|
archivePath,
|
|
FileMode.Open,
|
|
FileAccess.ReadWrite,
|
|
FileShare.None,
|
|
bufferSize: 8,
|
|
options: FileOptions.Asynchronous);
|
|
|
|
if (stream.Length < 10)
|
|
{
|
|
return;
|
|
}
|
|
|
|
stream.Position = 4;
|
|
var zeros = new byte[4];
|
|
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
|
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private static string ToGoDuration(TimeSpan span)
|
|
{
|
|
if (span <= TimeSpan.Zero)
|
|
{
|
|
return "0s";
|
|
}
|
|
|
|
span = span.Duration();
|
|
var builder = new StringBuilder();
|
|
|
|
var totalHours = (int)span.TotalHours;
|
|
if (totalHours > 0)
|
|
{
|
|
builder.Append(totalHours);
|
|
builder.Append('h');
|
|
}
|
|
|
|
var minutes = span.Minutes;
|
|
if (minutes > 0)
|
|
{
|
|
builder.Append(minutes);
|
|
builder.Append('m');
|
|
}
|
|
|
|
var seconds = span.Seconds + span.Milliseconds / 1000.0;
|
|
if (seconds > 0 || builder.Length == 0)
|
|
{
|
|
if (span.Milliseconds == 0)
|
|
{
|
|
builder.Append(span.Seconds);
|
|
}
|
|
else
|
|
{
|
|
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
|
|
}
|
|
builder.Append('s');
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
}
|