Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,376 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Formats.Tar;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
|
||||
{
|
||||
private readonly TrivyDbExportOptions _options;
|
||||
private readonly ILogger<TrivyDbBoltBuilder> _logger;
|
||||
|
||||
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
|
||||
{
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<TrivyDbBuilderResult> BuildAsync(
|
||||
JsonExportResult jsonTree,
|
||||
DateTimeOffset exportedAt,
|
||||
string exportId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jsonTree);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exportId);
|
||||
|
||||
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
|
||||
var outputDir = Path.Combine(builderRoot, "out");
|
||||
Directory.CreateDirectory(outputDir);
|
||||
|
||||
try
|
||||
{
|
||||
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw;
|
||||
}
|
||||
|
||||
var metadataPath = Path.Combine(outputDir, "metadata.json");
|
||||
var dbPath = Path.Combine(outputDir, "trivy.db");
|
||||
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
|
||||
}
|
||||
|
||||
if (!File.Exists(dbPath))
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
|
||||
}
|
||||
|
||||
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
|
||||
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
||||
var length = new FileInfo(archivePath).Length;
|
||||
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new TrivyDbBuilderResult(
|
||||
archivePath,
|
||||
digest,
|
||||
length,
|
||||
builderMetadata,
|
||||
builderRoot);
|
||||
}
|
||||
|
||||
private string PrepareBuilderRoot(string exportDirectory, string exportId)
|
||||
{
|
||||
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
|
||||
if (Directory.Exists(root))
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore cleanup failures
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
|
||||
{
|
||||
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
|
||||
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
|
||||
? "trivy-db"
|
||||
: builderOptions.ExecutablePath;
|
||||
|
||||
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
|
||||
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = executable,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
|
||||
startInfo.ArgumentList.Add("build");
|
||||
startInfo.ArgumentList.Add("--cache-dir");
|
||||
startInfo.ArgumentList.Add(cacheDir);
|
||||
startInfo.ArgumentList.Add("--output-dir");
|
||||
startInfo.ArgumentList.Add(outputDir);
|
||||
|
||||
if (builderOptions.UpdateInterval != default)
|
||||
{
|
||||
startInfo.ArgumentList.Add("--update-interval");
|
||||
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
|
||||
}
|
||||
|
||||
if (targets.Count > 0)
|
||||
{
|
||||
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
|
||||
{
|
||||
startInfo.ArgumentList.Add("--only-update");
|
||||
startInfo.ArgumentList.Add(target);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
|
||||
{
|
||||
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
|
||||
}
|
||||
|
||||
if (!builderOptions.InheritEnvironment)
|
||||
{
|
||||
startInfo.Environment.Clear();
|
||||
}
|
||||
|
||||
foreach (var kvp in environment)
|
||||
{
|
||||
startInfo.Environment[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
|
||||
|
||||
var stdOut = new StringBuilder();
|
||||
var stdErr = new StringBuilder();
|
||||
|
||||
var stdoutCompletion = new TaskCompletionSource<object?>();
|
||||
var stderrCompletion = new TaskCompletionSource<object?>();
|
||||
|
||||
process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stdoutCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stdOut.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stderrCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stdErr.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
|
||||
|
||||
try
|
||||
{
|
||||
if (!process.Start())
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
|
||||
}
|
||||
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
using var registration = cancellationToken.Register(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore kill failures.
|
||||
}
|
||||
});
|
||||
|
||||
#if NET8_0_OR_GREATER
|
||||
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
|
||||
#else
|
||||
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
|
||||
#endif
|
||||
|
||||
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
|
||||
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
|
||||
}
|
||||
|
||||
if (stdOut.Length > 0)
|
||||
{
|
||||
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
|
||||
}
|
||||
|
||||
if (stdErr.Length > 0)
|
||||
{
|
||||
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task CreateArchiveAsync(
|
||||
string archivePath,
|
||||
DateTimeOffset exportedAt,
|
||||
string metadataPath,
|
||||
string dbPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var archiveStream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
|
||||
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
|
||||
|
||||
var timestamp = exportedAt.UtcDateTime;
|
||||
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
|
||||
};
|
||||
|
||||
await using var source = new FileStream(
|
||||
file.Path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
entry.DataStream = source;
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
|
||||
await writer.DisposeAsync().ConfigureAwait(false);
|
||||
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
|
||||
{
|
||||
yield return ("metadata.json", metadataPath);
|
||||
yield return ("trivy.db", dbPath);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 8,
|
||||
options: FileOptions.Asynchronous);
|
||||
|
||||
if (stream.Length < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
stream.Position = 4;
|
||||
var zeros = new byte[4];
|
||||
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string ToGoDuration(TimeSpan span)
|
||||
{
|
||||
if (span <= TimeSpan.Zero)
|
||||
{
|
||||
return "0s";
|
||||
}
|
||||
|
||||
span = span.Duration();
|
||||
var builder = new StringBuilder();
|
||||
|
||||
var totalHours = (int)span.TotalHours;
|
||||
if (totalHours > 0)
|
||||
{
|
||||
builder.Append(totalHours);
|
||||
builder.Append('h');
|
||||
}
|
||||
|
||||
var minutes = span.Minutes;
|
||||
if (minutes > 0)
|
||||
{
|
||||
builder.Append(minutes);
|
||||
builder.Append('m');
|
||||
}
|
||||
|
||||
var seconds = span.Seconds + span.Milliseconds / 1000.0;
|
||||
if (seconds > 0 || builder.Length == 0)
|
||||
{
|
||||
if (span.Milliseconds == 0)
|
||||
{
|
||||
builder.Append(span.Seconds);
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
|
||||
}
|
||||
builder.Append('s');
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user