up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,15 +1,15 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbBuilder
{
Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbBuilder
{
Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken);
}

View File

@@ -1,9 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbOrasPusher
{
Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken);
}
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbOrasPusher
{
Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken);
}

View File

@@ -1,10 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciDescriptor(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations = null);
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciDescriptor(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations = null);

View File

@@ -1,8 +1,8 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciIndex(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("manifests")] IReadOnlyList<OciDescriptor> Manifests);
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciIndex(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("manifests")] IReadOnlyList<OciDescriptor> Manifests);

View File

@@ -1,10 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciManifest(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("config")] OciDescriptor Config,
[property: JsonPropertyName("layers")] IReadOnlyList<OciDescriptor> Layers);
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciManifest(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("config")] OciDescriptor Config,
[property: JsonPropertyName("layers")] IReadOnlyList<OciDescriptor> Layers);

View File

@@ -1,11 +1,11 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyConfigDocument(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
[property: JsonPropertyName("databaseVersion")] string DatabaseVersion,
[property: JsonPropertyName("databaseDigest")] string DatabaseDigest,
[property: JsonPropertyName("databaseSize")] long DatabaseSize);
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyConfigDocument(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
[property: JsonPropertyName("databaseVersion")] string DatabaseVersion,
[property: JsonPropertyName("databaseDigest")] string DatabaseDigest,
[property: JsonPropertyName("databaseSize")] long DatabaseSize);

View File

@@ -1,78 +1,78 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBlob
{
private readonly Func<CancellationToken, ValueTask<Stream>> _openReadAsync;
private TrivyDbBlob(Func<CancellationToken, ValueTask<Stream>> openReadAsync, long length)
{
_openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync));
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Length = length;
}
public long Length { get; }
public ValueTask<Stream> OpenReadAsync(CancellationToken cancellationToken)
=> _openReadAsync(cancellationToken);
public static TrivyDbBlob FromBytes(ReadOnlyMemory<byte> payload)
{
if (payload.IsEmpty)
{
return new TrivyDbBlob(static _ => ValueTask.FromResult<Stream>(Stream.Null), 0);
}
if (MemoryMarshal.TryGetArray(payload, out ArraySegment<byte> segment) && segment.Array is not null && segment.Offset == 0)
{
return FromArray(segment.Array);
}
return FromArray(payload.ToArray());
}
public static TrivyDbBlob FromFile(string path, long length)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("File path must be provided.", nameof(path));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
return new TrivyDbBlob(
cancellationToken => ValueTask.FromResult<Stream>(new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan)),
length);
}
public static TrivyDbBlob FromArray(byte[] buffer)
{
if (buffer is null)
{
throw new ArgumentNullException(nameof(buffer));
}
return new TrivyDbBlob(
_ => ValueTask.FromResult<Stream>(new MemoryStream(buffer, writable: false)),
buffer.LongLength);
}
}
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBlob
{
private readonly Func<CancellationToken, ValueTask<Stream>> _openReadAsync;
private TrivyDbBlob(Func<CancellationToken, ValueTask<Stream>> openReadAsync, long length)
{
_openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync));
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Length = length;
}
public long Length { get; }
public ValueTask<Stream> OpenReadAsync(CancellationToken cancellationToken)
=> _openReadAsync(cancellationToken);
public static TrivyDbBlob FromBytes(ReadOnlyMemory<byte> payload)
{
if (payload.IsEmpty)
{
return new TrivyDbBlob(static _ => ValueTask.FromResult<Stream>(Stream.Null), 0);
}
if (MemoryMarshal.TryGetArray(payload, out ArraySegment<byte> segment) && segment.Array is not null && segment.Offset == 0)
{
return FromArray(segment.Array);
}
return FromArray(payload.ToArray());
}
public static TrivyDbBlob FromFile(string path, long length)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("File path must be provided.", nameof(path));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
return new TrivyDbBlob(
cancellationToken => ValueTask.FromResult<Stream>(new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan)),
length);
}
public static TrivyDbBlob FromArray(byte[] buffer)
{
if (buffer is null)
{
throw new ArgumentNullException(nameof(buffer));
}
return new TrivyDbBlob(
_ => ValueTask.FromResult<Stream>(new MemoryStream(buffer, writable: false)),
buffer.LongLength);
}
}

View File

@@ -1,376 +1,376 @@
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbBoltBuilder> _logger;
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(jsonTree);
ArgumentException.ThrowIfNullOrEmpty(exportId);
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
var outputDir = Path.Combine(builderRoot, "out");
Directory.CreateDirectory(outputDir);
try
{
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
}
catch
{
TryDeleteDirectory(builderRoot);
throw;
}
var metadataPath = Path.Combine(outputDir, "metadata.json");
var dbPath = Path.Combine(outputDir, "trivy.db");
if (!File.Exists(metadataPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
}
if (!File.Exists(dbPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
}
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(archivePath).Length;
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
return new TrivyDbBuilderResult(
archivePath,
digest,
length,
builderMetadata,
builderRoot);
}
private string PrepareBuilderRoot(string exportDirectory, string exportId)
{
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
return root;
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// ignore cleanup failures
}
}
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
{
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
? "trivy-db"
: builderOptions.ExecutablePath;
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("build");
startInfo.ArgumentList.Add("--cache-dir");
startInfo.ArgumentList.Add(cacheDir);
startInfo.ArgumentList.Add("--output-dir");
startInfo.ArgumentList.Add(outputDir);
if (builderOptions.UpdateInterval != default)
{
startInfo.ArgumentList.Add("--update-interval");
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
}
if (targets.Count > 0)
{
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
{
startInfo.ArgumentList.Add("--only-update");
startInfo.ArgumentList.Add(target);
}
}
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
}
if (!builderOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
foreach (var kvp in environment)
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
var stdOut = new StringBuilder();
var stdErr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdOut.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stdErr.AppendLine(e.Data);
}
};
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// Ignore kill failures.
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdOut.Length > 0)
{
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
}
if (stdErr.Length > 0)
{
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
}
}
private static async Task CreateArchiveAsync(
string archivePath,
DateTimeOffset exportedAt,
string metadataPath,
string dbPath,
CancellationToken cancellationToken)
{
await using var archiveStream = new FileStream(
archivePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
var timestamp = exportedAt.UtcDateTime;
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
};
await using var source = new FileStream(
file.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
await writer.DisposeAsync().ConfigureAwait(false);
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
}
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
{
yield return ("metadata.json", metadataPath);
yield return ("trivy.db", dbPath);
}
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static string ToGoDuration(TimeSpan span)
{
if (span <= TimeSpan.Zero)
{
return "0s";
}
span = span.Duration();
var builder = new StringBuilder();
var totalHours = (int)span.TotalHours;
if (totalHours > 0)
{
builder.Append(totalHours);
builder.Append('h');
}
var minutes = span.Minutes;
if (minutes > 0)
{
builder.Append(minutes);
builder.Append('m');
}
var seconds = span.Seconds + span.Milliseconds / 1000.0;
if (seconds > 0 || builder.Length == 0)
{
if (span.Milliseconds == 0)
{
builder.Append(span.Seconds);
}
else
{
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
}
builder.Append('s');
}
return builder.ToString();
}
}
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbBoltBuilder> _logger;
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(jsonTree);
ArgumentException.ThrowIfNullOrEmpty(exportId);
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
var outputDir = Path.Combine(builderRoot, "out");
Directory.CreateDirectory(outputDir);
try
{
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
}
catch
{
TryDeleteDirectory(builderRoot);
throw;
}
var metadataPath = Path.Combine(outputDir, "metadata.json");
var dbPath = Path.Combine(outputDir, "trivy.db");
if (!File.Exists(metadataPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
}
if (!File.Exists(dbPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
}
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(archivePath).Length;
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
return new TrivyDbBuilderResult(
archivePath,
digest,
length,
builderMetadata,
builderRoot);
}
private string PrepareBuilderRoot(string exportDirectory, string exportId)
{
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
return root;
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// ignore cleanup failures
}
}
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
{
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
? "trivy-db"
: builderOptions.ExecutablePath;
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("build");
startInfo.ArgumentList.Add("--cache-dir");
startInfo.ArgumentList.Add(cacheDir);
startInfo.ArgumentList.Add("--output-dir");
startInfo.ArgumentList.Add(outputDir);
if (builderOptions.UpdateInterval != default)
{
startInfo.ArgumentList.Add("--update-interval");
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
}
if (targets.Count > 0)
{
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
{
startInfo.ArgumentList.Add("--only-update");
startInfo.ArgumentList.Add(target);
}
}
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
}
if (!builderOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
foreach (var kvp in environment)
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
var stdOut = new StringBuilder();
var stdErr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdOut.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stdErr.AppendLine(e.Data);
}
};
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// Ignore kill failures.
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdOut.Length > 0)
{
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
}
if (stdErr.Length > 0)
{
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
}
}
private static async Task CreateArchiveAsync(
string archivePath,
DateTimeOffset exportedAt,
string metadataPath,
string dbPath,
CancellationToken cancellationToken)
{
await using var archiveStream = new FileStream(
archivePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
var timestamp = exportedAt.UtcDateTime;
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
};
await using var source = new FileStream(
file.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
await writer.DisposeAsync().ConfigureAwait(false);
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
}
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
{
yield return ("metadata.json", metadataPath);
yield return ("trivy.db", dbPath);
}
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static string ToGoDuration(TimeSpan span)
{
if (span <= TimeSpan.Zero)
{
return "0s";
}
span = span.Duration();
var builder = new StringBuilder();
var totalHours = (int)span.TotalHours;
if (totalHours > 0)
{
builder.Append(totalHours);
builder.Append('h');
}
var minutes = span.Minutes;
if (minutes > 0)
{
builder.Append(minutes);
builder.Append('m');
}
var seconds = span.Seconds + span.Milliseconds / 1000.0;
if (seconds > 0 || builder.Length == 0)
{
if (span.Milliseconds == 0)
{
builder.Append(span.Seconds);
}
else
{
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
}
builder.Append('s');
}
return builder.ToString();
}
}

View File

@@ -1,10 +1,10 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbBuilderResult(
string ArchivePath,
string ArchiveDigest,
long ArchiveLength,
ReadOnlyMemory<byte> BuilderMetadata,
string WorkingDirectory);
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbBuilderResult(
string ArchivePath,
string ArchiveDigest,
long ArchiveLength,
ReadOnlyMemory<byte> BuilderMetadata,
string WorkingDirectory);

View File

@@ -1,94 +1,94 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportJob : IJob
{
public const string JobKind = "export:trivy-db";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10);
private readonly TrivyDbFeedExporter _exporter;
private readonly ILogger<TrivyDbExportJob> _logger;
public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger<TrivyDbExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId);
var overrides = CreateOverrides(context.Parameters);
if (overrides?.HasOverrides == true)
{
using var scope = TrivyDbExportOverrideScope.Begin(overrides);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
else
{
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId);
}
private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary<string, object?> parameters)
{
if (parameters is null || parameters.Count == 0)
{
return null;
}
var publishFull = TryReadBoolean(parameters, "publishFull");
var publishDelta = TryReadBoolean(parameters, "publishDelta");
var includeFull = TryReadBoolean(parameters, "includeFull");
var includeDelta = TryReadBoolean(parameters, "includeDelta");
var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta);
return overrides.HasOverrides ? overrides : null;
}
private static bool? TryReadBoolean(IReadOnlyDictionary<string, object?> parameters, string key)
{
if (!parameters.TryGetValue(key, out var value) || value is null)
{
return null;
}
switch (value)
{
case bool b:
return b;
case string s when bool.TryParse(s, out var result):
return result;
case JsonElement element:
return element.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed,
_ => null,
};
case IConvertible convertible:
try
{
return convertible.ToBoolean(CultureInfo.InvariantCulture);
}
catch
{
return null;
}
}
return null;
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportJob : IJob
{
public const string JobKind = "export:trivy-db";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10);
private readonly TrivyDbFeedExporter _exporter;
private readonly ILogger<TrivyDbExportJob> _logger;
public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger<TrivyDbExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId);
var overrides = CreateOverrides(context.Parameters);
if (overrides?.HasOverrides == true)
{
using var scope = TrivyDbExportOverrideScope.Begin(overrides);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
else
{
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId);
}
private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary<string, object?> parameters)
{
if (parameters is null || parameters.Count == 0)
{
return null;
}
var publishFull = TryReadBoolean(parameters, "publishFull");
var publishDelta = TryReadBoolean(parameters, "publishDelta");
var includeFull = TryReadBoolean(parameters, "includeFull");
var includeDelta = TryReadBoolean(parameters, "includeDelta");
var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta);
return overrides.HasOverrides ? overrides : null;
}
private static bool? TryReadBoolean(IReadOnlyDictionary<string, object?> parameters, string key)
{
if (!parameters.TryGetValue(key, out var value) || value is null)
{
return null;
}
switch (value)
{
case bool b:
return b;
case string s when bool.TryParse(s, out var result):
return result;
case JsonElement element:
return element.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed,
_ => null,
};
case IConvertible convertible:
try
{
return convertible.ToBoolean(CultureInfo.InvariantCulture);
}
catch
{
return null;
}
}
return null;
}
}

View File

@@ -1,8 +1,8 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public enum TrivyDbExportMode
{
Full,
Delta,
Skip,
}
namespace StellaOps.Concelier.Exporter.TrivyDb;
public enum TrivyDbExportMode
{
Full,
Delta,
Skip,
}

View File

@@ -1,29 +1,29 @@
using System;
using System.IO;
using System.Collections.Generic;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportOptions
{
public string OutputRoot { get; set; } = Path.Combine("exports", "trivy");
public string ReferencePrefix { get; set; } = "concelier/trivy";
public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public bool KeepWorkingTree { get; set; }
public string? TargetRepository { get; set; }
public JsonExportOptions Json { get; set; } = new()
{
OutputRoot = Path.Combine("exports", "trivy", "tree")
};
using System;
using System.IO;
using System.Collections.Generic;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportOptions
{
public string OutputRoot { get; set; } = Path.Combine("exports", "trivy");
public string ReferencePrefix { get; set; } = "concelier/trivy";
public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public bool KeepWorkingTree { get; set; }
public string? TargetRepository { get; set; }
public JsonExportOptions Json { get; set; } = new()
{
OutputRoot = Path.Combine("exports", "trivy", "tree")
};
public TrivyDbBuilderOptions Builder { get; set; } = new();
public TrivyDbOrasOptions Oras { get; set; } = new();
@@ -61,16 +61,16 @@ public sealed class TrivyDbBuilderOptions
public string ExecutablePath { get; set; } = "trivy-db";
public string? WorkingDirectory { get; set; }
public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24);
public List<string> OnlyUpdateTargets { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool InheritEnvironment { get; set; } = true;
}
public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24);
public List<string> OnlyUpdateTargets { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool InheritEnvironment { get; set; } = true;
}
public sealed class TrivyDbOrasOptions
{
public bool Enabled { get; set; }
@@ -84,16 +84,16 @@ public sealed class TrivyDbOrasOptions
public string? WorkingDirectory { get; set; }
public bool InheritEnvironment { get; set; } = true;
public List<string> AdditionalArguments { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool SkipTlsVerify { get; set; }
public bool UseHttp { get; set; }
}
public List<string> AdditionalArguments { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool SkipTlsVerify { get; set; }
public bool UseHttp { get; set; }
}
public sealed class TrivyDbOfflineBundleOptions
{
public bool Enabled { get; set; }

View File

@@ -1,50 +1,50 @@
using System;
using System.Threading;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal sealed record TrivyDbExportOverrides(
bool? PublishFull,
bool? PublishDelta,
bool? IncludeFull,
bool? IncludeDelta)
{
public bool HasOverrides =>
PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue;
}
internal static class TrivyDbExportOverrideScope
{
private sealed class Scope : IDisposable
{
private readonly TrivyDbExportOverrides? _previous;
private bool _disposed;
public Scope(TrivyDbExportOverrides? previous)
{
_previous = previous;
}
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
CurrentOverrides.Value = _previous;
}
}
private static readonly AsyncLocal<TrivyDbExportOverrides?> CurrentOverrides = new();
public static TrivyDbExportOverrides? Current => CurrentOverrides.Value;
public static IDisposable Begin(TrivyDbExportOverrides overrides)
{
var previous = CurrentOverrides.Value;
CurrentOverrides.Value = overrides;
return new Scope(previous);
}
}
using System;
using System.Threading;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal sealed record TrivyDbExportOverrides(
bool? PublishFull,
bool? PublishDelta,
bool? IncludeFull,
bool? IncludeDelta)
{
public bool HasOverrides =>
PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue;
}
internal static class TrivyDbExportOverrideScope
{
private sealed class Scope : IDisposable
{
private readonly TrivyDbExportOverrides? _previous;
private bool _disposed;
public Scope(TrivyDbExportOverrides? previous)
{
_previous = previous;
}
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
CurrentOverrides.Value = _previous;
}
}
private static readonly AsyncLocal<TrivyDbExportOverrides?> CurrentOverrides = new();
public static TrivyDbExportOverrides? Current => CurrentOverrides.Value;
public static IDisposable Begin(TrivyDbExportOverrides overrides)
{
var previous = CurrentOverrides.Value;
CurrentOverrides.Value = overrides;
return new Scope(previous);
}
}

View File

@@ -1,14 +1,14 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System.Collections.Generic;
using StellaOps.Concelier.Storage.Exporting;
public sealed record TrivyDbExportPlan(
TrivyDbExportMode Mode,
string TreeDigest,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
IReadOnlyList<ExportFileRecord> Manifest,
IReadOnlyList<ExportFileRecord> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System.Collections.Generic;
using StellaOps.Concelier.Storage.Exporting;
public sealed record TrivyDbExportPlan(
TrivyDbExportMode Mode,
string TreeDigest,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
IReadOnlyList<ExportFileRecord> Manifest,
IReadOnlyList<ExportFileRecord> ChangedFiles,
IReadOnlyList<string> RemovedPaths);

View File

@@ -1,115 +1,115 @@
using System;
using StellaOps.Concelier.Storage.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Storage.Exporting;
public sealed class TrivyDbExportPlanner
{
public TrivyDbExportPlan CreatePlan(
ExportStateRecord? existingState,
string treeDigest,
IReadOnlyList<ExportFileRecord> manifest)
{
ArgumentException.ThrowIfNullOrEmpty(treeDigest);
manifest ??= Array.Empty<ExportFileRecord>();
if (existingState is null || (existingState.Files?.Count ?? 0) == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
BaseExportId: existingState?.BaseExportId,
BaseManifestDigest: existingState?.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>();
var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal);
if (cursorMatches)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var removed = existingMap.Keys
.Where(path => !newMap.ContainsKey(path))
.ToArray();
if (removed.Length > 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: removed);
}
var changed = new List<ExportFileRecord>();
foreach (var file in manifest)
{
if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal))
{
changed.Add(file);
}
}
if (changed.Count == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var hasOutstandingDelta = existingState.LastDeltaDigest is not null;
if (hasOutstandingDelta)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
return new TrivyDbExportPlan(
TrivyDbExportMode.Delta,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: manifest,
ChangedFiles: changed,
RemovedPaths: Array.Empty<string>());
}
}
using System;
using StellaOps.Concelier.Storage.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Storage.Exporting;
public sealed class TrivyDbExportPlanner
{
public TrivyDbExportPlan CreatePlan(
ExportStateRecord? existingState,
string treeDigest,
IReadOnlyList<ExportFileRecord> manifest)
{
ArgumentException.ThrowIfNullOrEmpty(treeDigest);
manifest ??= Array.Empty<ExportFileRecord>();
if (existingState is null || (existingState.Files?.Count ?? 0) == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
BaseExportId: existingState?.BaseExportId,
BaseManifestDigest: existingState?.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>();
var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal);
if (cursorMatches)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var removed = existingMap.Keys
.Where(path => !newMap.ContainsKey(path))
.ToArray();
if (removed.Length > 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: removed);
}
var changed = new List<ExportFileRecord>();
foreach (var file in manifest)
{
if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal))
{
changed.Add(file);
}
}
if (changed.Count == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var hasOutstandingDelta = existingState.LastDeltaDigest is not null;
if (hasOutstandingDelta)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
return new TrivyDbExportPlan(
TrivyDbExportMode.Delta,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: manifest,
ChangedFiles: changed,
RemovedPaths: Array.Empty<string>());
}
}

View File

@@ -1,64 +1,64 @@
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Storage.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:trivyDb";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<TrivyDbExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy"));
options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree"));
options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat;
options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat;
options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix;
});
services.AddSingleton<TrivyDbPackageBuilder>();
services.AddSingleton<TrivyDbOciWriter>();
services.AddSingleton<TrivyDbExportPlanner>();
services.AddSingleton<ITrivyDbBuilder, TrivyDbBoltBuilder>();
services.AddSingleton<ITrivyDbOrasPusher, TrivyDbOrasPusher>();
services.AddSingleton<TrivyDbFeedExporter>();
services.AddTransient<TrivyDbExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind))
{
options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition(
TrivyDbExportJob.JobKind,
typeof(TrivyDbExportJob),
TrivyDbExportJob.DefaultTimeout,
TrivyDbExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
private static string Normalize(string? value, string fallback)
=> string.IsNullOrWhiteSpace(value) ? fallback : value;
}
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Storage.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:trivyDb";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<TrivyDbExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy"));
options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree"));
options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat;
options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat;
options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix;
});
services.AddSingleton<TrivyDbPackageBuilder>();
services.AddSingleton<TrivyDbOciWriter>();
services.AddSingleton<TrivyDbExportPlanner>();
services.AddSingleton<ITrivyDbBuilder, TrivyDbBoltBuilder>();
services.AddSingleton<ITrivyDbOrasPusher, TrivyDbOrasPusher>();
services.AddSingleton<TrivyDbFeedExporter>();
services.AddTransient<TrivyDbExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind))
{
options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition(
TrivyDbExportJob.JobKind,
typeof(TrivyDbExportJob),
TrivyDbExportJob.DefaultTimeout,
TrivyDbExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
private static string Normalize(string? value, string fallback)
=> string.IsNullOrWhiteSpace(value) ? fallback : value;
}

View File

@@ -1,23 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterPlugin : IExporterPlugin
{
public string Name => TrivyDbFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<TrivyDbFeedExporter>(services);
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterPlugin : IExporterPlugin
{
public string Name => TrivyDbFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<TrivyDbFeedExporter>(services);
}
}

View File

@@ -1,128 +1,128 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbFeedExporter : IFeedExporter
{
public const string ExporterName = "trivy-db";
public const string ExporterId = "export:trivy-db";
private readonly IAdvisoryStore _advisoryStore;
private readonly IJsonExportPathResolver _pathResolver;
private readonly TrivyDbExportOptions _options;
private readonly TrivyDbPackageBuilder _packageBuilder;
private readonly TrivyDbOciWriter _ociWriter;
private readonly ExportStateManager _stateManager;
private readonly TrivyDbExportPlanner _exportPlanner;
private readonly ITrivyDbBuilder _builder;
private readonly ITrivyDbOrasPusher _orasPusher;
private readonly ILogger<TrivyDbFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public TrivyDbFeedExporter(
IAdvisoryStore advisoryStore,
IJsonExportPathResolver pathResolver,
IOptions<TrivyDbExportOptions> options,
TrivyDbPackageBuilder packageBuilder,
TrivyDbOciWriter ociWriter,
ExportStateManager stateManager,
TrivyDbExportPlanner exportPlanner,
ITrivyDbBuilder builder,
ITrivyDbOrasPusher orasPusher,
ILogger<TrivyDbFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
var reference = $"{_options.ReferencePrefix}:{exportId}";
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
exportId,
jsonResult.AdvisoryCount,
jsonResult.TotalBytes);
var manifest = jsonResult.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
if (plan.Mode == TrivyDbExportMode.Skip)
{
_logger.LogInformation(
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
exportId,
plan.BaseExportId ?? "(none)");
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
return;
}
if (plan.Mode == TrivyDbExportMode.Delta)
{
_logger.LogInformation(
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
exportId,
plan.ChangedFiles.Count);
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbFeedExporter : IFeedExporter
{
public const string ExporterName = "trivy-db";
public const string ExporterId = "export:trivy-db";
private readonly IAdvisoryStore _advisoryStore;
private readonly IJsonExportPathResolver _pathResolver;
private readonly TrivyDbExportOptions _options;
private readonly TrivyDbPackageBuilder _packageBuilder;
private readonly TrivyDbOciWriter _ociWriter;
private readonly ExportStateManager _stateManager;
private readonly TrivyDbExportPlanner _exportPlanner;
private readonly ITrivyDbBuilder _builder;
private readonly ITrivyDbOrasPusher _orasPusher;
private readonly ILogger<TrivyDbFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public TrivyDbFeedExporter(
IAdvisoryStore advisoryStore,
IJsonExportPathResolver pathResolver,
IOptions<TrivyDbExportOptions> options,
TrivyDbPackageBuilder packageBuilder,
TrivyDbOciWriter ociWriter,
ExportStateManager stateManager,
TrivyDbExportPlanner exportPlanner,
ITrivyDbBuilder builder,
ITrivyDbOrasPusher orasPusher,
ILogger<TrivyDbFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
var reference = $"{_options.ReferencePrefix}:{exportId}";
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
exportId,
jsonResult.AdvisoryCount,
jsonResult.TotalBytes);
var manifest = jsonResult.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
if (plan.Mode == TrivyDbExportMode.Skip)
{
_logger.LogInformation(
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
exportId,
plan.BaseExportId ?? "(none)");
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
return;
}
if (plan.Mode == TrivyDbExportMode.Delta)
{
_logger.LogInformation(
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
exportId,
plan.ChangedFiles.Count);
}
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
var metadataDigest = ComputeDigest(metadataBytes);
var metadataLength = metadataBytes.LongLength;
try
{
try
{
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
metadataBytes,
builderResult.ArchivePath,
@@ -155,83 +155,83 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
exportedAt,
_logger,
cancellationToken).ConfigureAwait(false);
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
{
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
exportId,
ociResult.ManifestDigest);
var resetBaseline = plan.ResetBaseline
|| existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
{
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
ociResult.ManifestDigest,
cursor: treeDigest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
await _stateManager.StoreDeltaExportAsync(
ExporterId,
deltaDigest: treeDigest,
cursor: treeDigest,
exporterVersion: _exporterVersion,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
exportId,
ociResult.ManifestDigest);
var resetBaseline = plan.ResetBaseline
|| existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
{
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
ociResult.ManifestDigest,
cursor: treeDigest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
await _stateManager.StoreDeltaExportAsync(
ExporterId,
deltaDigest: treeDigest,
cursor: treeDigest,
exporterVersion: _exporterVersion,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
}
finally
{
TryDeleteDirectory(builderResult.WorkingDirectory);
}
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
}
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
{
var advisories = new List<Advisory>();
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
{
if (advisory is null)
{
continue;
}
advisories.Add(advisory);
}
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
return advisories;
}
}
finally
{
TryDeleteDirectory(builderResult.WorkingDirectory);
}
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
}
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
{
var advisories = new List<Advisory>();
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
{
if (advisory is null)
{
continue;
}
advisories.Add(advisory);
}
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
return advisories;
}
private byte[] CreateMetadataJson(
TrivyDbExportPlan plan,
ReadOnlyMemory<byte> builderMetadata,
@@ -267,31 +267,31 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
});
}
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
{
if (payload.IsEmpty)
{
return null;
}
try
{
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
});
}
catch
{
return null;
}
}
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
});
}
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
{
if (payload.IsEmpty)
{
return null;
}
try
{
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
});
}
catch
{
return null;
}
}
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
{
if (!_options.OfflineBundle.Enabled)
@@ -303,135 +303,135 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
{
return;
}
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
? $"{exportId}.offline.tar.gz"
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
if (File.Exists(bundlePath))
{
File.Delete(bundlePath);
}
var normalizedRoot = Path.GetFullPath(layoutPath);
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(file => NormalizeTarPath(normalizedRoot, file))
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
await using (var archiveStream = new FileStream(
bundlePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
{
var timestamp = exportedAt.UtcDateTime;
foreach (var directory in directories)
{
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
};
writer.WriteEntry(entry);
}
foreach (var relativePath in files)
{
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
UnixFileMode.GroupRead |
UnixFileMode.OtherRead,
};
await using var source = new FileStream(
fullPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
}
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(bundlePath).Length;
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// Best effort cleanup ignore failures.
}
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
? $"{exportId}.offline.tar.gz"
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
if (File.Exists(bundlePath))
{
File.Delete(bundlePath);
}
var normalizedRoot = Path.GetFullPath(layoutPath);
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(file => NormalizeTarPath(normalizedRoot, file))
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
await using (var archiveStream = new FileStream(
bundlePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
{
var timestamp = exportedAt.UtcDateTime;
foreach (var directory in directories)
{
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
};
writer.WriteEntry(entry);
}
foreach (var relativePath in files)
{
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
UnixFileMode.GroupRead |
UnixFileMode.OtherRead,
};
await using var source = new FileStream(
fullPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
}
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(bundlePath).Length;
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// Best effort cleanup ignore failures.
}
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static string NormalizeTarPath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
@@ -500,16 +500,16 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
public DeltaMetadata? Delta { get; set; }
}
private sealed class BuilderMetadata
{
[JsonPropertyName("Version")]
public int Version { get; set; }
public DateTime NextUpdate { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? DownloadedAt { get; set; }
}
}
private sealed class BuilderMetadata
{
[JsonPropertyName("Version")]
public int Version { get; set; }
public DateTime NextUpdate { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? DownloadedAt { get; set; }
}
}

View File

@@ -1,9 +1,9 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public static class TrivyDbMediaTypes
{
public const string OciManifest = "application/vnd.oci.image.manifest.v1+json";
public const string OciImageIndex = "application/vnd.oci.image.index.v1+json";
public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json";
public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip";
}
namespace StellaOps.Concelier.Exporter.TrivyDb;
public static class TrivyDbMediaTypes
{
public const string OciManifest = "application/vnd.oci.image.manifest.v1+json";
public const string OciImageIndex = "application/vnd.oci.image.index.v1+json";
public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json";
public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip";
}

View File

@@ -1,392 +1,392 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal static class TrivyDbMirrorBundleWriter
{
private const int SchemaVersion = 1;
private const string DefaultDirectoryName = "mirror";
private const string MetadataFileName = "metadata.json";
private const string DatabaseFileName = "db.tar.gz";
private const string ManifestFileName = "manifest.json";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public static async Task WriteAsync(
string layoutRoot,
JsonExportResult jsonResult,
TrivyDbExportOptions options,
TrivyDbExportPlan plan,
TrivyDbBuilderResult builderResult,
string reference,
string manifestDigest,
ReadOnlyMemory<byte> metadataBytes,
string metadataDigest,
long metadataLength,
string exporterVersion,
DateTimeOffset exportedAt,
ILogger logger,
CancellationToken cancellationToken)
{
if (options?.Mirror is null || !options.Mirror.Enabled || options.Mirror.Domains.Count == 0)
{
return;
}
if (string.IsNullOrWhiteSpace(layoutRoot))
{
throw new ArgumentException("Layout root must be provided.", nameof(layoutRoot));
}
if (builderResult is null)
{
throw new ArgumentNullException(nameof(builderResult));
}
if (jsonResult is null)
{
throw new ArgumentNullException(nameof(jsonResult));
}
var directoryName = string.IsNullOrWhiteSpace(options.Mirror.DirectoryName)
? DefaultDirectoryName
: options.Mirror.DirectoryName.Trim();
if (directoryName.Length == 0)
{
directoryName = DefaultDirectoryName;
}
var root = Path.Combine(layoutRoot, directoryName);
Directory.CreateDirectory(root);
var timestamp = exportedAt.UtcDateTime;
TrySetDirectoryTimestamp(root, timestamp);
var advisories = jsonResult.Advisories.IsDefaultOrEmpty
? Array.Empty<Advisory>()
: jsonResult.Advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
var domains = new List<MirrorIndexDomainEntry>();
foreach (var domainOption in options.Mirror.Domains)
{
cancellationToken.ThrowIfCancellationRequested();
if (domainOption is null)
{
logger.LogWarning("Encountered null Trivy mirror domain configuration; skipping.");
continue;
}
var domainId = (domainOption.Id ?? string.Empty).Trim();
if (domainId.Length == 0)
{
logger.LogWarning("Skipping Trivy mirror domain with empty id.");
continue;
}
var displayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
? domainId
: domainOption.DisplayName!.Trim();
var domainDirectory = Path.Combine(root, domainId);
Directory.CreateDirectory(domainDirectory);
TrySetDirectoryTimestamp(domainDirectory, timestamp);
var metadataPath = Path.Combine(domainDirectory, MetadataFileName);
await WriteFileAsync(metadataPath, metadataBytes, timestamp, cancellationToken).ConfigureAwait(false);
var metadataRelativePath = ToRelativePath(layoutRoot, metadataPath);
var databasePath = Path.Combine(domainDirectory, DatabaseFileName);
await CopyDatabaseAsync(builderResult.ArchivePath, databasePath, timestamp, cancellationToken).ConfigureAwait(false);
var databaseRelativePath = ToRelativePath(layoutRoot, databasePath);
var sources = BuildSourceSummaries(advisories);
var manifestDocument = new MirrorDomainManifestDocument(
SchemaVersion,
exportedAt,
exporterVersion,
reference,
manifestDigest,
options.TargetRepository,
domainId,
displayName,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources);
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions);
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
await WriteFileAsync(manifestPath, manifestBytes, timestamp, cancellationToken).ConfigureAwait(false);
var manifestRelativePath = ToRelativePath(layoutRoot, manifestPath);
var manifestDigestValue = ComputeDigest(manifestBytes);
domains.Add(new MirrorIndexDomainEntry(
domainId,
displayName,
advisories.Length,
new MirrorFileDescriptor(manifestRelativePath, manifestBytes.LongLength, manifestDigestValue),
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources));
}
if (domains.Count == 0)
{
Directory.Delete(root, recursive: true);
return;
}
domains.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
var delta = plan.Mode == TrivyDbExportMode.Delta
? new MirrorDeltaMetadata(
plan.ChangedFiles.Select(static file => new MirrorDeltaFile(file.Path, file.Digest)).ToArray(),
plan.RemovedPaths.ToArray())
: null;
var indexDocument = new MirrorIndexDocument(
SchemaVersion,
exportedAt,
exporterVersion,
options.TargetRepository,
reference,
manifestDigest,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
delta,
domains);
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(indexDocument, SerializerOptions);
var indexPath = Path.Combine(root, "index.json");
await WriteFileAsync(indexPath, indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Generated {DomainCount} Trivy DB mirror bundle(s) under {Directory}.",
domains.Count,
directoryName);
}
private static IReadOnlyList<TrivyMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
{
if (advisories.Count == 0)
{
return Array.Empty<TrivyMirrorSourceSummary>();
}
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provenance in advisory.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
var source = provenance.Source.Trim();
if (!builders.TryGetValue(source, out var accumulator))
{
accumulator = new SourceAccumulator();
builders[source] = accumulator;
}
accumulator.Record(provenance.RecordedAt);
if (counted.Add(source))
{
accumulator.Increment();
}
}
}
var entries = builders
.Select(static pair => new TrivyMirrorSourceSummary(
pair.Key,
pair.Value.FirstRecordedAt,
pair.Value.LastRecordedAt,
pair.Value.Count))
.OrderBy(static summary => summary.Source, StringComparer.Ordinal)
.ToArray();
return entries;
}
private static async Task CopyDatabaseAsync(
string sourcePath,
string destinationPath,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
}
private static async Task WriteFileAsync(
string path,
ReadOnlyMemory<byte> bytes,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await using var stream = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, timestamp);
}
private static string ToRelativePath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
return string.IsNullOrEmpty(normalized) ? "." : normalized;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private static void TrySetDirectoryTimestamp(string directory, DateTime timestamp)
{
try
{
Directory.SetLastWriteTimeUtc(directory, timestamp);
}
catch
{
// Best effort ignore failures.
}
}
private sealed record MirrorIndexDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string? TargetRepository,
string Reference,
string ManifestDigest,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorDeltaMetadata? Delta,
IReadOnlyList<MirrorIndexDomainEntry> Domains);
private sealed record MirrorDeltaMetadata(
IReadOnlyList<MirrorDeltaFile> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
private sealed record MirrorDeltaFile(string Path, string Digest);
private sealed record MirrorIndexDomainEntry(
string DomainId,
string DisplayName,
int AdvisoryCount,
MirrorFileDescriptor Manifest,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorDomainManifestDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string Reference,
string ManifestDigest,
string? TargetRepository,
string DomainId,
string DisplayName,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorFileDescriptor(string Path, long SizeBytes, string Digest);
private sealed record TrivyMirrorSourceSummary(
string Source,
DateTimeOffset? FirstRecordedAt,
DateTimeOffset? LastRecordedAt,
int AdvisoryCount);
private sealed class SourceAccumulator
{
public DateTimeOffset? FirstRecordedAt { get; private set; }
public DateTimeOffset? LastRecordedAt { get; private set; }
public int Count { get; private set; }
public void Record(DateTimeOffset recordedAt)
{
var utc = recordedAt.ToUniversalTime();
if (FirstRecordedAt is null || utc < FirstRecordedAt.Value)
{
FirstRecordedAt = utc;
}
if (LastRecordedAt is null || utc > LastRecordedAt.Value)
{
LastRecordedAt = utc;
}
}
public void Increment() => Count++;
}
}
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal static class TrivyDbMirrorBundleWriter
{
private const int SchemaVersion = 1;
private const string DefaultDirectoryName = "mirror";
private const string MetadataFileName = "metadata.json";
private const string DatabaseFileName = "db.tar.gz";
private const string ManifestFileName = "manifest.json";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public static async Task WriteAsync(
string layoutRoot,
JsonExportResult jsonResult,
TrivyDbExportOptions options,
TrivyDbExportPlan plan,
TrivyDbBuilderResult builderResult,
string reference,
string manifestDigest,
ReadOnlyMemory<byte> metadataBytes,
string metadataDigest,
long metadataLength,
string exporterVersion,
DateTimeOffset exportedAt,
ILogger logger,
CancellationToken cancellationToken)
{
if (options?.Mirror is null || !options.Mirror.Enabled || options.Mirror.Domains.Count == 0)
{
return;
}
if (string.IsNullOrWhiteSpace(layoutRoot))
{
throw new ArgumentException("Layout root must be provided.", nameof(layoutRoot));
}
if (builderResult is null)
{
throw new ArgumentNullException(nameof(builderResult));
}
if (jsonResult is null)
{
throw new ArgumentNullException(nameof(jsonResult));
}
var directoryName = string.IsNullOrWhiteSpace(options.Mirror.DirectoryName)
? DefaultDirectoryName
: options.Mirror.DirectoryName.Trim();
if (directoryName.Length == 0)
{
directoryName = DefaultDirectoryName;
}
var root = Path.Combine(layoutRoot, directoryName);
Directory.CreateDirectory(root);
var timestamp = exportedAt.UtcDateTime;
TrySetDirectoryTimestamp(root, timestamp);
var advisories = jsonResult.Advisories.IsDefaultOrEmpty
? Array.Empty<Advisory>()
: jsonResult.Advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
var domains = new List<MirrorIndexDomainEntry>();
foreach (var domainOption in options.Mirror.Domains)
{
cancellationToken.ThrowIfCancellationRequested();
if (domainOption is null)
{
logger.LogWarning("Encountered null Trivy mirror domain configuration; skipping.");
continue;
}
var domainId = (domainOption.Id ?? string.Empty).Trim();
if (domainId.Length == 0)
{
logger.LogWarning("Skipping Trivy mirror domain with empty id.");
continue;
}
var displayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
? domainId
: domainOption.DisplayName!.Trim();
var domainDirectory = Path.Combine(root, domainId);
Directory.CreateDirectory(domainDirectory);
TrySetDirectoryTimestamp(domainDirectory, timestamp);
var metadataPath = Path.Combine(domainDirectory, MetadataFileName);
await WriteFileAsync(metadataPath, metadataBytes, timestamp, cancellationToken).ConfigureAwait(false);
var metadataRelativePath = ToRelativePath(layoutRoot, metadataPath);
var databasePath = Path.Combine(domainDirectory, DatabaseFileName);
await CopyDatabaseAsync(builderResult.ArchivePath, databasePath, timestamp, cancellationToken).ConfigureAwait(false);
var databaseRelativePath = ToRelativePath(layoutRoot, databasePath);
var sources = BuildSourceSummaries(advisories);
var manifestDocument = new MirrorDomainManifestDocument(
SchemaVersion,
exportedAt,
exporterVersion,
reference,
manifestDigest,
options.TargetRepository,
domainId,
displayName,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources);
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions);
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
await WriteFileAsync(manifestPath, manifestBytes, timestamp, cancellationToken).ConfigureAwait(false);
var manifestRelativePath = ToRelativePath(layoutRoot, manifestPath);
var manifestDigestValue = ComputeDigest(manifestBytes);
domains.Add(new MirrorIndexDomainEntry(
domainId,
displayName,
advisories.Length,
new MirrorFileDescriptor(manifestRelativePath, manifestBytes.LongLength, manifestDigestValue),
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources));
}
if (domains.Count == 0)
{
Directory.Delete(root, recursive: true);
return;
}
domains.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
var delta = plan.Mode == TrivyDbExportMode.Delta
? new MirrorDeltaMetadata(
plan.ChangedFiles.Select(static file => new MirrorDeltaFile(file.Path, file.Digest)).ToArray(),
plan.RemovedPaths.ToArray())
: null;
var indexDocument = new MirrorIndexDocument(
SchemaVersion,
exportedAt,
exporterVersion,
options.TargetRepository,
reference,
manifestDigest,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
delta,
domains);
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(indexDocument, SerializerOptions);
var indexPath = Path.Combine(root, "index.json");
await WriteFileAsync(indexPath, indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Generated {DomainCount} Trivy DB mirror bundle(s) under {Directory}.",
domains.Count,
directoryName);
}
private static IReadOnlyList<TrivyMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
{
if (advisories.Count == 0)
{
return Array.Empty<TrivyMirrorSourceSummary>();
}
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provenance in advisory.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
var source = provenance.Source.Trim();
if (!builders.TryGetValue(source, out var accumulator))
{
accumulator = new SourceAccumulator();
builders[source] = accumulator;
}
accumulator.Record(provenance.RecordedAt);
if (counted.Add(source))
{
accumulator.Increment();
}
}
}
var entries = builders
.Select(static pair => new TrivyMirrorSourceSummary(
pair.Key,
pair.Value.FirstRecordedAt,
pair.Value.LastRecordedAt,
pair.Value.Count))
.OrderBy(static summary => summary.Source, StringComparer.Ordinal)
.ToArray();
return entries;
}
private static async Task CopyDatabaseAsync(
string sourcePath,
string destinationPath,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
}
private static async Task WriteFileAsync(
string path,
ReadOnlyMemory<byte> bytes,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await using var stream = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, timestamp);
}
private static string ToRelativePath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
return string.IsNullOrEmpty(normalized) ? "." : normalized;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private static void TrySetDirectoryTimestamp(string directory, DateTime timestamp)
{
try
{
Directory.SetLastWriteTimeUtc(directory, timestamp);
}
catch
{
// Best effort ignore failures.
}
}
private sealed record MirrorIndexDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string? TargetRepository,
string Reference,
string ManifestDigest,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorDeltaMetadata? Delta,
IReadOnlyList<MirrorIndexDomainEntry> Domains);
private sealed record MirrorDeltaMetadata(
IReadOnlyList<MirrorDeltaFile> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
private sealed record MirrorDeltaFile(string Path, string Digest);
private sealed record MirrorIndexDomainEntry(
string DomainId,
string DisplayName,
int AdvisoryCount,
MirrorFileDescriptor Manifest,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorDomainManifestDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string Reference,
string ManifestDigest,
string? TargetRepository,
string DomainId,
string DisplayName,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorFileDescriptor(string Path, long SizeBytes, string Digest);
private sealed record TrivyMirrorSourceSummary(
string Source,
DateTimeOffset? FirstRecordedAt,
DateTimeOffset? LastRecordedAt,
int AdvisoryCount);
private sealed class SourceAccumulator
{
public DateTimeOffset? FirstRecordedAt { get; private set; }
public DateTimeOffset? LastRecordedAt { get; private set; }
public int Count { get; private set; }
public void Record(DateTimeOffset recordedAt)
{
var utc = recordedAt.ToUniversalTime();
if (FirstRecordedAt is null || utc < FirstRecordedAt.Value)
{
FirstRecordedAt = utc;
}
if (LastRecordedAt is null || utc > LastRecordedAt.Value)
{
LastRecordedAt = utc;
}
}
public void Increment() => Count++;
}
}

View File

@@ -1,8 +1,8 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbOciWriteResult(
string RootDirectory,
string ManifestDigest,
IReadOnlyCollection<string> BlobDigests);
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbOciWriteResult(
string RootDirectory,
string ManifestDigest,
IReadOnlyCollection<string> BlobDigests);

View File

@@ -1,375 +1,375 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
/// <summary>
/// Writes a Trivy DB package to an OCI image layout directory with deterministic content.
/// </summary>
public sealed class TrivyDbOciWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}");
public async Task<TrivyDbOciWriteResult> WriteAsync(
TrivyDbPackage package,
string destination,
string reference,
TrivyDbExportPlan plan,
string? baseLayoutPath,
CancellationToken cancellationToken)
{
if (package is null)
{
throw new ArgumentNullException(nameof(package));
}
if (string.IsNullOrWhiteSpace(destination))
{
throw new ArgumentException("Destination directory must be provided.", nameof(destination));
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new ArgumentException("Reference tag must be provided.", nameof(reference));
}
if (plan is null)
{
throw new ArgumentNullException(nameof(plan));
}
var root = Path.GetFullPath(destination);
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
var timestamp = package.Config.GeneratedAt.UtcDateTime;
await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false);
await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false);
var blobsRoot = Path.Combine(root, "blobs", "sha256");
Directory.CreateDirectory(blobsRoot);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp);
Directory.SetLastWriteTimeUtc(blobsRoot, timestamp);
var writtenDigests = new HashSet<string>(StringComparer.Ordinal);
foreach (var pair in package.Blobs)
{
if (writtenDigests.Contains(pair.Key))
{
continue;
}
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (reused)
{
writtenDigests.Add(pair.Key);
continue;
}
if (writtenDigests.Add(pair.Key))
{
await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false);
}
}
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions);
var manifestDigest = ComputeDigest(manifestBytes);
if (!writtenDigests.Contains(manifestDigest))
{
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (!reused)
{
await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false);
}
writtenDigests.Add(manifestDigest);
}
var manifestDescriptor = new OciDescriptor(
TrivyDbMediaTypes.OciManifest,
manifestDigest,
manifestBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.ref.name"] = reference,
});
var index = new OciIndex(2, new[] { manifestDescriptor });
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions);
await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath))
{
var reuseDigests = await TryReuseBaseBlobsAsync(
blobsRoot,
timestamp,
writtenDigests,
plan,
baseLayoutPath,
cancellationToken).ConfigureAwait(false);
foreach (var digest in reuseDigests)
{
writtenDigests.Add(digest);
}
}
Directory.SetLastWriteTimeUtc(root, timestamp);
var blobDigests = writtenDigests.ToArray();
Array.Sort(blobDigests, StringComparer.Ordinal);
return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests);
}
private static async Task WriteFileAsync(string path, ReadOnlyMemory<byte> bytes, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var fileName = ResolveDigestFileName(digest);
var path = Path.Combine(blobsRoot, fileName);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false);
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static string ResolveDigestFileName(string digest)
{
if (!digest.StartsWith("sha256:", StringComparison.Ordinal))
{
throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'.");
}
var hex = digest[7..];
if (hex.Length == 0)
{
throw new InvalidOperationException("Digest hex component cannot be empty.");
}
return hex;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = System.Security.Cryptography.SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
private static async Task<IReadOnlyCollection<string>> TryReuseBaseBlobsAsync(
string destinationBlobsRoot,
DateTime timestamp,
HashSet<string> written,
TrivyDbExportPlan plan,
string baseLayoutPath,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest))
{
return Array.Empty<string>();
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
if (!Directory.Exists(baseRoot))
{
return Array.Empty<string>();
}
var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest);
if (!File.Exists(manifestPath))
{
return Array.Empty<string>();
}
await using var stream = new FileStream(
manifestPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
var digests = new SortedSet<string>(StringComparer.Ordinal)
{
plan.BaseManifestDigest,
};
if (root.TryGetProperty("config", out var configNode))
{
var digest = configNode.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
if (root.TryGetProperty("layers", out var layersNode))
{
foreach (var layer in layersNode.EnumerateArray())
{
var digest = layer.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
}
var copied = new List<string>();
foreach (var digest in digests)
{
if (written.Contains(digest))
{
continue;
}
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
continue;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
copied.Add(digest);
}
if (copied.Count > 0)
{
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
}
return copied;
}
private static string ResolveBlobPath(string layoutRoot, string digest)
{
var fileName = ResolveDigestFileName(digest);
return Path.Combine(layoutRoot, "blobs", "sha256", fileName);
}
private static async Task<bool> TryReuseExistingBlobAsync(
string? baseLayoutPath,
string digest,
string destinationBlobsRoot,
DateTime timestamp,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(baseLayoutPath))
{
return false;
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
return false;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
return true;
}
}
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
/// <summary>
/// Writes a Trivy DB package to an OCI image layout directory with deterministic content.
/// </summary>
public sealed class TrivyDbOciWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}");
public async Task<TrivyDbOciWriteResult> WriteAsync(
TrivyDbPackage package,
string destination,
string reference,
TrivyDbExportPlan plan,
string? baseLayoutPath,
CancellationToken cancellationToken)
{
if (package is null)
{
throw new ArgumentNullException(nameof(package));
}
if (string.IsNullOrWhiteSpace(destination))
{
throw new ArgumentException("Destination directory must be provided.", nameof(destination));
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new ArgumentException("Reference tag must be provided.", nameof(reference));
}
if (plan is null)
{
throw new ArgumentNullException(nameof(plan));
}
var root = Path.GetFullPath(destination);
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
var timestamp = package.Config.GeneratedAt.UtcDateTime;
await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false);
await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false);
var blobsRoot = Path.Combine(root, "blobs", "sha256");
Directory.CreateDirectory(blobsRoot);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp);
Directory.SetLastWriteTimeUtc(blobsRoot, timestamp);
var writtenDigests = new HashSet<string>(StringComparer.Ordinal);
foreach (var pair in package.Blobs)
{
if (writtenDigests.Contains(pair.Key))
{
continue;
}
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (reused)
{
writtenDigests.Add(pair.Key);
continue;
}
if (writtenDigests.Add(pair.Key))
{
await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false);
}
}
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions);
var manifestDigest = ComputeDigest(manifestBytes);
if (!writtenDigests.Contains(manifestDigest))
{
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (!reused)
{
await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false);
}
writtenDigests.Add(manifestDigest);
}
var manifestDescriptor = new OciDescriptor(
TrivyDbMediaTypes.OciManifest,
manifestDigest,
manifestBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.ref.name"] = reference,
});
var index = new OciIndex(2, new[] { manifestDescriptor });
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions);
await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath))
{
var reuseDigests = await TryReuseBaseBlobsAsync(
blobsRoot,
timestamp,
writtenDigests,
plan,
baseLayoutPath,
cancellationToken).ConfigureAwait(false);
foreach (var digest in reuseDigests)
{
writtenDigests.Add(digest);
}
}
Directory.SetLastWriteTimeUtc(root, timestamp);
var blobDigests = writtenDigests.ToArray();
Array.Sort(blobDigests, StringComparer.Ordinal);
return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests);
}
private static async Task WriteFileAsync(string path, ReadOnlyMemory<byte> bytes, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var fileName = ResolveDigestFileName(digest);
var path = Path.Combine(blobsRoot, fileName);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false);
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static string ResolveDigestFileName(string digest)
{
if (!digest.StartsWith("sha256:", StringComparison.Ordinal))
{
throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'.");
}
var hex = digest[7..];
if (hex.Length == 0)
{
throw new InvalidOperationException("Digest hex component cannot be empty.");
}
return hex;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = System.Security.Cryptography.SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
private static async Task<IReadOnlyCollection<string>> TryReuseBaseBlobsAsync(
string destinationBlobsRoot,
DateTime timestamp,
HashSet<string> written,
TrivyDbExportPlan plan,
string baseLayoutPath,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest))
{
return Array.Empty<string>();
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
if (!Directory.Exists(baseRoot))
{
return Array.Empty<string>();
}
var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest);
if (!File.Exists(manifestPath))
{
return Array.Empty<string>();
}
await using var stream = new FileStream(
manifestPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
var digests = new SortedSet<string>(StringComparer.Ordinal)
{
plan.BaseManifestDigest,
};
if (root.TryGetProperty("config", out var configNode))
{
var digest = configNode.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
if (root.TryGetProperty("layers", out var layersNode))
{
foreach (var layer in layersNode.EnumerateArray())
{
var digest = layer.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
}
var copied = new List<string>();
foreach (var digest in digests)
{
if (written.Contains(digest))
{
continue;
}
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
continue;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
copied.Add(digest);
}
if (copied.Count > 0)
{
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
}
return copied;
}
private static string ResolveBlobPath(string layoutRoot, string digest)
{
var fileName = ResolveDigestFileName(digest);
return Path.Combine(layoutRoot, "blobs", "sha256", fileName);
}
private static async Task<bool> TryReuseExistingBlobAsync(
string? baseLayoutPath,
string digest,
string destinationBlobsRoot,
DateTime timestamp,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(baseLayoutPath))
{
return false;
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
return false;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
return true;
}
}

View File

@@ -1,209 +1,209 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbOrasPusher> _logger;
public TrivyDbOrasPusher(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbOrasPusher> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken)
{
var orasOptions = _options.Oras;
if (!orasOptions.Enabled)
{
return;
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("ORAS push requested but reference is empty.");
}
if (!Directory.Exists(layoutPath))
{
throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist.");
}
var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath;
var tag = ResolveTag(reference, exportId);
var layoutReference = $"{layoutPath}:{tag}";
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("cp");
startInfo.ArgumentList.Add("--from-oci-layout");
startInfo.ArgumentList.Add(layoutReference);
if (orasOptions.SkipTlsVerify)
{
startInfo.ArgumentList.Add("--insecure");
}
if (orasOptions.UseHttp)
{
startInfo.ArgumentList.Add("--plain-http");
}
if (orasOptions.AdditionalArguments is { Count: > 0 })
{
foreach (var arg in orasOptions.AdditionalArguments)
{
if (!string.IsNullOrWhiteSpace(arg))
{
startInfo.ArgumentList.Add(arg);
}
}
}
startInfo.ArgumentList.Add(reference);
if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = orasOptions.WorkingDirectory;
}
if (!orasOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
if (orasOptions.Environment is { Count: > 0 })
{
foreach (var kvp in orasOptions.Environment)
{
if (!string.IsNullOrEmpty(kvp.Key))
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
}
}
using var process = new Process { StartInfo = startInfo };
var stdout = new StringBuilder();
var stderr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdout.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stderr.AppendLine(e.Data);
}
};
_logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// ignore
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdout.Length > 0)
{
_logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString());
}
if (stderr.Length > 0)
{
_logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString());
}
}
private static string ResolveTag(string reference, string fallback)
{
if (string.IsNullOrWhiteSpace(reference))
{
return fallback;
}
var atIndex = reference.IndexOf('@');
if (atIndex >= 0)
{
reference = reference[..atIndex];
}
var slashIndex = reference.LastIndexOf('/');
var colonIndex = reference.LastIndexOf(':');
if (colonIndex > slashIndex && colonIndex >= 0)
{
return reference[(colonIndex + 1)..];
}
return fallback;
}
}
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbOrasPusher> _logger;
public TrivyDbOrasPusher(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbOrasPusher> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken)
{
var orasOptions = _options.Oras;
if (!orasOptions.Enabled)
{
return;
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("ORAS push requested but reference is empty.");
}
if (!Directory.Exists(layoutPath))
{
throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist.");
}
var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath;
var tag = ResolveTag(reference, exportId);
var layoutReference = $"{layoutPath}:{tag}";
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("cp");
startInfo.ArgumentList.Add("--from-oci-layout");
startInfo.ArgumentList.Add(layoutReference);
if (orasOptions.SkipTlsVerify)
{
startInfo.ArgumentList.Add("--insecure");
}
if (orasOptions.UseHttp)
{
startInfo.ArgumentList.Add("--plain-http");
}
if (orasOptions.AdditionalArguments is { Count: > 0 })
{
foreach (var arg in orasOptions.AdditionalArguments)
{
if (!string.IsNullOrWhiteSpace(arg))
{
startInfo.ArgumentList.Add(arg);
}
}
}
startInfo.ArgumentList.Add(reference);
if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = orasOptions.WorkingDirectory;
}
if (!orasOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
if (orasOptions.Environment is { Count: > 0 })
{
foreach (var kvp in orasOptions.Environment)
{
if (!string.IsNullOrEmpty(kvp.Key))
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
}
}
using var process = new Process { StartInfo = startInfo };
var stdout = new StringBuilder();
var stderr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdout.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stderr.AppendLine(e.Data);
}
};
_logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// ignore
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdout.Length > 0)
{
_logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString());
}
if (stderr.Length > 0)
{
_logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString());
}
}
private static string ResolveTag(string reference, string fallback)
{
if (string.IsNullOrWhiteSpace(reference))
{
return fallback;
}
var atIndex = reference.IndexOf('@');
if (atIndex >= 0)
{
reference = reference[..atIndex];
}
var slashIndex = reference.LastIndexOf('/');
var colonIndex = reference.LastIndexOf(':');
if (colonIndex > slashIndex && colonIndex >= 0)
{
return reference[(colonIndex + 1)..];
}
return fallback;
}
}

View File

@@ -1,9 +1,9 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackage(
OciManifest Manifest,
TrivyConfigDocument Config,
IReadOnlyDictionary<string, TrivyDbBlob> Blobs,
ReadOnlyMemory<byte> MetadataJson);
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackage(
OciManifest Manifest,
TrivyConfigDocument Config,
IReadOnlyDictionary<string, TrivyDbBlob> Blobs,
ReadOnlyMemory<byte> MetadataJson);

View File

@@ -1,116 +1,116 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbPackageBuilder
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (request.MetadataJson.IsEmpty)
{
throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request));
}
if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath))
{
throw new ArgumentException("Database archive path must be provided.", nameof(request));
}
if (!File.Exists(request.DatabaseArchivePath))
{
throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath);
}
if (string.IsNullOrWhiteSpace(request.DatabaseDigest))
{
throw new ArgumentException("Database archive digest must be provided.", nameof(request));
}
if (request.DatabaseLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength));
}
var metadataBytes = request.MetadataJson;
var generatedAt = request.GeneratedAt.ToUniversalTime();
var configDocument = new TrivyConfigDocument(
TrivyDbMediaTypes.TrivyConfig,
generatedAt,
request.DatabaseVersion,
request.DatabaseDigest,
request.DatabaseLength);
var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions);
var configDigest = ComputeDigest(configBytes);
var configDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyConfig,
configDigest,
configBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "config.json",
});
var layerDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyLayer,
request.DatabaseDigest,
request.DatabaseLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "db.tar.gz",
});
var manifest = new OciManifest(
2,
TrivyDbMediaTypes.OciManifest,
configDescriptor,
ImmutableArray.Create(layerDescriptor));
var blobs = new SortedDictionary<string, TrivyDbBlob>(StringComparer.Ordinal)
{
[configDigest] = TrivyDbBlob.FromBytes(configBytes),
[request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength),
};
return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes);
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
}
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbPackageBuilder
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (request.MetadataJson.IsEmpty)
{
throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request));
}
if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath))
{
throw new ArgumentException("Database archive path must be provided.", nameof(request));
}
if (!File.Exists(request.DatabaseArchivePath))
{
throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath);
}
if (string.IsNullOrWhiteSpace(request.DatabaseDigest))
{
throw new ArgumentException("Database archive digest must be provided.", nameof(request));
}
if (request.DatabaseLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength));
}
var metadataBytes = request.MetadataJson;
var generatedAt = request.GeneratedAt.ToUniversalTime();
var configDocument = new TrivyConfigDocument(
TrivyDbMediaTypes.TrivyConfig,
generatedAt,
request.DatabaseVersion,
request.DatabaseDigest,
request.DatabaseLength);
var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions);
var configDigest = ComputeDigest(configBytes);
var configDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyConfig,
configDigest,
configBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "config.json",
});
var layerDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyLayer,
request.DatabaseDigest,
request.DatabaseLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "db.tar.gz",
});
var manifest = new OciManifest(
2,
TrivyDbMediaTypes.OciManifest,
configDescriptor,
ImmutableArray.Create(layerDescriptor));
var blobs = new SortedDictionary<string, TrivyDbBlob>(StringComparer.Ordinal)
{
[configDigest] = TrivyDbBlob.FromBytes(configBytes),
[request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength),
};
return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes);
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
}

View File

@@ -1,11 +1,11 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackageRequest(
ReadOnlyMemory<byte> MetadataJson,
string DatabaseArchivePath,
string DatabaseDigest,
long DatabaseLength,
DateTimeOffset GeneratedAt,
string DatabaseVersion);
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackageRequest(
ReadOnlyMemory<byte> MetadataJson,
string DatabaseArchivePath,
string DatabaseDigest,
long DatabaseLength,
DateTimeOffset GeneratedAt,
string DatabaseVersion);