491 lines
19 KiB
C#
491 lines
19 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Globalization;
|
||
using System.IO;
|
||
using System.IO.Compression;
|
||
using System.Linq;
|
||
using System.Security.Cryptography;
|
||
using System.Text.Json;
|
||
using System.Text.Json.Serialization;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using System.Formats.Tar;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Options;
|
||
using StellaOps.Concelier.Exporter.Json;
|
||
using StellaOps.Concelier.Models;
|
||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||
using StellaOps.Plugin;
|
||
|
||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||
|
||
public sealed class TrivyDbFeedExporter : IFeedExporter
|
||
{
|
||
public const string ExporterName = "trivy-db";
|
||
public const string ExporterId = "export:trivy-db";
|
||
|
||
private readonly IAdvisoryStore _advisoryStore;
|
||
private readonly IJsonExportPathResolver _pathResolver;
|
||
private readonly TrivyDbExportOptions _options;
|
||
private readonly TrivyDbPackageBuilder _packageBuilder;
|
||
private readonly TrivyDbOciWriter _ociWriter;
|
||
private readonly ExportStateManager _stateManager;
|
||
private readonly TrivyDbExportPlanner _exportPlanner;
|
||
private readonly ITrivyDbBuilder _builder;
|
||
private readonly ITrivyDbOrasPusher _orasPusher;
|
||
private readonly ILogger<TrivyDbFeedExporter> _logger;
|
||
private readonly TimeProvider _timeProvider;
|
||
private readonly string _exporterVersion;
|
||
|
||
public TrivyDbFeedExporter(
|
||
IAdvisoryStore advisoryStore,
|
||
IJsonExportPathResolver pathResolver,
|
||
IOptions<TrivyDbExportOptions> options,
|
||
TrivyDbPackageBuilder packageBuilder,
|
||
TrivyDbOciWriter ociWriter,
|
||
ExportStateManager stateManager,
|
||
TrivyDbExportPlanner exportPlanner,
|
||
ITrivyDbBuilder builder,
|
||
ITrivyDbOrasPusher orasPusher,
|
||
ILogger<TrivyDbFeedExporter> logger,
|
||
TimeProvider? timeProvider = null)
|
||
{
|
||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
|
||
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
|
||
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
|
||
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
|
||
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
|
||
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
|
||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
|
||
}
|
||
|
||
public string Name => ExporterName;
|
||
|
||
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||
{
|
||
var exportedAt = _timeProvider.GetUtcNow();
|
||
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
|
||
var reference = $"{_options.ReferencePrefix}:{exportId}";
|
||
|
||
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
|
||
|
||
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
|
||
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
|
||
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||
|
||
_logger.LogInformation(
|
||
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
|
||
exportId,
|
||
jsonResult.AdvisoryCount,
|
||
jsonResult.TotalBytes);
|
||
|
||
var manifest = jsonResult.Files
|
||
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
|
||
.ToArray();
|
||
|
||
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
|
||
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
|
||
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
|
||
|
||
if (plan.Mode == TrivyDbExportMode.Skip)
|
||
{
|
||
_logger.LogInformation(
|
||
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
|
||
exportId,
|
||
plan.BaseExportId ?? "(none)");
|
||
|
||
if (!_options.KeepWorkingTree)
|
||
{
|
||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
if (plan.Mode == TrivyDbExportMode.Delta)
|
||
{
|
||
_logger.LogInformation(
|
||
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
|
||
exportId,
|
||
plan.ChangedFiles.Count);
|
||
}
|
||
|
||
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
|
||
|
||
try
|
||
{
|
||
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
|
||
metadataBytes,
|
||
builderResult.ArchivePath,
|
||
builderResult.ArchiveDigest,
|
||
builderResult.ArchiveLength,
|
||
exportedAt,
|
||
exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture)));
|
||
|
||
var destination = _options.GetExportRoot(exportId);
|
||
string? baseLayout = null;
|
||
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId))
|
||
{
|
||
baseLayout = _options.GetExportRoot(plan.BaseExportId);
|
||
}
|
||
|
||
var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false);
|
||
|
||
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
|
||
{
|
||
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
_logger.LogInformation(
|
||
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
|
||
exportId,
|
||
ociResult.ManifestDigest);
|
||
|
||
var resetBaseline = plan.ResetBaseline
|
||
|| existingState is null
|
||
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|
||
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
|
||
|
||
if (existingState is not null
|
||
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
|
||
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
|
||
{
|
||
resetBaseline = true;
|
||
}
|
||
|
||
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
|
||
{
|
||
await _stateManager.StoreFullExportAsync(
|
||
ExporterId,
|
||
exportId,
|
||
ociResult.ManifestDigest,
|
||
cursor: treeDigest,
|
||
targetRepository: _options.TargetRepository,
|
||
exporterVersion: _exporterVersion,
|
||
resetBaseline: resetBaseline,
|
||
manifest: plan.Manifest,
|
||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||
}
|
||
else
|
||
{
|
||
await _stateManager.StoreDeltaExportAsync(
|
||
ExporterId,
|
||
deltaDigest: treeDigest,
|
||
cursor: treeDigest,
|
||
exporterVersion: _exporterVersion,
|
||
manifest: plan.Manifest,
|
||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
|
||
}
|
||
finally
|
||
{
|
||
TryDeleteDirectory(builderResult.WorkingDirectory);
|
||
}
|
||
|
||
if (!_options.KeepWorkingTree)
|
||
{
|
||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||
}
|
||
}
|
||
|
||
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
|
||
{
|
||
var advisories = new List<Advisory>();
|
||
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
|
||
{
|
||
if (advisory is null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
advisories.Add(advisory);
|
||
}
|
||
|
||
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
|
||
return advisories;
|
||
}
|
||
|
||
private byte[] CreateMetadataJson(
|
||
TrivyDbExportPlan plan,
|
||
ReadOnlyMemory<byte> builderMetadata,
|
||
string treeDigest,
|
||
JsonExportResult result,
|
||
DateTimeOffset exportedAt)
|
||
{
|
||
var metadata = new TrivyMetadata
|
||
{
|
||
GeneratedAt = exportedAt.UtcDateTime,
|
||
AdvisoryCount = result.AdvisoryCount,
|
||
TreeDigest = treeDigest,
|
||
TreeBytes = result.TotalBytes,
|
||
ExporterVersion = _exporterVersion,
|
||
Builder = ParseBuilderMetadata(builderMetadata.Span),
|
||
Mode = plan.Mode switch
|
||
{
|
||
TrivyDbExportMode.Full => "full",
|
||
TrivyDbExportMode.Delta => "delta",
|
||
TrivyDbExportMode.Skip => "skip",
|
||
_ => "unknown",
|
||
},
|
||
BaseExportId = plan.BaseExportId,
|
||
BaseManifestDigest = plan.BaseManifestDigest,
|
||
ResetBaseline = plan.ResetBaseline,
|
||
Delta = plan.Mode == TrivyDbExportMode.Delta
|
||
? new DeltaMetadata(
|
||
plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(),
|
||
plan.RemovedPaths.ToArray())
|
||
: null,
|
||
};
|
||
|
||
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
|
||
{
|
||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||
WriteIndented = false,
|
||
});
|
||
}
|
||
|
||
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
|
||
{
|
||
if (payload.IsEmpty)
|
||
{
|
||
return null;
|
||
}
|
||
|
||
try
|
||
{
|
||
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
|
||
{
|
||
PropertyNameCaseInsensitive = true,
|
||
});
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
|
||
{
|
||
if (!_options.OfflineBundle.Enabled)
|
||
{
|
||
return;
|
||
}
|
||
|
||
if (!ShouldIncludeInBundle(mode))
|
||
{
|
||
return;
|
||
}
|
||
|
||
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
|
||
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
|
||
? $"{exportId}.offline.tar.gz"
|
||
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
|
||
|
||
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
|
||
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
|
||
|
||
if (File.Exists(bundlePath))
|
||
{
|
||
File.Delete(bundlePath);
|
||
}
|
||
|
||
var normalizedRoot = Path.GetFullPath(layoutPath);
|
||
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
|
||
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
|
||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||
.ToArray();
|
||
|
||
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
|
||
.Select(file => NormalizeTarPath(normalizedRoot, file))
|
||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||
.ToArray();
|
||
|
||
await using (var archiveStream = new FileStream(
|
||
bundlePath,
|
||
FileMode.Create,
|
||
FileAccess.Write,
|
||
FileShare.None,
|
||
bufferSize: 81920,
|
||
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
|
||
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
|
||
{
|
||
var timestamp = exportedAt.UtcDateTime;
|
||
|
||
foreach (var directory in directories)
|
||
{
|
||
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
|
||
{
|
||
ModificationTime = timestamp,
|
||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
|
||
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
|
||
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
|
||
};
|
||
|
||
writer.WriteEntry(entry);
|
||
}
|
||
|
||
foreach (var relativePath in files)
|
||
{
|
||
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
|
||
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
|
||
{
|
||
ModificationTime = timestamp,
|
||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
|
||
UnixFileMode.GroupRead |
|
||
UnixFileMode.OtherRead,
|
||
};
|
||
|
||
await using var source = new FileStream(
|
||
fullPath,
|
||
FileMode.Open,
|
||
FileAccess.Read,
|
||
FileShare.Read,
|
||
bufferSize: 81920,
|
||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||
entry.DataStream = source;
|
||
writer.WriteEntry(entry);
|
||
}
|
||
}
|
||
|
||
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
|
||
|
||
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
|
||
var length = new FileInfo(bundlePath).Length;
|
||
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
|
||
}
|
||
|
||
private static void TryDeleteDirectory(string directory)
|
||
{
|
||
try
|
||
{
|
||
if (Directory.Exists(directory))
|
||
{
|
||
Directory.Delete(directory, recursive: true);
|
||
}
|
||
}
|
||
catch
|
||
{
|
||
// Best effort cleanup – ignore failures.
|
||
}
|
||
}
|
||
|
||
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
||
{
|
||
await using var stream = new FileStream(
|
||
archivePath,
|
||
FileMode.Open,
|
||
FileAccess.ReadWrite,
|
||
FileShare.None,
|
||
bufferSize: 8,
|
||
options: FileOptions.Asynchronous);
|
||
|
||
if (stream.Length < 10)
|
||
{
|
||
return;
|
||
}
|
||
|
||
stream.Position = 4;
|
||
var zeros = new byte[4];
|
||
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||
}
|
||
|
||
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
|
||
{
|
||
await using var stream = new FileStream(
|
||
path,
|
||
FileMode.Open,
|
||
FileAccess.Read,
|
||
FileShare.Read,
|
||
bufferSize: 81920,
|
||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||
}
|
||
|
||
private static string NormalizeTarPath(string root, string fullPath)
|
||
{
|
||
var relative = Path.GetRelativePath(root, fullPath);
|
||
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
|
||
return string.IsNullOrEmpty(normalized) ? "." : normalized;
|
||
}
|
||
|
||
private bool ShouldPublishToOras(TrivyDbExportMode mode)
|
||
{
|
||
var overrides = TrivyDbExportOverrideScope.Current;
|
||
|
||
return mode switch
|
||
{
|
||
TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull,
|
||
TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta,
|
||
_ => false,
|
||
};
|
||
}
|
||
|
||
private bool ShouldIncludeInBundle(TrivyDbExportMode mode)
|
||
{
|
||
var overrides = TrivyDbExportOverrideScope.Current;
|
||
|
||
return mode switch
|
||
{
|
||
TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull,
|
||
TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta,
|
||
_ => false,
|
||
};
|
||
}
|
||
|
||
private sealed record DeltaMetadata(
|
||
IReadOnlyList<DeltaFileMetadata> ChangedFiles,
|
||
IReadOnlyList<string> RemovedPaths);
|
||
|
||
private sealed record DeltaFileMetadata(string Path, long Length, string Digest);
|
||
|
||
private sealed class TrivyMetadata
|
||
{
|
||
public DateTime GeneratedAt { get; set; }
|
||
|
||
public int AdvisoryCount { get; set; }
|
||
|
||
public string TreeDigest { get; set; } = string.Empty;
|
||
|
||
public long TreeBytes { get; set; }
|
||
|
||
public string ExporterVersion { get; set; } = string.Empty;
|
||
|
||
public BuilderMetadata? Builder { get; set; }
|
||
|
||
public string Mode { get; set; } = "full";
|
||
|
||
public string? BaseExportId { get; set; }
|
||
|
||
public string? BaseManifestDigest { get; set; }
|
||
|
||
public bool ResetBaseline { get; set; }
|
||
|
||
public DeltaMetadata? Delta { get; set; }
|
||
}
|
||
|
||
private sealed class BuilderMetadata
|
||
{
|
||
[JsonPropertyName("Version")]
|
||
public int Version { get; set; }
|
||
|
||
public DateTime NextUpdate { get; set; }
|
||
|
||
public DateTime UpdatedAt { get; set; }
|
||
|
||
public DateTime? DownloadedAt { get; set; }
|
||
}
|
||
}
|