using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using System.Formats.Tar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Concelier.Exporter.Json; using StellaOps.Concelier.Models; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Exporting; using StellaOps.Plugin; namespace StellaOps.Concelier.Exporter.TrivyDb; public sealed class TrivyDbFeedExporter : IFeedExporter { public const string ExporterName = "trivy-db"; public const string ExporterId = "export:trivy-db"; private readonly IAdvisoryStore _advisoryStore; private readonly IJsonExportPathResolver _pathResolver; private readonly TrivyDbExportOptions _options; private readonly TrivyDbPackageBuilder _packageBuilder; private readonly TrivyDbOciWriter _ociWriter; private readonly ExportStateManager _stateManager; private readonly TrivyDbExportPlanner _exportPlanner; private readonly ITrivyDbBuilder _builder; private readonly ITrivyDbOrasPusher _orasPusher; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; private readonly string _exporterVersion; public TrivyDbFeedExporter( IAdvisoryStore advisoryStore, IJsonExportPathResolver pathResolver, IOptions options, TrivyDbPackageBuilder packageBuilder, TrivyDbOciWriter ociWriter, ExportStateManager stateManager, TrivyDbExportPlanner exportPlanner, ITrivyDbBuilder builder, ITrivyDbOrasPusher orasPusher, ILogger logger, TimeProvider? timeProvider = null) { _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder)); _ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter)); _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); _exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner)); _builder = builder ?? throw new ArgumentNullException(nameof(builder)); _orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; _exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter)); } public string Name => ExporterName; public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) { var exportedAt = _timeProvider.GetUtcNow(); var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture); var reference = $"{_options.ReferencePrefix}:{exportId}"; _logger.LogInformation("Starting Trivy DB export {ExportId}", exportId); var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver); var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false); var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false); _logger.LogInformation( "Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)", exportId, jsonResult.AdvisoryCount, jsonResult.TotalBytes); var manifest = jsonResult.Files .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) .ToArray(); var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult); var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest); if (plan.Mode == TrivyDbExportMode.Skip) { _logger.LogInformation( "Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.", exportId, plan.BaseExportId ?? "(none)"); if (!_options.KeepWorkingTree) { TryDeleteDirectory(jsonResult.ExportDirectory); } return; } if (plan.Mode == TrivyDbExportMode.Delta) { _logger.LogInformation( "Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.", exportId, plan.ChangedFiles.Count); } var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); try { var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest( metadataBytes, builderResult.ArchivePath, builderResult.ArchiveDigest, builderResult.ArchiveLength, exportedAt, exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture))); var destination = _options.GetExportRoot(exportId); string? baseLayout = null; if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId)) { baseLayout = _options.GetExportRoot(plan.BaseExportId); } var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false); if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode)) { await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false); } _logger.LogInformation( "Trivy DB export {ExportId} wrote manifest {ManifestDigest}", exportId, ociResult.ManifestDigest); var resetBaseline = plan.ResetBaseline || existingState is null || string.IsNullOrWhiteSpace(existingState.BaseExportId) || string.IsNullOrWhiteSpace(existingState.BaseDigest); if (existingState is not null && !string.IsNullOrWhiteSpace(_options.TargetRepository) && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) { resetBaseline = true; } if (plan.Mode == TrivyDbExportMode.Full || resetBaseline) { await _stateManager.StoreFullExportAsync( ExporterId, exportId, ociResult.ManifestDigest, cursor: treeDigest, targetRepository: _options.TargetRepository, exporterVersion: _exporterVersion, resetBaseline: resetBaseline, manifest: plan.Manifest, cancellationToken: cancellationToken).ConfigureAwait(false); } else { await _stateManager.StoreDeltaExportAsync( ExporterId, deltaDigest: treeDigest, cursor: treeDigest, exporterVersion: _exporterVersion, manifest: plan.Manifest, cancellationToken: cancellationToken).ConfigureAwait(false); } await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false); } finally { TryDeleteDirectory(builderResult.WorkingDirectory); } if (!_options.KeepWorkingTree) { TryDeleteDirectory(jsonResult.ExportDirectory); } } private async Task> LoadAdvisoriesAsync(CancellationToken cancellationToken) { var advisories = new List(); await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false)) { if (advisory is null) { continue; } advisories.Add(advisory); } advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey)); return advisories; } private byte[] CreateMetadataJson( TrivyDbExportPlan plan, ReadOnlyMemory builderMetadata, string treeDigest, JsonExportResult result, DateTimeOffset exportedAt) { var metadata = new TrivyMetadata { GeneratedAt = exportedAt.UtcDateTime, AdvisoryCount = result.AdvisoryCount, TreeDigest = treeDigest, TreeBytes = result.TotalBytes, ExporterVersion = _exporterVersion, Builder = ParseBuilderMetadata(builderMetadata.Span), Mode = plan.Mode switch { TrivyDbExportMode.Full => "full", TrivyDbExportMode.Delta => "delta", TrivyDbExportMode.Skip => "skip", _ => "unknown", }, BaseExportId = plan.BaseExportId, BaseManifestDigest = plan.BaseManifestDigest, ResetBaseline = plan.ResetBaseline, Delta = plan.Mode == TrivyDbExportMode.Delta ? new DeltaMetadata( plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(), plan.RemovedPaths.ToArray()) : null, }; return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, WriteIndented = false, }); } private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan payload) { if (payload.IsEmpty) { return null; } try { return JsonSerializer.Deserialize(payload, new JsonSerializerOptions { PropertyNameCaseInsensitive = true, }); } catch { return null; } } private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken) { if (!_options.OfflineBundle.Enabled) { return; } if (!ShouldIncludeInBundle(mode)) { return; } var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath; var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName) ? $"{exportId}.offline.tar.gz" : _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal); var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName); Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!); if (File.Exists(bundlePath)) { File.Delete(bundlePath); } var normalizedRoot = Path.GetFullPath(layoutPath); var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories) .Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/") .OrderBy(static path => path, StringComparer.Ordinal) .ToArray(); var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories) .Select(file => NormalizeTarPath(normalizedRoot, file)) .OrderBy(static path => path, StringComparer.Ordinal) .ToArray(); await using (var archiveStream = new FileStream( bundlePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan)) await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true)) await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false)) { var timestamp = exportedAt.UtcDateTime; foreach (var directory in directories) { var entry = new PaxTarEntry(TarEntryType.Directory, directory) { ModificationTime = timestamp, Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, }; writer.WriteEntry(entry); } foreach (var relativePath in files) { var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar)); var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath) { ModificationTime = timestamp, Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, }; await using var source = new FileStream( fullPath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan); entry.DataStream = source; writer.WriteEntry(entry); } } await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false); var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false); var length = new FileInfo(bundlePath).Length; _logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest); } private static void TryDeleteDirectory(string directory) { try { if (Directory.Exists(directory)) { Directory.Delete(directory, recursive: true); } } catch { // Best effort cleanup – ignore failures. } } private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) { await using var stream = new FileStream( archivePath, FileMode.Open, FileAccess.ReadWrite, FileShare.None, bufferSize: 8, options: FileOptions.Asynchronous); if (stream.Length < 10) { return; } stream.Position = 4; var zeros = new byte[4]; await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); } private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) { await using var stream = new FileStream( path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920, options: FileOptions.Asynchronous | FileOptions.SequentialScan); var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; } private static string NormalizeTarPath(string root, string fullPath) { var relative = Path.GetRelativePath(root, fullPath); var normalized = relative.Replace(Path.DirectorySeparatorChar, '/'); return string.IsNullOrEmpty(normalized) ? "." : normalized; } private bool ShouldPublishToOras(TrivyDbExportMode mode) { var overrides = TrivyDbExportOverrideScope.Current; return mode switch { TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull, TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta, _ => false, }; } private bool ShouldIncludeInBundle(TrivyDbExportMode mode) { var overrides = TrivyDbExportOverrideScope.Current; return mode switch { TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull, TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta, _ => false, }; } private sealed record DeltaMetadata( IReadOnlyList ChangedFiles, IReadOnlyList RemovedPaths); private sealed record DeltaFileMetadata(string Path, long Length, string Digest); private sealed class TrivyMetadata { public DateTime GeneratedAt { get; set; } public int AdvisoryCount { get; set; } public string TreeDigest { get; set; } = string.Empty; public long TreeBytes { get; set; } public string ExporterVersion { get; set; } = string.Empty; public BuilderMetadata? Builder { get; set; } public string Mode { get; set; } = "full"; public string? BaseExportId { get; set; } public string? BaseManifestDigest { get; set; } public bool ResetBaseline { get; set; } public DeltaMetadata? Delta { get; set; } } private sealed class BuilderMetadata { [JsonPropertyName("Version")] public int Version { get; set; } public DateTime NextUpdate { get; set; } public DateTime UpdatedAt { get; set; } public DateTime? DownloadedAt { get; set; } } }