Rename Feedser to Concelier
This commit is contained in:
490
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs
Normal file
490
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs
Normal file
@@ -0,0 +1,490 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Formats.Tar;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbFeedExporter : IFeedExporter
|
||||
{
|
||||
public const string ExporterName = "trivy-db";
|
||||
public const string ExporterId = "export:trivy-db";
|
||||
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
private readonly TrivyDbExportOptions _options;
|
||||
private readonly TrivyDbPackageBuilder _packageBuilder;
|
||||
private readonly TrivyDbOciWriter _ociWriter;
|
||||
private readonly ExportStateManager _stateManager;
|
||||
private readonly TrivyDbExportPlanner _exportPlanner;
|
||||
private readonly ITrivyDbBuilder _builder;
|
||||
private readonly ITrivyDbOrasPusher _orasPusher;
|
||||
private readonly ILogger<TrivyDbFeedExporter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _exporterVersion;
|
||||
|
||||
public TrivyDbFeedExporter(
|
||||
IAdvisoryStore advisoryStore,
|
||||
IJsonExportPathResolver pathResolver,
|
||||
IOptions<TrivyDbExportOptions> options,
|
||||
TrivyDbPackageBuilder packageBuilder,
|
||||
TrivyDbOciWriter ociWriter,
|
||||
ExportStateManager stateManager,
|
||||
TrivyDbExportPlanner exportPlanner,
|
||||
ITrivyDbBuilder builder,
|
||||
ITrivyDbOrasPusher orasPusher,
|
||||
ILogger<TrivyDbFeedExporter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
|
||||
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
|
||||
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
|
||||
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
|
||||
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
|
||||
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
|
||||
}
|
||||
|
||||
public string Name => ExporterName;
|
||||
|
||||
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var exportedAt = _timeProvider.GetUtcNow();
|
||||
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
|
||||
var reference = $"{_options.ReferencePrefix}:{exportId}";
|
||||
|
||||
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
|
||||
|
||||
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
|
||||
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
|
||||
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
|
||||
exportId,
|
||||
jsonResult.AdvisoryCount,
|
||||
jsonResult.TotalBytes);
|
||||
|
||||
var manifest = jsonResult.Files
|
||||
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
|
||||
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
|
||||
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Skip)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
|
||||
exportId,
|
||||
plan.BaseExportId ?? "(none)");
|
||||
|
||||
if (!_options.KeepWorkingTree)
|
||||
{
|
||||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Delta)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
|
||||
exportId,
|
||||
plan.ChangedFiles.Count);
|
||||
}
|
||||
|
||||
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
|
||||
|
||||
try
|
||||
{
|
||||
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
|
||||
metadataBytes,
|
||||
builderResult.ArchivePath,
|
||||
builderResult.ArchiveDigest,
|
||||
builderResult.ArchiveLength,
|
||||
exportedAt,
|
||||
exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture)));
|
||||
|
||||
var destination = _options.GetExportRoot(exportId);
|
||||
string? baseLayout = null;
|
||||
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId))
|
||||
{
|
||||
baseLayout = _options.GetExportRoot(plan.BaseExportId);
|
||||
}
|
||||
|
||||
var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
|
||||
{
|
||||
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
|
||||
exportId,
|
||||
ociResult.ManifestDigest);
|
||||
|
||||
var resetBaseline = plan.ResetBaseline
|
||||
|| existingState is null
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
|
||||
|
||||
if (existingState is not null
|
||||
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
|
||||
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
|
||||
{
|
||||
resetBaseline = true;
|
||||
}
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
|
||||
{
|
||||
await _stateManager.StoreFullExportAsync(
|
||||
ExporterId,
|
||||
exportId,
|
||||
ociResult.ManifestDigest,
|
||||
cursor: treeDigest,
|
||||
targetRepository: _options.TargetRepository,
|
||||
exporterVersion: _exporterVersion,
|
||||
resetBaseline: resetBaseline,
|
||||
manifest: plan.Manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _stateManager.StoreDeltaExportAsync(
|
||||
ExporterId,
|
||||
deltaDigest: treeDigest,
|
||||
cursor: treeDigest,
|
||||
exporterVersion: _exporterVersion,
|
||||
manifest: plan.Manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDeleteDirectory(builderResult.WorkingDirectory);
|
||||
}
|
||||
|
||||
if (!_options.KeepWorkingTree)
|
||||
{
|
||||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var advisories = new List<Advisory>();
|
||||
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
|
||||
return advisories;
|
||||
}
|
||||
|
||||
private byte[] CreateMetadataJson(
|
||||
TrivyDbExportPlan plan,
|
||||
ReadOnlyMemory<byte> builderMetadata,
|
||||
string treeDigest,
|
||||
JsonExportResult result,
|
||||
DateTimeOffset exportedAt)
|
||||
{
|
||||
var metadata = new TrivyMetadata
|
||||
{
|
||||
GeneratedAt = exportedAt.UtcDateTime,
|
||||
AdvisoryCount = result.AdvisoryCount,
|
||||
TreeDigest = treeDigest,
|
||||
TreeBytes = result.TotalBytes,
|
||||
ExporterVersion = _exporterVersion,
|
||||
Builder = ParseBuilderMetadata(builderMetadata.Span),
|
||||
Mode = plan.Mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => "full",
|
||||
TrivyDbExportMode.Delta => "delta",
|
||||
TrivyDbExportMode.Skip => "skip",
|
||||
_ => "unknown",
|
||||
},
|
||||
BaseExportId = plan.BaseExportId,
|
||||
BaseManifestDigest = plan.BaseManifestDigest,
|
||||
ResetBaseline = plan.ResetBaseline,
|
||||
Delta = plan.Mode == TrivyDbExportMode.Delta
|
||||
? new DeltaMetadata(
|
||||
plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(),
|
||||
plan.RemovedPaths.ToArray())
|
||||
: null,
|
||||
};
|
||||
|
||||
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
});
|
||||
}
|
||||
|
||||
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
});
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_options.OfflineBundle.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ShouldIncludeInBundle(mode))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
|
||||
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
|
||||
? $"{exportId}.offline.tar.gz"
|
||||
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
|
||||
|
||||
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
|
||||
|
||||
if (File.Exists(bundlePath))
|
||||
{
|
||||
File.Delete(bundlePath);
|
||||
}
|
||||
|
||||
var normalizedRoot = Path.GetFullPath(layoutPath);
|
||||
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
|
||||
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
|
||||
.Select(file => NormalizeTarPath(normalizedRoot, file))
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
await using (var archiveStream = new FileStream(
|
||||
bundlePath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
|
||||
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
|
||||
{
|
||||
var timestamp = exportedAt.UtcDateTime;
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
|
||||
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
|
||||
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
|
||||
};
|
||||
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
|
||||
foreach (var relativePath in files)
|
||||
{
|
||||
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
|
||||
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
|
||||
UnixFileMode.GroupRead |
|
||||
UnixFileMode.OtherRead,
|
||||
};
|
||||
|
||||
await using var source = new FileStream(
|
||||
fullPath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
entry.DataStream = source;
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
}
|
||||
|
||||
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
|
||||
var length = new FileInfo(bundlePath).Length;
|
||||
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort cleanup – ignore failures.
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 8,
|
||||
options: FileOptions.Asynchronous);
|
||||
|
||||
if (stream.Length < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
stream.Position = 4;
|
||||
var zeros = new byte[4];
|
||||
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string NormalizeTarPath(string root, string fullPath)
|
||||
{
|
||||
var relative = Path.GetRelativePath(root, fullPath);
|
||||
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
|
||||
return string.IsNullOrEmpty(normalized) ? "." : normalized;
|
||||
}
|
||||
|
||||
private bool ShouldPublishToOras(TrivyDbExportMode mode)
|
||||
{
|
||||
var overrides = TrivyDbExportOverrideScope.Current;
|
||||
|
||||
return mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull,
|
||||
TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta,
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
|
||||
private bool ShouldIncludeInBundle(TrivyDbExportMode mode)
|
||||
{
|
||||
var overrides = TrivyDbExportOverrideScope.Current;
|
||||
|
||||
return mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull,
|
||||
TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta,
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
|
||||
private sealed record DeltaMetadata(
|
||||
IReadOnlyList<DeltaFileMetadata> ChangedFiles,
|
||||
IReadOnlyList<string> RemovedPaths);
|
||||
|
||||
private sealed record DeltaFileMetadata(string Path, long Length, string Digest);
|
||||
|
||||
private sealed class TrivyMetadata
|
||||
{
|
||||
public DateTime GeneratedAt { get; set; }
|
||||
|
||||
public int AdvisoryCount { get; set; }
|
||||
|
||||
public string TreeDigest { get; set; } = string.Empty;
|
||||
|
||||
public long TreeBytes { get; set; }
|
||||
|
||||
public string ExporterVersion { get; set; } = string.Empty;
|
||||
|
||||
public BuilderMetadata? Builder { get; set; }
|
||||
|
||||
public string Mode { get; set; } = "full";
|
||||
|
||||
public string? BaseExportId { get; set; }
|
||||
|
||||
public string? BaseManifestDigest { get; set; }
|
||||
|
||||
public bool ResetBaseline { get; set; }
|
||||
|
||||
public DeltaMetadata? Delta { get; set; }
|
||||
}
|
||||
|
||||
private sealed class BuilderMetadata
|
||||
{
|
||||
[JsonPropertyName("Version")]
|
||||
public int Version { get; set; }
|
||||
|
||||
public DateTime NextUpdate { get; set; }
|
||||
|
||||
public DateTime UpdatedAt { get; set; }
|
||||
|
||||
public DateTime? DownloadedAt { get; set; }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user