Restructure solution layout by module

This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions

View File

@@ -0,0 +1,29 @@
# AGENTS
## Role
Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS.
## Scope
- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts).
- Output root: exports/trivy/<yyyyMMddHHmmss>; deterministic path components.
- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2).
- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json).
- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine.
- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging.
## Participants
- Storage.Mongo.AdvisoryStore as input.
- Core scheduler runs export job; WebService/Plugins trigger it.
- JSON exporter (optional precursor) if choosing the builder path.
## Interfaces & contracts
- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken).
- ConcelierOptions.packaging.trivy governs repo/tag/publish/offline_bundle.
- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs).
## In/Out of scope
In: assembling builder inputs, packing tar.gz, pushing to registry when configured.
Out: signing (external pipeline), scanner behavior.
## Observability & security expectations
- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail.
- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors.
## Tests
- Author and review coverage in `../StellaOps.Concelier.Exporter.TrivyDb.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.

View File

@@ -0,0 +1,15 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbBuilder
{
Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbOrasPusher
{
Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciDescriptor(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations = null);

View File

@@ -0,0 +1,8 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciIndex(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("manifests")] IReadOnlyList<OciDescriptor> Manifests);

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciManifest(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("config")] OciDescriptor Config,
[property: JsonPropertyName("layers")] IReadOnlyList<OciDescriptor> Layers);

View File

@@ -0,0 +1,22 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Exporter.Json\StellaOps.Concelier.Exporter.Json.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,15 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Fix method name typo GetExportRoot' -> GetExportRoot|BE-Export|Exporters|DONE `TrivyDbExportOptions.GetExportRoot` helper added with unit coverage.|
|Implement BoltDB builder integration (v0 via trivy-db CLI)|BE-Export|Env|DONE `TrivyDbBoltBuilder` shells `trivy-db build` against our JSON tree with deterministic packaging.|
|Pack db.tar.gz + metadata.json|BE-Export|Exporters|DONE Builder output re-packed with fixed timestamps and zeroed gzip mtime.|
|ORAS push support|BE-Export|Exporters|DONE Optional `TrivyDbOrasPusher` shells `oras cp --from-oci-layout` with configurable args/env.|
|Offline bundle toggle|BE-Export|Exporters|DONE Deterministic OCI layout bundle emitted when enabled.|
|Deterministic ordering of advisories|BE-Export|Models|DONE exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.|
|End-to-end tests with small dataset|QA|Exporters|DONE added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.|
|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.|
|Streamed package building to avoid large copies|BE-Export|Exporters|DONE metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.|
|Plan incremental/delta exports|BE-Export|Exporters|DONE state captures per-file manifests, planner schedules delta vs full resets, layer reuse smoke test verifies OCI reuse, and operator guide documents the validation flow.|
|Advisory schema parity export (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) exporter/test fixtures updated to handle description/CWEs/canonical metric fields during Trivy DB packaging; `dotnet test src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Exporter.TrivyDb.Tests` re-run 2025-10-15 to confirm coverage.|
|CONCELIER-EXPORT-08-202 Mirror-ready Trivy DB bundles|Team Concelier Export|CONCELIER-EXPORT-08-201|**DONE (2025-10-19)** Added mirror export options and writer emitting `mirror/index.json` plus per-domain `manifest.json`/`metadata.json`/`db.tar.gz` with deterministic SHA-256 digests; regression covered via `dotnet test src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj`.|

View File

@@ -0,0 +1,11 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyConfigDocument(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
[property: JsonPropertyName("databaseVersion")] string DatabaseVersion,
[property: JsonPropertyName("databaseDigest")] string DatabaseDigest,
[property: JsonPropertyName("databaseSize")] long DatabaseSize);

View File

@@ -0,0 +1,78 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBlob
{
private readonly Func<CancellationToken, ValueTask<Stream>> _openReadAsync;
private TrivyDbBlob(Func<CancellationToken, ValueTask<Stream>> openReadAsync, long length)
{
_openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync));
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Length = length;
}
public long Length { get; }
public ValueTask<Stream> OpenReadAsync(CancellationToken cancellationToken)
=> _openReadAsync(cancellationToken);
public static TrivyDbBlob FromBytes(ReadOnlyMemory<byte> payload)
{
if (payload.IsEmpty)
{
return new TrivyDbBlob(static _ => ValueTask.FromResult<Stream>(Stream.Null), 0);
}
if (MemoryMarshal.TryGetArray(payload, out ArraySegment<byte> segment) && segment.Array is not null && segment.Offset == 0)
{
return FromArray(segment.Array);
}
return FromArray(payload.ToArray());
}
public static TrivyDbBlob FromFile(string path, long length)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("File path must be provided.", nameof(path));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
return new TrivyDbBlob(
cancellationToken => ValueTask.FromResult<Stream>(new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan)),
length);
}
public static TrivyDbBlob FromArray(byte[] buffer)
{
if (buffer is null)
{
throw new ArgumentNullException(nameof(buffer));
}
return new TrivyDbBlob(
_ => ValueTask.FromResult<Stream>(new MemoryStream(buffer, writable: false)),
buffer.LongLength);
}
}

View File

@@ -0,0 +1,376 @@
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbBoltBuilder> _logger;
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(jsonTree);
ArgumentException.ThrowIfNullOrEmpty(exportId);
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
var outputDir = Path.Combine(builderRoot, "out");
Directory.CreateDirectory(outputDir);
try
{
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
}
catch
{
TryDeleteDirectory(builderRoot);
throw;
}
var metadataPath = Path.Combine(outputDir, "metadata.json");
var dbPath = Path.Combine(outputDir, "trivy.db");
if (!File.Exists(metadataPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
}
if (!File.Exists(dbPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
}
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(archivePath).Length;
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
return new TrivyDbBuilderResult(
archivePath,
digest,
length,
builderMetadata,
builderRoot);
}
private string PrepareBuilderRoot(string exportDirectory, string exportId)
{
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
return root;
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// ignore cleanup failures
}
}
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
{
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
? "trivy-db"
: builderOptions.ExecutablePath;
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("build");
startInfo.ArgumentList.Add("--cache-dir");
startInfo.ArgumentList.Add(cacheDir);
startInfo.ArgumentList.Add("--output-dir");
startInfo.ArgumentList.Add(outputDir);
if (builderOptions.UpdateInterval != default)
{
startInfo.ArgumentList.Add("--update-interval");
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
}
if (targets.Count > 0)
{
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
{
startInfo.ArgumentList.Add("--only-update");
startInfo.ArgumentList.Add(target);
}
}
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
}
if (!builderOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
foreach (var kvp in environment)
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
var stdOut = new StringBuilder();
var stdErr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdOut.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stdErr.AppendLine(e.Data);
}
};
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// Ignore kill failures.
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdOut.Length > 0)
{
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
}
if (stdErr.Length > 0)
{
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
}
}
private static async Task CreateArchiveAsync(
string archivePath,
DateTimeOffset exportedAt,
string metadataPath,
string dbPath,
CancellationToken cancellationToken)
{
await using var archiveStream = new FileStream(
archivePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
var timestamp = exportedAt.UtcDateTime;
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
};
await using var source = new FileStream(
file.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
await writer.DisposeAsync().ConfigureAwait(false);
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
}
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
{
yield return ("metadata.json", metadataPath);
yield return ("trivy.db", dbPath);
}
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static string ToGoDuration(TimeSpan span)
{
if (span <= TimeSpan.Zero)
{
return "0s";
}
span = span.Duration();
var builder = new StringBuilder();
var totalHours = (int)span.TotalHours;
if (totalHours > 0)
{
builder.Append(totalHours);
builder.Append('h');
}
var minutes = span.Minutes;
if (minutes > 0)
{
builder.Append(minutes);
builder.Append('m');
}
var seconds = span.Seconds + span.Milliseconds / 1000.0;
if (seconds > 0 || builder.Length == 0)
{
if (span.Milliseconds == 0)
{
builder.Append(span.Seconds);
}
else
{
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
}
builder.Append('s');
}
return builder.ToString();
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbBuilderResult(
string ArchivePath,
string ArchiveDigest,
long ArchiveLength,
ReadOnlyMemory<byte> BuilderMetadata,
string WorkingDirectory);

View File

@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportJob : IJob
{
public const string JobKind = "export:trivy-db";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10);
private readonly TrivyDbFeedExporter _exporter;
private readonly ILogger<TrivyDbExportJob> _logger;
public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger<TrivyDbExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId);
var overrides = CreateOverrides(context.Parameters);
if (overrides?.HasOverrides == true)
{
using var scope = TrivyDbExportOverrideScope.Begin(overrides);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
else
{
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId);
}
private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary<string, object?> parameters)
{
if (parameters is null || parameters.Count == 0)
{
return null;
}
var publishFull = TryReadBoolean(parameters, "publishFull");
var publishDelta = TryReadBoolean(parameters, "publishDelta");
var includeFull = TryReadBoolean(parameters, "includeFull");
var includeDelta = TryReadBoolean(parameters, "includeDelta");
var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta);
return overrides.HasOverrides ? overrides : null;
}
private static bool? TryReadBoolean(IReadOnlyDictionary<string, object?> parameters, string key)
{
if (!parameters.TryGetValue(key, out var value) || value is null)
{
return null;
}
switch (value)
{
case bool b:
return b;
case string s when bool.TryParse(s, out var result):
return result;
case JsonElement element:
return element.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed,
_ => null,
};
case IConvertible convertible:
try
{
return convertible.ToBoolean(CultureInfo.InvariantCulture);
}
catch
{
return null;
}
}
return null;
}
}

View File

@@ -0,0 +1,8 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public enum TrivyDbExportMode
{
Full,
Delta,
Skip,
}

View File

@@ -0,0 +1,106 @@
using System;
using System.IO;
using System.Collections.Generic;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportOptions
{
public string OutputRoot { get; set; } = Path.Combine("exports", "trivy");
public string ReferencePrefix { get; set; } = "concelier/trivy";
public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public bool KeepWorkingTree { get; set; }
public string? TargetRepository { get; set; }
public JsonExportOptions Json { get; set; } = new()
{
OutputRoot = Path.Combine("exports", "trivy", "tree")
};
public TrivyDbBuilderOptions Builder { get; set; } = new();
public TrivyDbOrasOptions Oras { get; set; } = new();
public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new();
public TrivyDbMirrorOptions Mirror { get; set; } = new();
public string GetExportRoot(string exportId)
{
ArgumentException.ThrowIfNullOrEmpty(exportId);
var root = Path.GetFullPath(OutputRoot);
return Path.Combine(root, exportId);
}
}
public sealed class TrivyDbMirrorOptions
{
public bool Enabled { get; set; }
public string DirectoryName { get; set; } = "mirror";
public IList<TrivyDbMirrorDomainOptions> Domains { get; } = new List<TrivyDbMirrorDomainOptions>();
}
public sealed class TrivyDbMirrorDomainOptions
{
public string Id { get; set; } = string.Empty;
public string? DisplayName { get; set; }
}
public sealed class TrivyDbBuilderOptions
{
public string ExecutablePath { get; set; } = "trivy-db";
public string? WorkingDirectory { get; set; }
public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24);
public List<string> OnlyUpdateTargets { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool InheritEnvironment { get; set; } = true;
}
public sealed class TrivyDbOrasOptions
{
public bool Enabled { get; set; }
public string ExecutablePath { get; set; } = "oras";
public bool PublishFull { get; set; } = true;
public bool PublishDelta { get; set; } = true;
public string? WorkingDirectory { get; set; }
public bool InheritEnvironment { get; set; } = true;
public List<string> AdditionalArguments { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool SkipTlsVerify { get; set; }
public bool UseHttp { get; set; }
}
public sealed class TrivyDbOfflineBundleOptions
{
public bool Enabled { get; set; }
public bool IncludeFull { get; set; } = true;
public bool IncludeDelta { get; set; } = true;
public string? FileName { get; set; }
}

View File

@@ -0,0 +1,50 @@
using System;
using System.Threading;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal sealed record TrivyDbExportOverrides(
bool? PublishFull,
bool? PublishDelta,
bool? IncludeFull,
bool? IncludeDelta)
{
public bool HasOverrides =>
PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue;
}
internal static class TrivyDbExportOverrideScope
{
private sealed class Scope : IDisposable
{
private readonly TrivyDbExportOverrides? _previous;
private bool _disposed;
public Scope(TrivyDbExportOverrides? previous)
{
_previous = previous;
}
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
CurrentOverrides.Value = _previous;
}
}
private static readonly AsyncLocal<TrivyDbExportOverrides?> CurrentOverrides = new();
public static TrivyDbExportOverrides? Current => CurrentOverrides.Value;
public static IDisposable Begin(TrivyDbExportOverrides overrides)
{
var previous = CurrentOverrides.Value;
CurrentOverrides.Value = overrides;
return new Scope(previous);
}
}

View File

@@ -0,0 +1,14 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System.Collections.Generic;
using StellaOps.Concelier.Storage.Mongo.Exporting;
public sealed record TrivyDbExportPlan(
TrivyDbExportMode Mode,
string TreeDigest,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
IReadOnlyList<ExportFileRecord> Manifest,
IReadOnlyList<ExportFileRecord> ChangedFiles,
IReadOnlyList<string> RemovedPaths);

View File

@@ -0,0 +1,115 @@
using System;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Storage.Mongo.Exporting;
public sealed class TrivyDbExportPlanner
{
public TrivyDbExportPlan CreatePlan(
ExportStateRecord? existingState,
string treeDigest,
IReadOnlyList<ExportFileRecord> manifest)
{
ArgumentException.ThrowIfNullOrEmpty(treeDigest);
manifest ??= Array.Empty<ExportFileRecord>();
if (existingState is null || (existingState.Files?.Count ?? 0) == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
BaseExportId: existingState?.BaseExportId,
BaseManifestDigest: existingState?.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>();
var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal);
if (cursorMatches)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var removed = existingMap.Keys
.Where(path => !newMap.ContainsKey(path))
.ToArray();
if (removed.Length > 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: removed);
}
var changed = new List<ExportFileRecord>();
foreach (var file in manifest)
{
if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal))
{
changed.Add(file);
}
}
if (changed.Count == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var hasOutstandingDelta = existingState.LastDeltaDigest is not null;
if (hasOutstandingDelta)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
return new TrivyDbExportPlan(
TrivyDbExportMode.Delta,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: manifest,
ChangedFiles: changed,
RemovedPaths: Array.Empty<string>());
}
}

View File

@@ -0,0 +1,64 @@
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:trivyDb";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<TrivyDbExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy"));
options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree"));
options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat;
options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat;
options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix;
});
services.AddSingleton<TrivyDbPackageBuilder>();
services.AddSingleton<TrivyDbOciWriter>();
services.AddSingleton<TrivyDbExportPlanner>();
services.AddSingleton<ITrivyDbBuilder, TrivyDbBoltBuilder>();
services.AddSingleton<ITrivyDbOrasPusher, TrivyDbOrasPusher>();
services.AddSingleton<TrivyDbFeedExporter>();
services.AddTransient<TrivyDbExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind))
{
options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition(
TrivyDbExportJob.JobKind,
typeof(TrivyDbExportJob),
TrivyDbExportJob.DefaultTimeout,
TrivyDbExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
private static string Normalize(string? value, string fallback)
=> string.IsNullOrWhiteSpace(value) ? fallback : value;
}

View File

@@ -0,0 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterPlugin : IExporterPlugin
{
public string Name => TrivyDbFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<TrivyDbFeedExporter>(services);
}
}

View File

@@ -0,0 +1,515 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbFeedExporter : IFeedExporter
{
public const string ExporterName = "trivy-db";
public const string ExporterId = "export:trivy-db";
private readonly IAdvisoryStore _advisoryStore;
private readonly IJsonExportPathResolver _pathResolver;
private readonly TrivyDbExportOptions _options;
private readonly TrivyDbPackageBuilder _packageBuilder;
private readonly TrivyDbOciWriter _ociWriter;
private readonly ExportStateManager _stateManager;
private readonly TrivyDbExportPlanner _exportPlanner;
private readonly ITrivyDbBuilder _builder;
private readonly ITrivyDbOrasPusher _orasPusher;
private readonly ILogger<TrivyDbFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public TrivyDbFeedExporter(
IAdvisoryStore advisoryStore,
IJsonExportPathResolver pathResolver,
IOptions<TrivyDbExportOptions> options,
TrivyDbPackageBuilder packageBuilder,
TrivyDbOciWriter ociWriter,
ExportStateManager stateManager,
TrivyDbExportPlanner exportPlanner,
ITrivyDbBuilder builder,
ITrivyDbOrasPusher orasPusher,
ILogger<TrivyDbFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
var reference = $"{_options.ReferencePrefix}:{exportId}";
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
exportId,
jsonResult.AdvisoryCount,
jsonResult.TotalBytes);
var manifest = jsonResult.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
if (plan.Mode == TrivyDbExportMode.Skip)
{
_logger.LogInformation(
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
exportId,
plan.BaseExportId ?? "(none)");
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
return;
}
if (plan.Mode == TrivyDbExportMode.Delta)
{
_logger.LogInformation(
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
exportId,
plan.ChangedFiles.Count);
}
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
var metadataDigest = ComputeDigest(metadataBytes);
var metadataLength = metadataBytes.LongLength;
try
{
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
metadataBytes,
builderResult.ArchivePath,
builderResult.ArchiveDigest,
builderResult.ArchiveLength,
exportedAt,
exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture)));
var destination = _options.GetExportRoot(exportId);
string? baseLayout = null;
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId))
{
baseLayout = _options.GetExportRoot(plan.BaseExportId);
}
var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false);
await TrivyDbMirrorBundleWriter.WriteAsync(
destination,
jsonResult,
_options,
plan,
builderResult,
reference,
ociResult.ManifestDigest,
metadataBytes,
metadataDigest,
metadataLength,
_exporterVersion,
exportedAt,
_logger,
cancellationToken).ConfigureAwait(false);
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
{
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
exportId,
ociResult.ManifestDigest);
var resetBaseline = plan.ResetBaseline
|| existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
{
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
ociResult.ManifestDigest,
cursor: treeDigest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
await _stateManager.StoreDeltaExportAsync(
ExporterId,
deltaDigest: treeDigest,
cursor: treeDigest,
exporterVersion: _exporterVersion,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
}
finally
{
TryDeleteDirectory(builderResult.WorkingDirectory);
}
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
}
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
{
var advisories = new List<Advisory>();
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
{
if (advisory is null)
{
continue;
}
advisories.Add(advisory);
}
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
return advisories;
}
private byte[] CreateMetadataJson(
TrivyDbExportPlan plan,
ReadOnlyMemory<byte> builderMetadata,
string treeDigest,
JsonExportResult result,
DateTimeOffset exportedAt)
{
var metadata = new TrivyMetadata
{
GeneratedAt = exportedAt.UtcDateTime,
AdvisoryCount = result.AdvisoryCount,
TreeDigest = treeDigest,
TreeBytes = result.TotalBytes,
ExporterVersion = _exporterVersion,
Builder = ParseBuilderMetadata(builderMetadata.Span),
Mode = plan.Mode switch
{
TrivyDbExportMode.Full => "full",
TrivyDbExportMode.Delta => "delta",
TrivyDbExportMode.Skip => "skip",
_ => "unknown",
},
BaseExportId = plan.BaseExportId,
BaseManifestDigest = plan.BaseManifestDigest,
ResetBaseline = plan.ResetBaseline,
Delta = plan.Mode == TrivyDbExportMode.Delta
? new DeltaMetadata(
plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(),
plan.RemovedPaths.ToArray())
: null,
};
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
});
}
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
{
if (payload.IsEmpty)
{
return null;
}
try
{
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
});
}
catch
{
return null;
}
}
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
{
if (!_options.OfflineBundle.Enabled)
{
return;
}
if (!ShouldIncludeInBundle(mode))
{
return;
}
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
? $"{exportId}.offline.tar.gz"
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
if (File.Exists(bundlePath))
{
File.Delete(bundlePath);
}
var normalizedRoot = Path.GetFullPath(layoutPath);
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(file => NormalizeTarPath(normalizedRoot, file))
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
await using (var archiveStream = new FileStream(
bundlePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
{
var timestamp = exportedAt.UtcDateTime;
foreach (var directory in directories)
{
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
};
writer.WriteEntry(entry);
}
foreach (var relativePath in files)
{
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
UnixFileMode.GroupRead |
UnixFileMode.OtherRead,
};
await using var source = new FileStream(
fullPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
}
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(bundlePath).Length;
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// Best effort cleanup ignore failures.
}
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static string NormalizeTarPath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
return string.IsNullOrEmpty(normalized) ? "." : normalized;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private bool ShouldPublishToOras(TrivyDbExportMode mode)
{
var overrides = TrivyDbExportOverrideScope.Current;
return mode switch
{
TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull,
TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta,
_ => false,
};
}
private bool ShouldIncludeInBundle(TrivyDbExportMode mode)
{
var overrides = TrivyDbExportOverrideScope.Current;
return mode switch
{
TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull,
TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta,
_ => false,
};
}
private sealed record DeltaMetadata(
IReadOnlyList<DeltaFileMetadata> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
private sealed record DeltaFileMetadata(string Path, long Length, string Digest);
private sealed class TrivyMetadata
{
public DateTime GeneratedAt { get; set; }
public int AdvisoryCount { get; set; }
public string TreeDigest { get; set; } = string.Empty;
public long TreeBytes { get; set; }
public string ExporterVersion { get; set; } = string.Empty;
public BuilderMetadata? Builder { get; set; }
public string Mode { get; set; } = "full";
public string? BaseExportId { get; set; }
public string? BaseManifestDigest { get; set; }
public bool ResetBaseline { get; set; }
public DeltaMetadata? Delta { get; set; }
}
private sealed class BuilderMetadata
{
[JsonPropertyName("Version")]
public int Version { get; set; }
public DateTime NextUpdate { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? DownloadedAt { get; set; }
}
}

View File

@@ -0,0 +1,9 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public static class TrivyDbMediaTypes
{
public const string OciManifest = "application/vnd.oci.image.manifest.v1+json";
public const string OciImageIndex = "application/vnd.oci.image.index.v1+json";
public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json";
public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip";
}

View File

@@ -0,0 +1,392 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal static class TrivyDbMirrorBundleWriter
{
private const int SchemaVersion = 1;
private const string DefaultDirectoryName = "mirror";
private const string MetadataFileName = "metadata.json";
private const string DatabaseFileName = "db.tar.gz";
private const string ManifestFileName = "manifest.json";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public static async Task WriteAsync(
string layoutRoot,
JsonExportResult jsonResult,
TrivyDbExportOptions options,
TrivyDbExportPlan plan,
TrivyDbBuilderResult builderResult,
string reference,
string manifestDigest,
ReadOnlyMemory<byte> metadataBytes,
string metadataDigest,
long metadataLength,
string exporterVersion,
DateTimeOffset exportedAt,
ILogger logger,
CancellationToken cancellationToken)
{
if (options?.Mirror is null || !options.Mirror.Enabled || options.Mirror.Domains.Count == 0)
{
return;
}
if (string.IsNullOrWhiteSpace(layoutRoot))
{
throw new ArgumentException("Layout root must be provided.", nameof(layoutRoot));
}
if (builderResult is null)
{
throw new ArgumentNullException(nameof(builderResult));
}
if (jsonResult is null)
{
throw new ArgumentNullException(nameof(jsonResult));
}
var directoryName = string.IsNullOrWhiteSpace(options.Mirror.DirectoryName)
? DefaultDirectoryName
: options.Mirror.DirectoryName.Trim();
if (directoryName.Length == 0)
{
directoryName = DefaultDirectoryName;
}
var root = Path.Combine(layoutRoot, directoryName);
Directory.CreateDirectory(root);
var timestamp = exportedAt.UtcDateTime;
TrySetDirectoryTimestamp(root, timestamp);
var advisories = jsonResult.Advisories.IsDefaultOrEmpty
? Array.Empty<Advisory>()
: jsonResult.Advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
var domains = new List<MirrorIndexDomainEntry>();
foreach (var domainOption in options.Mirror.Domains)
{
cancellationToken.ThrowIfCancellationRequested();
if (domainOption is null)
{
logger.LogWarning("Encountered null Trivy mirror domain configuration; skipping.");
continue;
}
var domainId = (domainOption.Id ?? string.Empty).Trim();
if (domainId.Length == 0)
{
logger.LogWarning("Skipping Trivy mirror domain with empty id.");
continue;
}
var displayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
? domainId
: domainOption.DisplayName!.Trim();
var domainDirectory = Path.Combine(root, domainId);
Directory.CreateDirectory(domainDirectory);
TrySetDirectoryTimestamp(domainDirectory, timestamp);
var metadataPath = Path.Combine(domainDirectory, MetadataFileName);
await WriteFileAsync(metadataPath, metadataBytes, timestamp, cancellationToken).ConfigureAwait(false);
var metadataRelativePath = ToRelativePath(layoutRoot, metadataPath);
var databasePath = Path.Combine(domainDirectory, DatabaseFileName);
await CopyDatabaseAsync(builderResult.ArchivePath, databasePath, timestamp, cancellationToken).ConfigureAwait(false);
var databaseRelativePath = ToRelativePath(layoutRoot, databasePath);
var sources = BuildSourceSummaries(advisories);
var manifestDocument = new MirrorDomainManifestDocument(
SchemaVersion,
exportedAt,
exporterVersion,
reference,
manifestDigest,
options.TargetRepository,
domainId,
displayName,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources);
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions);
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
await WriteFileAsync(manifestPath, manifestBytes, timestamp, cancellationToken).ConfigureAwait(false);
var manifestRelativePath = ToRelativePath(layoutRoot, manifestPath);
var manifestDigestValue = ComputeDigest(manifestBytes);
domains.Add(new MirrorIndexDomainEntry(
domainId,
displayName,
advisories.Length,
new MirrorFileDescriptor(manifestRelativePath, manifestBytes.LongLength, manifestDigestValue),
new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest),
new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest),
sources));
}
if (domains.Count == 0)
{
Directory.Delete(root, recursive: true);
return;
}
domains.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
var delta = plan.Mode == TrivyDbExportMode.Delta
? new MirrorDeltaMetadata(
plan.ChangedFiles.Select(static file => new MirrorDeltaFile(file.Path, file.Digest)).ToArray(),
plan.RemovedPaths.ToArray())
: null;
var indexDocument = new MirrorIndexDocument(
SchemaVersion,
exportedAt,
exporterVersion,
options.TargetRepository,
reference,
manifestDigest,
plan.Mode.ToString().ToLowerInvariant(),
plan.BaseExportId,
plan.BaseManifestDigest,
plan.ResetBaseline,
delta,
domains);
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(indexDocument, SerializerOptions);
var indexPath = Path.Combine(root, "index.json");
await WriteFileAsync(indexPath, indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
logger.LogInformation(
"Generated {DomainCount} Trivy DB mirror bundle(s) under {Directory}.",
domains.Count,
directoryName);
}
private static IReadOnlyList<TrivyMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
{
if (advisories.Count == 0)
{
return Array.Empty<TrivyMirrorSourceSummary>();
}
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provenance in advisory.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
var source = provenance.Source.Trim();
if (!builders.TryGetValue(source, out var accumulator))
{
accumulator = new SourceAccumulator();
builders[source] = accumulator;
}
accumulator.Record(provenance.RecordedAt);
if (counted.Add(source))
{
accumulator.Increment();
}
}
}
var entries = builders
.Select(static pair => new TrivyMirrorSourceSummary(
pair.Key,
pair.Value.FirstRecordedAt,
pair.Value.LastRecordedAt,
pair.Value.Count))
.OrderBy(static summary => summary.Source, StringComparer.Ordinal)
.ToArray();
return entries;
}
private static async Task CopyDatabaseAsync(
string sourcePath,
string destinationPath,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
}
private static async Task WriteFileAsync(
string path,
ReadOnlyMemory<byte> bytes,
DateTime timestamp,
CancellationToken cancellationToken)
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await using var stream = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, timestamp);
}
private static string ToRelativePath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
return string.IsNullOrEmpty(normalized) ? "." : normalized;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private static void TrySetDirectoryTimestamp(string directory, DateTime timestamp)
{
try
{
Directory.SetLastWriteTimeUtc(directory, timestamp);
}
catch
{
// Best effort ignore failures.
}
}
private sealed record MirrorIndexDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string? TargetRepository,
string Reference,
string ManifestDigest,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorDeltaMetadata? Delta,
IReadOnlyList<MirrorIndexDomainEntry> Domains);
private sealed record MirrorDeltaMetadata(
IReadOnlyList<MirrorDeltaFile> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
private sealed record MirrorDeltaFile(string Path, string Digest);
private sealed record MirrorIndexDomainEntry(
string DomainId,
string DisplayName,
int AdvisoryCount,
MirrorFileDescriptor Manifest,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorDomainManifestDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string ExporterVersion,
string Reference,
string ManifestDigest,
string? TargetRepository,
string DomainId,
string DisplayName,
string Mode,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
MirrorFileDescriptor Metadata,
MirrorFileDescriptor Database,
IReadOnlyList<TrivyMirrorSourceSummary> Sources);
private sealed record MirrorFileDescriptor(string Path, long SizeBytes, string Digest);
private sealed record TrivyMirrorSourceSummary(
string Source,
DateTimeOffset? FirstRecordedAt,
DateTimeOffset? LastRecordedAt,
int AdvisoryCount);
private sealed class SourceAccumulator
{
public DateTimeOffset? FirstRecordedAt { get; private set; }
public DateTimeOffset? LastRecordedAt { get; private set; }
public int Count { get; private set; }
public void Record(DateTimeOffset recordedAt)
{
var utc = recordedAt.ToUniversalTime();
if (FirstRecordedAt is null || utc < FirstRecordedAt.Value)
{
FirstRecordedAt = utc;
}
if (LastRecordedAt is null || utc > LastRecordedAt.Value)
{
LastRecordedAt = utc;
}
}
public void Increment() => Count++;
}
}

View File

@@ -0,0 +1,8 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbOciWriteResult(
string RootDirectory,
string ManifestDigest,
IReadOnlyCollection<string> BlobDigests);

View File

@@ -0,0 +1,375 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
/// <summary>
/// Writes a Trivy DB package to an OCI image layout directory with deterministic content.
/// </summary>
public sealed class TrivyDbOciWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}");
public async Task<TrivyDbOciWriteResult> WriteAsync(
TrivyDbPackage package,
string destination,
string reference,
TrivyDbExportPlan plan,
string? baseLayoutPath,
CancellationToken cancellationToken)
{
if (package is null)
{
throw new ArgumentNullException(nameof(package));
}
if (string.IsNullOrWhiteSpace(destination))
{
throw new ArgumentException("Destination directory must be provided.", nameof(destination));
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new ArgumentException("Reference tag must be provided.", nameof(reference));
}
if (plan is null)
{
throw new ArgumentNullException(nameof(plan));
}
var root = Path.GetFullPath(destination);
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
var timestamp = package.Config.GeneratedAt.UtcDateTime;
await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false);
await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false);
var blobsRoot = Path.Combine(root, "blobs", "sha256");
Directory.CreateDirectory(blobsRoot);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp);
Directory.SetLastWriteTimeUtc(blobsRoot, timestamp);
var writtenDigests = new HashSet<string>(StringComparer.Ordinal);
foreach (var pair in package.Blobs)
{
if (writtenDigests.Contains(pair.Key))
{
continue;
}
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (reused)
{
writtenDigests.Add(pair.Key);
continue;
}
if (writtenDigests.Add(pair.Key))
{
await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false);
}
}
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions);
var manifestDigest = ComputeDigest(manifestBytes);
if (!writtenDigests.Contains(manifestDigest))
{
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (!reused)
{
await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false);
}
writtenDigests.Add(manifestDigest);
}
var manifestDescriptor = new OciDescriptor(
TrivyDbMediaTypes.OciManifest,
manifestDigest,
manifestBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.ref.name"] = reference,
});
var index = new OciIndex(2, new[] { manifestDescriptor });
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions);
await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath))
{
var reuseDigests = await TryReuseBaseBlobsAsync(
blobsRoot,
timestamp,
writtenDigests,
plan,
baseLayoutPath,
cancellationToken).ConfigureAwait(false);
foreach (var digest in reuseDigests)
{
writtenDigests.Add(digest);
}
}
Directory.SetLastWriteTimeUtc(root, timestamp);
var blobDigests = writtenDigests.ToArray();
Array.Sort(blobDigests, StringComparer.Ordinal);
return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests);
}
private static async Task WriteFileAsync(string path, ReadOnlyMemory<byte> bytes, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var fileName = ResolveDigestFileName(digest);
var path = Path.Combine(blobsRoot, fileName);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false);
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static string ResolveDigestFileName(string digest)
{
if (!digest.StartsWith("sha256:", StringComparison.Ordinal))
{
throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'.");
}
var hex = digest[7..];
if (hex.Length == 0)
{
throw new InvalidOperationException("Digest hex component cannot be empty.");
}
return hex;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = System.Security.Cryptography.SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
private static async Task<IReadOnlyCollection<string>> TryReuseBaseBlobsAsync(
string destinationBlobsRoot,
DateTime timestamp,
HashSet<string> written,
TrivyDbExportPlan plan,
string baseLayoutPath,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest))
{
return Array.Empty<string>();
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
if (!Directory.Exists(baseRoot))
{
return Array.Empty<string>();
}
var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest);
if (!File.Exists(manifestPath))
{
return Array.Empty<string>();
}
await using var stream = new FileStream(
manifestPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
var digests = new SortedSet<string>(StringComparer.Ordinal)
{
plan.BaseManifestDigest,
};
if (root.TryGetProperty("config", out var configNode))
{
var digest = configNode.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
if (root.TryGetProperty("layers", out var layersNode))
{
foreach (var layer in layersNode.EnumerateArray())
{
var digest = layer.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
}
var copied = new List<string>();
foreach (var digest in digests)
{
if (written.Contains(digest))
{
continue;
}
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
continue;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
copied.Add(digest);
}
if (copied.Count > 0)
{
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
}
return copied;
}
private static string ResolveBlobPath(string layoutRoot, string digest)
{
var fileName = ResolveDigestFileName(digest);
return Path.Combine(layoutRoot, "blobs", "sha256", fileName);
}
private static async Task<bool> TryReuseExistingBlobAsync(
string? baseLayoutPath,
string digest,
string destinationBlobsRoot,
DateTime timestamp,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(baseLayoutPath))
{
return false;
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
return false;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
return true;
}
}

View File

@@ -0,0 +1,209 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbOrasPusher> _logger;
public TrivyDbOrasPusher(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbOrasPusher> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken)
{
var orasOptions = _options.Oras;
if (!orasOptions.Enabled)
{
return;
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("ORAS push requested but reference is empty.");
}
if (!Directory.Exists(layoutPath))
{
throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist.");
}
var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath;
var tag = ResolveTag(reference, exportId);
var layoutReference = $"{layoutPath}:{tag}";
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("cp");
startInfo.ArgumentList.Add("--from-oci-layout");
startInfo.ArgumentList.Add(layoutReference);
if (orasOptions.SkipTlsVerify)
{
startInfo.ArgumentList.Add("--insecure");
}
if (orasOptions.UseHttp)
{
startInfo.ArgumentList.Add("--plain-http");
}
if (orasOptions.AdditionalArguments is { Count: > 0 })
{
foreach (var arg in orasOptions.AdditionalArguments)
{
if (!string.IsNullOrWhiteSpace(arg))
{
startInfo.ArgumentList.Add(arg);
}
}
}
startInfo.ArgumentList.Add(reference);
if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = orasOptions.WorkingDirectory;
}
if (!orasOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
if (orasOptions.Environment is { Count: > 0 })
{
foreach (var kvp in orasOptions.Environment)
{
if (!string.IsNullOrEmpty(kvp.Key))
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
}
}
using var process = new Process { StartInfo = startInfo };
var stdout = new StringBuilder();
var stderr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdout.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stderr.AppendLine(e.Data);
}
};
_logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// ignore
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdout.Length > 0)
{
_logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString());
}
if (stderr.Length > 0)
{
_logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString());
}
}
private static string ResolveTag(string reference, string fallback)
{
if (string.IsNullOrWhiteSpace(reference))
{
return fallback;
}
var atIndex = reference.IndexOf('@');
if (atIndex >= 0)
{
reference = reference[..atIndex];
}
var slashIndex = reference.LastIndexOf('/');
var colonIndex = reference.LastIndexOf(':');
if (colonIndex > slashIndex && colonIndex >= 0)
{
return reference[(colonIndex + 1)..];
}
return fallback;
}
}

View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackage(
OciManifest Manifest,
TrivyConfigDocument Config,
IReadOnlyDictionary<string, TrivyDbBlob> Blobs,
ReadOnlyMemory<byte> MetadataJson);

View File

@@ -0,0 +1,116 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbPackageBuilder
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (request.MetadataJson.IsEmpty)
{
throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request));
}
if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath))
{
throw new ArgumentException("Database archive path must be provided.", nameof(request));
}
if (!File.Exists(request.DatabaseArchivePath))
{
throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath);
}
if (string.IsNullOrWhiteSpace(request.DatabaseDigest))
{
throw new ArgumentException("Database archive digest must be provided.", nameof(request));
}
if (request.DatabaseLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength));
}
var metadataBytes = request.MetadataJson;
var generatedAt = request.GeneratedAt.ToUniversalTime();
var configDocument = new TrivyConfigDocument(
TrivyDbMediaTypes.TrivyConfig,
generatedAt,
request.DatabaseVersion,
request.DatabaseDigest,
request.DatabaseLength);
var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions);
var configDigest = ComputeDigest(configBytes);
var configDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyConfig,
configDigest,
configBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "config.json",
});
var layerDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyLayer,
request.DatabaseDigest,
request.DatabaseLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "db.tar.gz",
});
var manifest = new OciManifest(
2,
TrivyDbMediaTypes.OciManifest,
configDescriptor,
ImmutableArray.Create(layerDescriptor));
var blobs = new SortedDictionary<string, TrivyDbBlob>(StringComparer.Ordinal)
{
[configDigest] = TrivyDbBlob.FromBytes(configBytes),
[request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength),
};
return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes);
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackageRequest(
ReadOnlyMemory<byte> MetadataJson,
string DatabaseArchivePath,
string DatabaseDigest,
long DatabaseLength,
DateTimeOffset GeneratedAt,
string DatabaseVersion);