Rename Feedser to Concelier

This commit is contained in:
2025-10-18 20:04:15 +03:00
parent 7e1b10d3b2
commit 0137856fdb
1208 changed files with 4370 additions and 4370 deletions

View File

@@ -0,0 +1,29 @@
# AGENTS
## Role
Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS.
## Scope
- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts).
- Output root: exports/trivy/<yyyyMMddHHmmss>; deterministic path components.
- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2).
- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json).
- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine.
- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging.
## Participants
- Storage.Mongo.AdvisoryStore as input.
- Core scheduler runs export job; WebService/Plugins trigger it.
- JSON exporter (optional precursor) if choosing the builder path.
## Interfaces & contracts
- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken).
- ConcelierOptions.packaging.trivy governs repo/tag/publish/offline_bundle.
- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs).
## In/Out of scope
In: assembling builder inputs, packing tar.gz, pushing to registry when configured.
Out: signing (external pipeline), scanner behavior.
## Observability & security expectations
- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail.
- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors.
## Tests
- Author and review coverage in `../StellaOps.Concelier.Exporter.TrivyDb.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.

View File

@@ -0,0 +1,15 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbBuilder
{
Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public interface ITrivyDbOrasPusher
{
Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciDescriptor(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations = null);

View File

@@ -0,0 +1,8 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciIndex(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("manifests")] IReadOnlyList<OciDescriptor> Manifests);

View File

@@ -0,0 +1,10 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record OciManifest(
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("config")] OciDescriptor Config,
[property: JsonPropertyName("layers")] IReadOnlyList<OciDescriptor> Layers);

View File

@@ -0,0 +1,22 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Exporter.Json\StellaOps.Concelier.Exporter.Json.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,14 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Fix method name typo GetExportRoot' -> GetExportRoot|BE-Export|Exporters|DONE `TrivyDbExportOptions.GetExportRoot` helper added with unit coverage.|
|Implement BoltDB builder integration (v0 via trivy-db CLI)|BE-Export|Env|DONE `TrivyDbBoltBuilder` shells `trivy-db build` against our JSON tree with deterministic packaging.|
|Pack db.tar.gz + metadata.json|BE-Export|Exporters|DONE Builder output re-packed with fixed timestamps and zeroed gzip mtime.|
|ORAS push support|BE-Export|Exporters|DONE Optional `TrivyDbOrasPusher` shells `oras cp --from-oci-layout` with configurable args/env.|
|Offline bundle toggle|BE-Export|Exporters|DONE Deterministic OCI layout bundle emitted when enabled.|
|Deterministic ordering of advisories|BE-Export|Models|DONE exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.|
|End-to-end tests with small dataset|QA|Exporters|DONE added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.|
|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.|
|Streamed package building to avoid large copies|BE-Export|Exporters|DONE metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.|
|Plan incremental/delta exports|BE-Export|Exporters|DONE state captures per-file manifests, planner schedules delta vs full resets, layer reuse smoke test verifies OCI reuse, and operator guide documents the validation flow.|
|Advisory schema parity export (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) exporter/test fixtures updated to handle description/CWEs/canonical metric fields during Trivy DB packaging; `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` re-run 2025-10-15 to confirm coverage.|

View File

@@ -0,0 +1,11 @@
using System;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyConfigDocument(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
[property: JsonPropertyName("databaseVersion")] string DatabaseVersion,
[property: JsonPropertyName("databaseDigest")] string DatabaseDigest,
[property: JsonPropertyName("databaseSize")] long DatabaseSize);

View File

@@ -0,0 +1,78 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBlob
{
private readonly Func<CancellationToken, ValueTask<Stream>> _openReadAsync;
private TrivyDbBlob(Func<CancellationToken, ValueTask<Stream>> openReadAsync, long length)
{
_openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync));
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Length = length;
}
public long Length { get; }
public ValueTask<Stream> OpenReadAsync(CancellationToken cancellationToken)
=> _openReadAsync(cancellationToken);
public static TrivyDbBlob FromBytes(ReadOnlyMemory<byte> payload)
{
if (payload.IsEmpty)
{
return new TrivyDbBlob(static _ => ValueTask.FromResult<Stream>(Stream.Null), 0);
}
if (MemoryMarshal.TryGetArray(payload, out ArraySegment<byte> segment) && segment.Array is not null && segment.Offset == 0)
{
return FromArray(segment.Array);
}
return FromArray(payload.ToArray());
}
public static TrivyDbBlob FromFile(string path, long length)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("File path must be provided.", nameof(path));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
return new TrivyDbBlob(
cancellationToken => ValueTask.FromResult<Stream>(new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan)),
length);
}
public static TrivyDbBlob FromArray(byte[] buffer)
{
if (buffer is null)
{
throw new ArgumentNullException(nameof(buffer));
}
return new TrivyDbBlob(
_ => ValueTask.FromResult<Stream>(new MemoryStream(buffer, writable: false)),
buffer.LongLength);
}
}

View File

@@ -0,0 +1,376 @@
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbBoltBuilder> _logger;
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<TrivyDbBuilderResult> BuildAsync(
JsonExportResult jsonTree,
DateTimeOffset exportedAt,
string exportId,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(jsonTree);
ArgumentException.ThrowIfNullOrEmpty(exportId);
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
var outputDir = Path.Combine(builderRoot, "out");
Directory.CreateDirectory(outputDir);
try
{
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
}
catch
{
TryDeleteDirectory(builderRoot);
throw;
}
var metadataPath = Path.Combine(outputDir, "metadata.json");
var dbPath = Path.Combine(outputDir, "trivy.db");
if (!File.Exists(metadataPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
}
if (!File.Exists(dbPath))
{
TryDeleteDirectory(builderRoot);
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
}
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(archivePath).Length;
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
return new TrivyDbBuilderResult(
archivePath,
digest,
length,
builderMetadata,
builderRoot);
}
private string PrepareBuilderRoot(string exportDirectory, string exportId)
{
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
return root;
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// ignore cleanup failures
}
}
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
{
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
? "trivy-db"
: builderOptions.ExecutablePath;
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("build");
startInfo.ArgumentList.Add("--cache-dir");
startInfo.ArgumentList.Add(cacheDir);
startInfo.ArgumentList.Add("--output-dir");
startInfo.ArgumentList.Add(outputDir);
if (builderOptions.UpdateInterval != default)
{
startInfo.ArgumentList.Add("--update-interval");
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
}
if (targets.Count > 0)
{
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
{
startInfo.ArgumentList.Add("--only-update");
startInfo.ArgumentList.Add(target);
}
}
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
}
if (!builderOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
foreach (var kvp in environment)
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
var stdOut = new StringBuilder();
var stdErr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdOut.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stdErr.AppendLine(e.Data);
}
};
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// Ignore kill failures.
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdOut.Length > 0)
{
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
}
if (stdErr.Length > 0)
{
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
}
}
private static async Task CreateArchiveAsync(
string archivePath,
DateTimeOffset exportedAt,
string metadataPath,
string dbPath,
CancellationToken cancellationToken)
{
await using var archiveStream = new FileStream(
archivePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
var timestamp = exportedAt.UtcDateTime;
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
};
await using var source = new FileStream(
file.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
await writer.DisposeAsync().ConfigureAwait(false);
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
}
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
{
yield return ("metadata.json", metadataPath);
yield return ("trivy.db", dbPath);
}
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static string ToGoDuration(TimeSpan span)
{
if (span <= TimeSpan.Zero)
{
return "0s";
}
span = span.Duration();
var builder = new StringBuilder();
var totalHours = (int)span.TotalHours;
if (totalHours > 0)
{
builder.Append(totalHours);
builder.Append('h');
}
var minutes = span.Minutes;
if (minutes > 0)
{
builder.Append(minutes);
builder.Append('m');
}
var seconds = span.Seconds + span.Milliseconds / 1000.0;
if (seconds > 0 || builder.Length == 0)
{
if (span.Milliseconds == 0)
{
builder.Append(span.Seconds);
}
else
{
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
}
builder.Append('s');
}
return builder.ToString();
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbBuilderResult(
string ArchivePath,
string ArchiveDigest,
long ArchiveLength,
ReadOnlyMemory<byte> BuilderMetadata,
string WorkingDirectory);

View File

@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportJob : IJob
{
public const string JobKind = "export:trivy-db";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10);
private readonly TrivyDbFeedExporter _exporter;
private readonly ILogger<TrivyDbExportJob> _logger;
public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger<TrivyDbExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId);
var overrides = CreateOverrides(context.Parameters);
if (overrides?.HasOverrides == true)
{
using var scope = TrivyDbExportOverrideScope.Begin(overrides);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
else
{
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId);
}
private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary<string, object?> parameters)
{
if (parameters is null || parameters.Count == 0)
{
return null;
}
var publishFull = TryReadBoolean(parameters, "publishFull");
var publishDelta = TryReadBoolean(parameters, "publishDelta");
var includeFull = TryReadBoolean(parameters, "includeFull");
var includeDelta = TryReadBoolean(parameters, "includeDelta");
var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta);
return overrides.HasOverrides ? overrides : null;
}
private static bool? TryReadBoolean(IReadOnlyDictionary<string, object?> parameters, string key)
{
if (!parameters.TryGetValue(key, out var value) || value is null)
{
return null;
}
switch (value)
{
case bool b:
return b;
case string s when bool.TryParse(s, out var result):
return result;
case JsonElement element:
return element.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed,
_ => null,
};
case IConvertible convertible:
try
{
return convertible.ToBoolean(CultureInfo.InvariantCulture);
}
catch
{
return null;
}
}
return null;
}
}

View File

@@ -0,0 +1,8 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public enum TrivyDbExportMode
{
Full,
Delta,
Skip,
}

View File

@@ -0,0 +1,88 @@
using System;
using System.IO;
using System.Collections.Generic;
using StellaOps.Concelier.Exporter.Json;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExportOptions
{
public string OutputRoot { get; set; } = Path.Combine("exports", "trivy");
public string ReferencePrefix { get; set; } = "concelier/trivy";
public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
public bool KeepWorkingTree { get; set; }
public string? TargetRepository { get; set; }
public JsonExportOptions Json { get; set; } = new()
{
OutputRoot = Path.Combine("exports", "trivy", "tree")
};
public TrivyDbBuilderOptions Builder { get; set; } = new();
public TrivyDbOrasOptions Oras { get; set; } = new();
public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new();
public string GetExportRoot(string exportId)
{
ArgumentException.ThrowIfNullOrEmpty(exportId);
var root = Path.GetFullPath(OutputRoot);
return Path.Combine(root, exportId);
}
}
public sealed class TrivyDbBuilderOptions
{
public string ExecutablePath { get; set; } = "trivy-db";
public string? WorkingDirectory { get; set; }
public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24);
public List<string> OnlyUpdateTargets { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool InheritEnvironment { get; set; } = true;
}
public sealed class TrivyDbOrasOptions
{
public bool Enabled { get; set; }
public string ExecutablePath { get; set; } = "oras";
public bool PublishFull { get; set; } = true;
public bool PublishDelta { get; set; } = true;
public string? WorkingDirectory { get; set; }
public bool InheritEnvironment { get; set; } = true;
public List<string> AdditionalArguments { get; set; } = new();
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
public bool SkipTlsVerify { get; set; }
public bool UseHttp { get; set; }
}
public sealed class TrivyDbOfflineBundleOptions
{
public bool Enabled { get; set; }
public bool IncludeFull { get; set; } = true;
public bool IncludeDelta { get; set; } = true;
public string? FileName { get; set; }
}

View File

@@ -0,0 +1,50 @@
using System;
using System.Threading;
namespace StellaOps.Concelier.Exporter.TrivyDb;
internal sealed record TrivyDbExportOverrides(
bool? PublishFull,
bool? PublishDelta,
bool? IncludeFull,
bool? IncludeDelta)
{
public bool HasOverrides =>
PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue;
}
internal static class TrivyDbExportOverrideScope
{
private sealed class Scope : IDisposable
{
private readonly TrivyDbExportOverrides? _previous;
private bool _disposed;
public Scope(TrivyDbExportOverrides? previous)
{
_previous = previous;
}
public void Dispose()
{
if (_disposed)
{
return;
}
_disposed = true;
CurrentOverrides.Value = _previous;
}
}
private static readonly AsyncLocal<TrivyDbExportOverrides?> CurrentOverrides = new();
public static TrivyDbExportOverrides? Current => CurrentOverrides.Value;
public static IDisposable Begin(TrivyDbExportOverrides overrides)
{
var previous = CurrentOverrides.Value;
CurrentOverrides.Value = overrides;
return new Scope(previous);
}
}

View File

@@ -0,0 +1,14 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System.Collections.Generic;
using StellaOps.Concelier.Storage.Mongo.Exporting;
public sealed record TrivyDbExportPlan(
TrivyDbExportMode Mode,
string TreeDigest,
string? BaseExportId,
string? BaseManifestDigest,
bool ResetBaseline,
IReadOnlyList<ExportFileRecord> Manifest,
IReadOnlyList<ExportFileRecord> ChangedFiles,
IReadOnlyList<string> RemovedPaths);

View File

@@ -0,0 +1,115 @@
using System;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Storage.Mongo.Exporting;
public sealed class TrivyDbExportPlanner
{
public TrivyDbExportPlan CreatePlan(
ExportStateRecord? existingState,
string treeDigest,
IReadOnlyList<ExportFileRecord> manifest)
{
ArgumentException.ThrowIfNullOrEmpty(treeDigest);
manifest ??= Array.Empty<ExportFileRecord>();
if (existingState is null || (existingState.Files?.Count ?? 0) == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
BaseExportId: existingState?.BaseExportId,
BaseManifestDigest: existingState?.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>();
var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal);
if (cursorMatches)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
var removed = existingMap.Keys
.Where(path => !newMap.ContainsKey(path))
.ToArray();
if (removed.Length > 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: removed);
}
var changed = new List<ExportFileRecord>();
foreach (var file in manifest)
{
if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal))
{
changed.Add(file);
}
}
if (changed.Count == 0)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: existingFiles,
ChangedFiles: Array.Empty<ExportFileRecord>(),
RemovedPaths: Array.Empty<string>());
}
var hasOutstandingDelta = existingState.LastDeltaDigest is not null;
if (hasOutstandingDelta)
{
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: true,
Manifest: manifest,
ChangedFiles: manifest,
RemovedPaths: Array.Empty<string>());
}
return new TrivyDbExportPlan(
TrivyDbExportMode.Delta,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest,
ResetBaseline: false,
Manifest: manifest,
ChangedFiles: changed,
RemovedPaths: Array.Empty<string>());
}
}

View File

@@ -0,0 +1,64 @@
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:trivyDb";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<TrivyDbExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy"));
options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree"));
options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat;
options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat;
options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix;
});
services.AddSingleton<TrivyDbPackageBuilder>();
services.AddSingleton<TrivyDbOciWriter>();
services.AddSingleton<TrivyDbExportPlanner>();
services.AddSingleton<ITrivyDbBuilder, TrivyDbBoltBuilder>();
services.AddSingleton<ITrivyDbOrasPusher, TrivyDbOrasPusher>();
services.AddSingleton<TrivyDbFeedExporter>();
services.AddTransient<TrivyDbExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind))
{
options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition(
TrivyDbExportJob.JobKind,
typeof(TrivyDbExportJob),
TrivyDbExportJob.DefaultTimeout,
TrivyDbExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
private static string Normalize(string? value, string fallback)
=> string.IsNullOrWhiteSpace(value) ? fallback : value;
}

View File

@@ -0,0 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbExporterPlugin : IExporterPlugin
{
public string Name => TrivyDbFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<TrivyDbFeedExporter>(services);
}
}

View File

@@ -0,0 +1,490 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Formats.Tar;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbFeedExporter : IFeedExporter
{
public const string ExporterName = "trivy-db";
public const string ExporterId = "export:trivy-db";
private readonly IAdvisoryStore _advisoryStore;
private readonly IJsonExportPathResolver _pathResolver;
private readonly TrivyDbExportOptions _options;
private readonly TrivyDbPackageBuilder _packageBuilder;
private readonly TrivyDbOciWriter _ociWriter;
private readonly ExportStateManager _stateManager;
private readonly TrivyDbExportPlanner _exportPlanner;
private readonly ITrivyDbBuilder _builder;
private readonly ITrivyDbOrasPusher _orasPusher;
private readonly ILogger<TrivyDbFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public TrivyDbFeedExporter(
IAdvisoryStore advisoryStore,
IJsonExportPathResolver pathResolver,
IOptions<TrivyDbExportOptions> options,
TrivyDbPackageBuilder packageBuilder,
TrivyDbOciWriter ociWriter,
ExportStateManager stateManager,
TrivyDbExportPlanner exportPlanner,
ITrivyDbBuilder builder,
ITrivyDbOrasPusher orasPusher,
ILogger<TrivyDbFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
var reference = $"{_options.ReferencePrefix}:{exportId}";
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
exportId,
jsonResult.AdvisoryCount,
jsonResult.TotalBytes);
var manifest = jsonResult.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
if (plan.Mode == TrivyDbExportMode.Skip)
{
_logger.LogInformation(
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
exportId,
plan.BaseExportId ?? "(none)");
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
return;
}
if (plan.Mode == TrivyDbExportMode.Delta)
{
_logger.LogInformation(
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
exportId,
plan.ChangedFiles.Count);
}
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
try
{
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
metadataBytes,
builderResult.ArchivePath,
builderResult.ArchiveDigest,
builderResult.ArchiveLength,
exportedAt,
exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture)));
var destination = _options.GetExportRoot(exportId);
string? baseLayout = null;
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId))
{
baseLayout = _options.GetExportRoot(plan.BaseExportId);
}
var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false);
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
{
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
exportId,
ociResult.ManifestDigest);
var resetBaseline = plan.ResetBaseline
|| existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
{
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
ociResult.ManifestDigest,
cursor: treeDigest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
await _stateManager.StoreDeltaExportAsync(
ExporterId,
deltaDigest: treeDigest,
cursor: treeDigest,
exporterVersion: _exporterVersion,
manifest: plan.Manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
}
finally
{
TryDeleteDirectory(builderResult.WorkingDirectory);
}
if (!_options.KeepWorkingTree)
{
TryDeleteDirectory(jsonResult.ExportDirectory);
}
}
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
{
var advisories = new List<Advisory>();
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
{
if (advisory is null)
{
continue;
}
advisories.Add(advisory);
}
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
return advisories;
}
private byte[] CreateMetadataJson(
TrivyDbExportPlan plan,
ReadOnlyMemory<byte> builderMetadata,
string treeDigest,
JsonExportResult result,
DateTimeOffset exportedAt)
{
var metadata = new TrivyMetadata
{
GeneratedAt = exportedAt.UtcDateTime,
AdvisoryCount = result.AdvisoryCount,
TreeDigest = treeDigest,
TreeBytes = result.TotalBytes,
ExporterVersion = _exporterVersion,
Builder = ParseBuilderMetadata(builderMetadata.Span),
Mode = plan.Mode switch
{
TrivyDbExportMode.Full => "full",
TrivyDbExportMode.Delta => "delta",
TrivyDbExportMode.Skip => "skip",
_ => "unknown",
},
BaseExportId = plan.BaseExportId,
BaseManifestDigest = plan.BaseManifestDigest,
ResetBaseline = plan.ResetBaseline,
Delta = plan.Mode == TrivyDbExportMode.Delta
? new DeltaMetadata(
plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(),
plan.RemovedPaths.ToArray())
: null,
};
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
});
}
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
{
if (payload.IsEmpty)
{
return null;
}
try
{
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true,
});
}
catch
{
return null;
}
}
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
{
if (!_options.OfflineBundle.Enabled)
{
return;
}
if (!ShouldIncludeInBundle(mode))
{
return;
}
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
? $"{exportId}.offline.tar.gz"
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
if (File.Exists(bundlePath))
{
File.Delete(bundlePath);
}
var normalizedRoot = Path.GetFullPath(layoutPath);
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
.Select(file => NormalizeTarPath(normalizedRoot, file))
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
await using (var archiveStream = new FileStream(
bundlePath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
{
var timestamp = exportedAt.UtcDateTime;
foreach (var directory in directories)
{
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
};
writer.WriteEntry(entry);
}
foreach (var relativePath in files)
{
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
{
ModificationTime = timestamp,
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
UnixFileMode.GroupRead |
UnixFileMode.OtherRead,
};
await using var source = new FileStream(
fullPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
entry.DataStream = source;
writer.WriteEntry(entry);
}
}
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
var length = new FileInfo(bundlePath).Length;
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
}
private static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch
{
// Best effort cleanup ignore failures.
}
}
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
archivePath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 8,
options: FileOptions.Asynchronous);
if (stream.Length < 10)
{
return;
}
stream.Position = 4;
var zeros = new byte[4];
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
path,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static string NormalizeTarPath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
return string.IsNullOrEmpty(normalized) ? "." : normalized;
}
private bool ShouldPublishToOras(TrivyDbExportMode mode)
{
var overrides = TrivyDbExportOverrideScope.Current;
return mode switch
{
TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull,
TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta,
_ => false,
};
}
private bool ShouldIncludeInBundle(TrivyDbExportMode mode)
{
var overrides = TrivyDbExportOverrideScope.Current;
return mode switch
{
TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull,
TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta,
_ => false,
};
}
private sealed record DeltaMetadata(
IReadOnlyList<DeltaFileMetadata> ChangedFiles,
IReadOnlyList<string> RemovedPaths);
private sealed record DeltaFileMetadata(string Path, long Length, string Digest);
private sealed class TrivyMetadata
{
public DateTime GeneratedAt { get; set; }
public int AdvisoryCount { get; set; }
public string TreeDigest { get; set; } = string.Empty;
public long TreeBytes { get; set; }
public string ExporterVersion { get; set; } = string.Empty;
public BuilderMetadata? Builder { get; set; }
public string Mode { get; set; } = "full";
public string? BaseExportId { get; set; }
public string? BaseManifestDigest { get; set; }
public bool ResetBaseline { get; set; }
public DeltaMetadata? Delta { get; set; }
}
private sealed class BuilderMetadata
{
[JsonPropertyName("Version")]
public int Version { get; set; }
public DateTime NextUpdate { get; set; }
public DateTime UpdatedAt { get; set; }
public DateTime? DownloadedAt { get; set; }
}
}

View File

@@ -0,0 +1,9 @@
namespace StellaOps.Concelier.Exporter.TrivyDb;
public static class TrivyDbMediaTypes
{
public const string OciManifest = "application/vnd.oci.image.manifest.v1+json";
public const string OciImageIndex = "application/vnd.oci.image.index.v1+json";
public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json";
public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip";
}

View File

@@ -0,0 +1,8 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbOciWriteResult(
string RootDirectory,
string ManifestDigest,
IReadOnlyCollection<string> BlobDigests);

View File

@@ -0,0 +1,375 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.TrivyDb;
/// <summary>
/// Writes a Trivy DB package to an OCI image layout directory with deterministic content.
/// </summary>
public sealed class TrivyDbOciWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}");
public async Task<TrivyDbOciWriteResult> WriteAsync(
TrivyDbPackage package,
string destination,
string reference,
TrivyDbExportPlan plan,
string? baseLayoutPath,
CancellationToken cancellationToken)
{
if (package is null)
{
throw new ArgumentNullException(nameof(package));
}
if (string.IsNullOrWhiteSpace(destination))
{
throw new ArgumentException("Destination directory must be provided.", nameof(destination));
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new ArgumentException("Reference tag must be provided.", nameof(reference));
}
if (plan is null)
{
throw new ArgumentNullException(nameof(plan));
}
var root = Path.GetFullPath(destination);
if (Directory.Exists(root))
{
Directory.Delete(root, recursive: true);
}
Directory.CreateDirectory(root);
var timestamp = package.Config.GeneratedAt.UtcDateTime;
await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false);
await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false);
var blobsRoot = Path.Combine(root, "blobs", "sha256");
Directory.CreateDirectory(blobsRoot);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp);
Directory.SetLastWriteTimeUtc(blobsRoot, timestamp);
var writtenDigests = new HashSet<string>(StringComparer.Ordinal);
foreach (var pair in package.Blobs)
{
if (writtenDigests.Contains(pair.Key))
{
continue;
}
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (reused)
{
writtenDigests.Add(pair.Key);
continue;
}
if (writtenDigests.Add(pair.Key))
{
await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false);
}
}
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions);
var manifestDigest = ComputeDigest(manifestBytes);
if (!writtenDigests.Contains(manifestDigest))
{
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
if (!reused)
{
await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false);
}
writtenDigests.Add(manifestDigest);
}
var manifestDescriptor = new OciDescriptor(
TrivyDbMediaTypes.OciManifest,
manifestDigest,
manifestBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.ref.name"] = reference,
});
var index = new OciIndex(2, new[] { manifestDescriptor });
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions);
await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath))
{
var reuseDigests = await TryReuseBaseBlobsAsync(
blobsRoot,
timestamp,
writtenDigests,
plan,
baseLayoutPath,
cancellationToken).ConfigureAwait(false);
foreach (var digest in reuseDigests)
{
writtenDigests.Add(digest);
}
}
Directory.SetLastWriteTimeUtc(root, timestamp);
var blobDigests = writtenDigests.ToArray();
Array.Sort(blobDigests, StringComparer.Ordinal);
return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests);
}
private static async Task WriteFileAsync(string path, ReadOnlyMemory<byte> bytes, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken)
{
var fileName = ResolveDigestFileName(digest);
var path = Path.Combine(blobsRoot, fileName);
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
}
await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false);
await using var destination = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, utcTimestamp);
}
private static string ResolveDigestFileName(string digest)
{
if (!digest.StartsWith("sha256:", StringComparison.Ordinal))
{
throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'.");
}
var hex = digest[7..];
if (hex.Length == 0)
{
throw new InvalidOperationException("Digest hex component cannot be empty.");
}
return hex;
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = System.Security.Cryptography.SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
private static async Task<IReadOnlyCollection<string>> TryReuseBaseBlobsAsync(
string destinationBlobsRoot,
DateTime timestamp,
HashSet<string> written,
TrivyDbExportPlan plan,
string baseLayoutPath,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest))
{
return Array.Empty<string>();
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
if (!Directory.Exists(baseRoot))
{
return Array.Empty<string>();
}
var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest);
if (!File.Exists(manifestPath))
{
return Array.Empty<string>();
}
await using var stream = new FileStream(
manifestPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
var digests = new SortedSet<string>(StringComparer.Ordinal)
{
plan.BaseManifestDigest,
};
if (root.TryGetProperty("config", out var configNode))
{
var digest = configNode.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
if (root.TryGetProperty("layers", out var layersNode))
{
foreach (var layer in layersNode.EnumerateArray())
{
var digest = layer.GetProperty("digest").GetString();
if (!string.IsNullOrWhiteSpace(digest))
{
digests.Add(digest);
}
}
}
var copied = new List<string>();
foreach (var digest in digests)
{
if (written.Contains(digest))
{
continue;
}
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
continue;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
copied.Add(digest);
}
if (copied.Count > 0)
{
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
}
return copied;
}
private static string ResolveBlobPath(string layoutRoot, string digest)
{
var fileName = ResolveDigestFileName(digest);
return Path.Combine(layoutRoot, "blobs", "sha256", fileName);
}
private static async Task<bool> TryReuseExistingBlobAsync(
string? baseLayoutPath,
string digest,
string destinationBlobsRoot,
DateTime timestamp,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(baseLayoutPath))
{
return false;
}
var baseRoot = Path.GetFullPath(baseLayoutPath);
var sourcePath = ResolveBlobPath(baseRoot, digest);
if (!File.Exists(sourcePath))
{
return false;
}
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var source = new FileStream(
sourcePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var destination = new FileStream(
destinationPath,
FileMode.Create,
FileAccess.Write,
FileShare.None,
bufferSize: 81920,
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destinationPath, timestamp);
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
return true;
}
}

View File

@@ -0,0 +1,209 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher
{
private readonly TrivyDbExportOptions _options;
private readonly ILogger<TrivyDbOrasPusher> _logger;
public TrivyDbOrasPusher(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbOrasPusher> logger)
{
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken)
{
var orasOptions = _options.Oras;
if (!orasOptions.Enabled)
{
return;
}
if (string.IsNullOrWhiteSpace(reference))
{
throw new InvalidOperationException("ORAS push requested but reference is empty.");
}
if (!Directory.Exists(layoutPath))
{
throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist.");
}
var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath;
var tag = ResolveTag(reference, exportId);
var layoutReference = $"{layoutPath}:{tag}";
var startInfo = new ProcessStartInfo
{
FileName = executable,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
};
startInfo.ArgumentList.Add("cp");
startInfo.ArgumentList.Add("--from-oci-layout");
startInfo.ArgumentList.Add(layoutReference);
if (orasOptions.SkipTlsVerify)
{
startInfo.ArgumentList.Add("--insecure");
}
if (orasOptions.UseHttp)
{
startInfo.ArgumentList.Add("--plain-http");
}
if (orasOptions.AdditionalArguments is { Count: > 0 })
{
foreach (var arg in orasOptions.AdditionalArguments)
{
if (!string.IsNullOrWhiteSpace(arg))
{
startInfo.ArgumentList.Add(arg);
}
}
}
startInfo.ArgumentList.Add(reference);
if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory))
{
startInfo.WorkingDirectory = orasOptions.WorkingDirectory;
}
if (!orasOptions.InheritEnvironment)
{
startInfo.Environment.Clear();
}
if (orasOptions.Environment is { Count: > 0 })
{
foreach (var kvp in orasOptions.Environment)
{
if (!string.IsNullOrEmpty(kvp.Key))
{
startInfo.Environment[kvp.Key] = kvp.Value;
}
}
}
using var process = new Process { StartInfo = startInfo };
var stdout = new StringBuilder();
var stderr = new StringBuilder();
var stdoutCompletion = new TaskCompletionSource<object?>();
var stderrCompletion = new TaskCompletionSource<object?>();
process.OutputDataReceived += (_, e) =>
{
if (e.Data is null)
{
stdoutCompletion.TrySetResult(null);
}
else
{
stdout.AppendLine(e.Data);
}
};
process.ErrorDataReceived += (_, e) =>
{
if (e.Data is null)
{
stderrCompletion.TrySetResult(null);
}
else
{
stderr.AppendLine(e.Data);
}
};
_logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable);
try
{
if (!process.Start())
{
throw new InvalidOperationException($"Failed to start '{executable}'.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
}
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var registration = cancellationToken.Register(() =>
{
try
{
if (!process.HasExited)
{
process.Kill(entireProcessTree: true);
}
}
catch
{
// ignore
}
});
#if NET8_0_OR_GREATER
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
#else
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
#endif
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
if (process.ExitCode != 0)
{
_logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString());
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
}
if (stdout.Length > 0)
{
_logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString());
}
if (stderr.Length > 0)
{
_logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString());
}
}
private static string ResolveTag(string reference, string fallback)
{
if (string.IsNullOrWhiteSpace(reference))
{
return fallback;
}
var atIndex = reference.IndexOf('@');
if (atIndex >= 0)
{
reference = reference[..atIndex];
}
var slashIndex = reference.LastIndexOf('/');
var colonIndex = reference.LastIndexOf(':');
if (colonIndex > slashIndex && colonIndex >= 0)
{
return reference[(colonIndex + 1)..];
}
return fallback;
}
}

View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackage(
OciManifest Manifest,
TrivyConfigDocument Config,
IReadOnlyDictionary<string, TrivyDbBlob> Blobs,
ReadOnlyMemory<byte> MetadataJson);

View File

@@ -0,0 +1,116 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed class TrivyDbPackageBuilder
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (request.MetadataJson.IsEmpty)
{
throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request));
}
if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath))
{
throw new ArgumentException("Database archive path must be provided.", nameof(request));
}
if (!File.Exists(request.DatabaseArchivePath))
{
throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath);
}
if (string.IsNullOrWhiteSpace(request.DatabaseDigest))
{
throw new ArgumentException("Database archive digest must be provided.", nameof(request));
}
if (request.DatabaseLength < 0)
{
throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength));
}
var metadataBytes = request.MetadataJson;
var generatedAt = request.GeneratedAt.ToUniversalTime();
var configDocument = new TrivyConfigDocument(
TrivyDbMediaTypes.TrivyConfig,
generatedAt,
request.DatabaseVersion,
request.DatabaseDigest,
request.DatabaseLength);
var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions);
var configDigest = ComputeDigest(configBytes);
var configDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyConfig,
configDigest,
configBytes.LongLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "config.json",
});
var layerDescriptor = new OciDescriptor(
TrivyDbMediaTypes.TrivyLayer,
request.DatabaseDigest,
request.DatabaseLength,
new Dictionary<string, string>
{
["org.opencontainers.image.title"] = "db.tar.gz",
});
var manifest = new OciManifest(
2,
TrivyDbMediaTypes.OciManifest,
configDescriptor,
ImmutableArray.Create(layerDescriptor));
var blobs = new SortedDictionary<string, TrivyDbBlob>(StringComparer.Ordinal)
{
[configDigest] = TrivyDbBlob.FromBytes(configBytes),
[request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength),
};
return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes);
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash);
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
buffer[0] = 's';
buffer[1] = 'h';
buffer[2] = 'a';
buffer[3] = '2';
buffer[4] = '5';
buffer[5] = '6';
buffer[6] = ':';
for (var i = 0; i < hex.Length; i++)
{
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
}
return new string(buffer);
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace StellaOps.Concelier.Exporter.TrivyDb;
public sealed record TrivyDbPackageRequest(
ReadOnlyMemory<byte> MetadataJson,
string DatabaseArchivePath,
string DatabaseDigest,
long DatabaseLength,
DateTimeOffset GeneratedAt,
string DatabaseVersion);