Rename Feedser to Concelier
This commit is contained in:
29
src/StellaOps.Concelier.Exporter.TrivyDb/AGENTS.md
Normal file
29
src/StellaOps.Concelier.Exporter.TrivyDb/AGENTS.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS.
|
||||
## Scope
|
||||
- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts).
|
||||
- Output root: exports/trivy/<yyyyMMddHHmmss>; deterministic path components.
|
||||
- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2).
|
||||
- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json).
|
||||
- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine.
|
||||
- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging.
|
||||
## Participants
|
||||
- Storage.Mongo.AdvisoryStore as input.
|
||||
- Core scheduler runs export job; WebService/Plugins trigger it.
|
||||
- JSON exporter (optional precursor) if choosing the builder path.
|
||||
## Interfaces & contracts
|
||||
- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken).
|
||||
- ConcelierOptions.packaging.trivy governs repo/tag/publish/offline_bundle.
|
||||
- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs).
|
||||
## In/Out of scope
|
||||
In: assembling builder inputs, packing tar.gz, pushing to registry when configured.
|
||||
Out: signing (external pipeline), scanner behavior.
|
||||
## Observability & security expectations
|
||||
- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail.
|
||||
- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.Exporter.TrivyDb.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
|
||||
15
src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbBuilder.cs
Normal file
15
src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbBuilder.cs
Normal file
@@ -0,0 +1,15 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public interface ITrivyDbBuilder
|
||||
{
|
||||
Task<TrivyDbBuilderResult> BuildAsync(
|
||||
JsonExportResult jsonTree,
|
||||
DateTimeOffset exportedAt,
|
||||
string exportId,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public interface ITrivyDbOrasPusher
|
||||
{
|
||||
Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken);
|
||||
}
|
||||
10
src/StellaOps.Concelier.Exporter.TrivyDb/OciDescriptor.cs
Normal file
10
src/StellaOps.Concelier.Exporter.TrivyDb/OciDescriptor.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record OciDescriptor(
|
||||
[property: JsonPropertyName("mediaType")] string MediaType,
|
||||
[property: JsonPropertyName("digest")] string Digest,
|
||||
[property: JsonPropertyName("size")] long Size,
|
||||
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string>? Annotations = null);
|
||||
8
src/StellaOps.Concelier.Exporter.TrivyDb/OciIndex.cs
Normal file
8
src/StellaOps.Concelier.Exporter.TrivyDb/OciIndex.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record OciIndex(
|
||||
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
|
||||
[property: JsonPropertyName("manifests")] IReadOnlyList<OciDescriptor> Manifests);
|
||||
10
src/StellaOps.Concelier.Exporter.TrivyDb/OciManifest.cs
Normal file
10
src/StellaOps.Concelier.Exporter.TrivyDb/OciManifest.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record OciManifest(
|
||||
[property: JsonPropertyName("schemaVersion")] int SchemaVersion,
|
||||
[property: JsonPropertyName("mediaType")] string MediaType,
|
||||
[property: JsonPropertyName("config")] OciDescriptor Config,
|
||||
[property: JsonPropertyName("layers")] IReadOnlyList<OciDescriptor> Layers);
|
||||
@@ -0,0 +1,22 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Exporter.Json\StellaOps.Concelier.Exporter.Json.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
14
src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md
Normal file
14
src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Fix method name typo GetExportRoot' -> GetExportRoot|BE-Export|Exporters|DONE – `TrivyDbExportOptions.GetExportRoot` helper added with unit coverage.|
|
||||
|Implement BoltDB builder integration (v0 via trivy-db CLI)|BE-Export|Env|DONE – `TrivyDbBoltBuilder` shells `trivy-db build` against our JSON tree with deterministic packaging.|
|
||||
|Pack db.tar.gz + metadata.json|BE-Export|Exporters|DONE – Builder output re-packed with fixed timestamps and zeroed gzip mtime.|
|
||||
|ORAS push support|BE-Export|Exporters|DONE – Optional `TrivyDbOrasPusher` shells `oras cp --from-oci-layout` with configurable args/env.|
|
||||
|Offline bundle toggle|BE-Export|Exporters|DONE – Deterministic OCI layout bundle emitted when enabled.|
|
||||
|Deterministic ordering of advisories|BE-Export|Models|DONE – exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.|
|
||||
|End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.|
|
||||
|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.|
|
||||
|Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.|
|
||||
|Plan incremental/delta exports|BE-Export|Exporters|DONE – state captures per-file manifests, planner schedules delta vs full resets, layer reuse smoke test verifies OCI reuse, and operator guide documents the validation flow.|
|
||||
|Advisory schema parity export (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – exporter/test fixtures updated to handle description/CWEs/canonical metric fields during Trivy DB packaging; `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` re-run 2025-10-15 to confirm coverage.|
|
||||
@@ -0,0 +1,11 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record TrivyConfigDocument(
|
||||
[property: JsonPropertyName("mediaType")] string MediaType,
|
||||
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
|
||||
[property: JsonPropertyName("databaseVersion")] string DatabaseVersion,
|
||||
[property: JsonPropertyName("databaseDigest")] string DatabaseDigest,
|
||||
[property: JsonPropertyName("databaseSize")] long DatabaseSize);
|
||||
78
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBlob.cs
Normal file
78
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBlob.cs
Normal file
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbBlob
|
||||
{
|
||||
private readonly Func<CancellationToken, ValueTask<Stream>> _openReadAsync;
|
||||
|
||||
private TrivyDbBlob(Func<CancellationToken, ValueTask<Stream>> openReadAsync, long length)
|
||||
{
|
||||
_openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync));
|
||||
if (length < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(length));
|
||||
}
|
||||
|
||||
Length = length;
|
||||
}
|
||||
|
||||
public long Length { get; }
|
||||
|
||||
public ValueTask<Stream> OpenReadAsync(CancellationToken cancellationToken)
|
||||
=> _openReadAsync(cancellationToken);
|
||||
|
||||
public static TrivyDbBlob FromBytes(ReadOnlyMemory<byte> payload)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return new TrivyDbBlob(static _ => ValueTask.FromResult<Stream>(Stream.Null), 0);
|
||||
}
|
||||
|
||||
if (MemoryMarshal.TryGetArray(payload, out ArraySegment<byte> segment) && segment.Array is not null && segment.Offset == 0)
|
||||
{
|
||||
return FromArray(segment.Array);
|
||||
}
|
||||
|
||||
return FromArray(payload.ToArray());
|
||||
}
|
||||
|
||||
public static TrivyDbBlob FromFile(string path, long length)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
throw new ArgumentException("File path must be provided.", nameof(path));
|
||||
}
|
||||
|
||||
if (length < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(length));
|
||||
}
|
||||
|
||||
return new TrivyDbBlob(
|
||||
cancellationToken => ValueTask.FromResult<Stream>(new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan)),
|
||||
length);
|
||||
}
|
||||
|
||||
public static TrivyDbBlob FromArray(byte[] buffer)
|
||||
{
|
||||
if (buffer is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(buffer));
|
||||
}
|
||||
|
||||
return new TrivyDbBlob(
|
||||
_ => ValueTask.FromResult<Stream>(new MemoryStream(buffer, writable: false)),
|
||||
buffer.LongLength);
|
||||
}
|
||||
}
|
||||
376
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBoltBuilder.cs
Normal file
376
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBoltBuilder.cs
Normal file
@@ -0,0 +1,376 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Formats.Tar;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder
|
||||
{
|
||||
private readonly TrivyDbExportOptions _options;
|
||||
private readonly ILogger<TrivyDbBoltBuilder> _logger;
|
||||
|
||||
public TrivyDbBoltBuilder(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbBoltBuilder> logger)
|
||||
{
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<TrivyDbBuilderResult> BuildAsync(
|
||||
JsonExportResult jsonTree,
|
||||
DateTimeOffset exportedAt,
|
||||
string exportId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(jsonTree);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exportId);
|
||||
|
||||
var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId);
|
||||
var outputDir = Path.Combine(builderRoot, "out");
|
||||
Directory.CreateDirectory(outputDir);
|
||||
|
||||
try
|
||||
{
|
||||
await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw;
|
||||
}
|
||||
|
||||
var metadataPath = Path.Combine(outputDir, "metadata.json");
|
||||
var dbPath = Path.Combine(outputDir, "trivy.db");
|
||||
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'.");
|
||||
}
|
||||
|
||||
if (!File.Exists(dbPath))
|
||||
{
|
||||
TryDeleteDirectory(builderRoot);
|
||||
throw new InvalidOperationException($"trivy.db not found at '{dbPath}'.");
|
||||
}
|
||||
|
||||
var archivePath = Path.Combine(builderRoot, "db.tar.gz");
|
||||
await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
||||
var length = new FileInfo(archivePath).Length;
|
||||
var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new TrivyDbBuilderResult(
|
||||
archivePath,
|
||||
digest,
|
||||
length,
|
||||
builderMetadata,
|
||||
builderRoot);
|
||||
}
|
||||
|
||||
private string PrepareBuilderRoot(string exportDirectory, string exportId)
|
||||
{
|
||||
var root = Path.Combine(exportDirectory, $".builder-{exportId}");
|
||||
if (Directory.Exists(root))
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore cleanup failures
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken)
|
||||
{
|
||||
var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions();
|
||||
var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath)
|
||||
? "trivy-db"
|
||||
: builderOptions.ExecutablePath;
|
||||
|
||||
var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List<string>();
|
||||
var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = executable,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
|
||||
startInfo.ArgumentList.Add("build");
|
||||
startInfo.ArgumentList.Add("--cache-dir");
|
||||
startInfo.ArgumentList.Add(cacheDir);
|
||||
startInfo.ArgumentList.Add("--output-dir");
|
||||
startInfo.ArgumentList.Add(outputDir);
|
||||
|
||||
if (builderOptions.UpdateInterval != default)
|
||||
{
|
||||
startInfo.ArgumentList.Add("--update-interval");
|
||||
startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval));
|
||||
}
|
||||
|
||||
if (targets.Count > 0)
|
||||
{
|
||||
foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t)))
|
||||
{
|
||||
startInfo.ArgumentList.Add("--only-update");
|
||||
startInfo.ArgumentList.Add(target);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory))
|
||||
{
|
||||
startInfo.WorkingDirectory = builderOptions.WorkingDirectory;
|
||||
}
|
||||
|
||||
if (!builderOptions.InheritEnvironment)
|
||||
{
|
||||
startInfo.Environment.Clear();
|
||||
}
|
||||
|
||||
foreach (var kvp in environment)
|
||||
{
|
||||
startInfo.Environment[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false };
|
||||
|
||||
var stdOut = new StringBuilder();
|
||||
var stdErr = new StringBuilder();
|
||||
|
||||
var stdoutCompletion = new TaskCompletionSource<object?>();
|
||||
var stderrCompletion = new TaskCompletionSource<object?>();
|
||||
|
||||
process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stdoutCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stdOut.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stderrCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stdErr.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
_logger.LogInformation("Running {Executable} to build Trivy DB", executable);
|
||||
|
||||
try
|
||||
{
|
||||
if (!process.Start())
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
|
||||
}
|
||||
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
using var registration = cancellationToken.Register(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore kill failures.
|
||||
}
|
||||
});
|
||||
|
||||
#if NET8_0_OR_GREATER
|
||||
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
|
||||
#else
|
||||
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
|
||||
#endif
|
||||
|
||||
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
_logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString());
|
||||
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
|
||||
}
|
||||
|
||||
if (stdOut.Length > 0)
|
||||
{
|
||||
_logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString());
|
||||
}
|
||||
|
||||
if (stdErr.Length > 0)
|
||||
{
|
||||
_logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task CreateArchiveAsync(
|
||||
string archivePath,
|
||||
DateTimeOffset exportedAt,
|
||||
string metadataPath,
|
||||
string dbPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var archiveStream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true);
|
||||
await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false);
|
||||
|
||||
var timestamp = exportedAt.UtcDateTime;
|
||||
foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead,
|
||||
};
|
||||
|
||||
await using var source = new FileStream(
|
||||
file.Path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
entry.DataStream = source;
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
|
||||
await writer.DisposeAsync().ConfigureAwait(false);
|
||||
await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath)
|
||||
{
|
||||
yield return ("metadata.json", metadataPath);
|
||||
yield return ("trivy.db", dbPath);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeDigestAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 8,
|
||||
options: FileOptions.Asynchronous);
|
||||
|
||||
if (stream.Length < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
stream.Position = 4;
|
||||
var zeros = new byte[4];
|
||||
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string ToGoDuration(TimeSpan span)
|
||||
{
|
||||
if (span <= TimeSpan.Zero)
|
||||
{
|
||||
return "0s";
|
||||
}
|
||||
|
||||
span = span.Duration();
|
||||
var builder = new StringBuilder();
|
||||
|
||||
var totalHours = (int)span.TotalHours;
|
||||
if (totalHours > 0)
|
||||
{
|
||||
builder.Append(totalHours);
|
||||
builder.Append('h');
|
||||
}
|
||||
|
||||
var minutes = span.Minutes;
|
||||
if (minutes > 0)
|
||||
{
|
||||
builder.Append(minutes);
|
||||
builder.Append('m');
|
||||
}
|
||||
|
||||
var seconds = span.Seconds + span.Milliseconds / 1000.0;
|
||||
if (seconds > 0 || builder.Length == 0)
|
||||
{
|
||||
if (span.Milliseconds == 0)
|
||||
{
|
||||
builder.Append(span.Seconds);
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture));
|
||||
}
|
||||
builder.Append('s');
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record TrivyDbBuilderResult(
|
||||
string ArchivePath,
|
||||
string ArchiveDigest,
|
||||
long ArchiveLength,
|
||||
ReadOnlyMemory<byte> BuilderMetadata,
|
||||
string WorkingDirectory);
|
||||
94
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs
Normal file
94
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs
Normal file
@@ -0,0 +1,94 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbExportJob : IJob
|
||||
{
|
||||
public const string JobKind = "export:trivy-db";
|
||||
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20);
|
||||
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10);
|
||||
|
||||
private readonly TrivyDbFeedExporter _exporter;
|
||||
private readonly ILogger<TrivyDbExportJob> _logger;
|
||||
|
||||
public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger<TrivyDbExportJob> logger)
|
||||
{
|
||||
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId);
|
||||
var overrides = CreateOverrides(context.Parameters);
|
||||
if (overrides?.HasOverrides == true)
|
||||
{
|
||||
using var scope = TrivyDbExportOverrideScope.Begin(overrides);
|
||||
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId);
|
||||
}
|
||||
|
||||
private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary<string, object?> parameters)
|
||||
{
|
||||
if (parameters is null || parameters.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var publishFull = TryReadBoolean(parameters, "publishFull");
|
||||
var publishDelta = TryReadBoolean(parameters, "publishDelta");
|
||||
var includeFull = TryReadBoolean(parameters, "includeFull");
|
||||
var includeDelta = TryReadBoolean(parameters, "includeDelta");
|
||||
|
||||
var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta);
|
||||
return overrides.HasOverrides ? overrides : null;
|
||||
}
|
||||
|
||||
private static bool? TryReadBoolean(IReadOnlyDictionary<string, object?> parameters, string key)
|
||||
{
|
||||
if (!parameters.TryGetValue(key, out var value) || value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
switch (value)
|
||||
{
|
||||
case bool b:
|
||||
return b;
|
||||
case string s when bool.TryParse(s, out var result):
|
||||
return result;
|
||||
case JsonElement element:
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed,
|
||||
_ => null,
|
||||
};
|
||||
case IConvertible convertible:
|
||||
try
|
||||
{
|
||||
return convertible.ToBoolean(CultureInfo.InvariantCulture);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public enum TrivyDbExportMode
|
||||
{
|
||||
Full,
|
||||
Delta,
|
||||
Skip,
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbExportOptions
|
||||
{
|
||||
public string OutputRoot { get; set; } = Path.Combine("exports", "trivy");
|
||||
|
||||
public string ReferencePrefix { get; set; } = "concelier/trivy";
|
||||
|
||||
public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
|
||||
|
||||
public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
|
||||
|
||||
public bool KeepWorkingTree { get; set; }
|
||||
|
||||
public string? TargetRepository { get; set; }
|
||||
|
||||
public JsonExportOptions Json { get; set; } = new()
|
||||
{
|
||||
OutputRoot = Path.Combine("exports", "trivy", "tree")
|
||||
};
|
||||
|
||||
public TrivyDbBuilderOptions Builder { get; set; } = new();
|
||||
|
||||
public TrivyDbOrasOptions Oras { get; set; } = new();
|
||||
|
||||
public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new();
|
||||
|
||||
public string GetExportRoot(string exportId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(exportId);
|
||||
var root = Path.GetFullPath(OutputRoot);
|
||||
return Path.Combine(root, exportId);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class TrivyDbBuilderOptions
|
||||
{
|
||||
public string ExecutablePath { get; set; } = "trivy-db";
|
||||
|
||||
public string? WorkingDirectory { get; set; }
|
||||
|
||||
public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
public List<string> OnlyUpdateTargets { get; set; } = new();
|
||||
|
||||
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public bool InheritEnvironment { get; set; } = true;
|
||||
}
|
||||
|
||||
public sealed class TrivyDbOrasOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public string ExecutablePath { get; set; } = "oras";
|
||||
|
||||
public bool PublishFull { get; set; } = true;
|
||||
|
||||
public bool PublishDelta { get; set; } = true;
|
||||
|
||||
public string? WorkingDirectory { get; set; }
|
||||
|
||||
public bool InheritEnvironment { get; set; } = true;
|
||||
|
||||
public List<string> AdditionalArguments { get; set; } = new();
|
||||
|
||||
public Dictionary<string, string> Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public bool SkipTlsVerify { get; set; }
|
||||
|
||||
public bool UseHttp { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TrivyDbOfflineBundleOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public bool IncludeFull { get; set; } = true;
|
||||
|
||||
public bool IncludeDelta { get; set; } = true;
|
||||
|
||||
public string? FileName { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
internal sealed record TrivyDbExportOverrides(
|
||||
bool? PublishFull,
|
||||
bool? PublishDelta,
|
||||
bool? IncludeFull,
|
||||
bool? IncludeDelta)
|
||||
{
|
||||
public bool HasOverrides =>
|
||||
PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue;
|
||||
}
|
||||
|
||||
internal static class TrivyDbExportOverrideScope
|
||||
{
|
||||
private sealed class Scope : IDisposable
|
||||
{
|
||||
private readonly TrivyDbExportOverrides? _previous;
|
||||
private bool _disposed;
|
||||
|
||||
public Scope(TrivyDbExportOverrides? previous)
|
||||
{
|
||||
_previous = previous;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
CurrentOverrides.Value = _previous;
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly AsyncLocal<TrivyDbExportOverrides?> CurrentOverrides = new();
|
||||
|
||||
public static TrivyDbExportOverrides? Current => CurrentOverrides.Value;
|
||||
|
||||
public static IDisposable Begin(TrivyDbExportOverrides overrides)
|
||||
{
|
||||
var previous = CurrentOverrides.Value;
|
||||
CurrentOverrides.Value = overrides;
|
||||
return new Scope(previous);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
public sealed record TrivyDbExportPlan(
|
||||
TrivyDbExportMode Mode,
|
||||
string TreeDigest,
|
||||
string? BaseExportId,
|
||||
string? BaseManifestDigest,
|
||||
bool ResetBaseline,
|
||||
IReadOnlyList<ExportFileRecord> Manifest,
|
||||
IReadOnlyList<ExportFileRecord> ChangedFiles,
|
||||
IReadOnlyList<string> RemovedPaths);
|
||||
115
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlanner.cs
Normal file
115
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlanner.cs
Normal file
@@ -0,0 +1,115 @@
|
||||
using System;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
public sealed class TrivyDbExportPlanner
|
||||
{
|
||||
public TrivyDbExportPlan CreatePlan(
|
||||
ExportStateRecord? existingState,
|
||||
string treeDigest,
|
||||
IReadOnlyList<ExportFileRecord> manifest)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(treeDigest);
|
||||
manifest ??= Array.Empty<ExportFileRecord>();
|
||||
|
||||
if (existingState is null || (existingState.Files?.Count ?? 0) == 0)
|
||||
{
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Full,
|
||||
treeDigest,
|
||||
BaseExportId: existingState?.BaseExportId,
|
||||
BaseManifestDigest: existingState?.LastFullDigest,
|
||||
ResetBaseline: true,
|
||||
Manifest: manifest,
|
||||
ChangedFiles: manifest,
|
||||
RemovedPaths: Array.Empty<string>());
|
||||
}
|
||||
|
||||
var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>();
|
||||
var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal);
|
||||
if (cursorMatches)
|
||||
{
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Skip,
|
||||
treeDigest,
|
||||
existingState.BaseExportId,
|
||||
existingState.LastFullDigest,
|
||||
ResetBaseline: false,
|
||||
Manifest: existingFiles,
|
||||
ChangedFiles: Array.Empty<ExportFileRecord>(),
|
||||
RemovedPaths: Array.Empty<string>());
|
||||
}
|
||||
|
||||
var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
|
||||
var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var removed = existingMap.Keys
|
||||
.Where(path => !newMap.ContainsKey(path))
|
||||
.ToArray();
|
||||
|
||||
if (removed.Length > 0)
|
||||
{
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Full,
|
||||
treeDigest,
|
||||
existingState.BaseExportId,
|
||||
existingState.LastFullDigest,
|
||||
ResetBaseline: true,
|
||||
Manifest: manifest,
|
||||
ChangedFiles: manifest,
|
||||
RemovedPaths: removed);
|
||||
}
|
||||
|
||||
var changed = new List<ExportFileRecord>();
|
||||
foreach (var file in manifest)
|
||||
{
|
||||
if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal))
|
||||
{
|
||||
changed.Add(file);
|
||||
}
|
||||
}
|
||||
|
||||
if (changed.Count == 0)
|
||||
{
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Skip,
|
||||
treeDigest,
|
||||
existingState.BaseExportId,
|
||||
existingState.LastFullDigest,
|
||||
ResetBaseline: false,
|
||||
Manifest: existingFiles,
|
||||
ChangedFiles: Array.Empty<ExportFileRecord>(),
|
||||
RemovedPaths: Array.Empty<string>());
|
||||
}
|
||||
|
||||
var hasOutstandingDelta = existingState.LastDeltaDigest is not null;
|
||||
if (hasOutstandingDelta)
|
||||
{
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Full,
|
||||
treeDigest,
|
||||
existingState.BaseExportId,
|
||||
existingState.LastFullDigest,
|
||||
ResetBaseline: true,
|
||||
Manifest: manifest,
|
||||
ChangedFiles: manifest,
|
||||
RemovedPaths: Array.Empty<string>());
|
||||
}
|
||||
|
||||
return new TrivyDbExportPlan(
|
||||
TrivyDbExportMode.Delta,
|
||||
treeDigest,
|
||||
existingState.BaseExportId,
|
||||
existingState.LastFullDigest,
|
||||
ResetBaseline: false,
|
||||
Manifest: manifest,
|
||||
ChangedFiles: changed,
|
||||
RemovedPaths: Array.Empty<string>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:exporters:trivyDb";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
|
||||
services.TryAddSingleton<ExportStateManager>();
|
||||
|
||||
services.AddOptions<TrivyDbExportOptions>()
|
||||
.Bind(configuration.GetSection(ConfigurationSection))
|
||||
.PostConfigure(static options =>
|
||||
{
|
||||
options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy"));
|
||||
options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree"));
|
||||
options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat;
|
||||
options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat;
|
||||
options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix;
|
||||
});
|
||||
|
||||
services.AddSingleton<TrivyDbPackageBuilder>();
|
||||
services.AddSingleton<TrivyDbOciWriter>();
|
||||
services.AddSingleton<TrivyDbExportPlanner>();
|
||||
services.AddSingleton<ITrivyDbBuilder, TrivyDbBoltBuilder>();
|
||||
services.AddSingleton<ITrivyDbOrasPusher, TrivyDbOrasPusher>();
|
||||
services.AddSingleton<TrivyDbFeedExporter>();
|
||||
services.AddTransient<TrivyDbExportJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind))
|
||||
{
|
||||
options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition(
|
||||
TrivyDbExportJob.JobKind,
|
||||
typeof(TrivyDbExportJob),
|
||||
TrivyDbExportJob.DefaultTimeout,
|
||||
TrivyDbExportJob.DefaultLeaseDuration,
|
||||
null,
|
||||
true);
|
||||
}
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static string Normalize(string? value, string fallback)
|
||||
=> string.IsNullOrWhiteSpace(value) ? fallback : value;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbExporterPlugin : IExporterPlugin
|
||||
{
|
||||
public string Name => TrivyDbFeedExporter.ExporterName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetService<IAdvisoryStore>() is not null;
|
||||
}
|
||||
|
||||
public IFeedExporter Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<TrivyDbFeedExporter>(services);
|
||||
}
|
||||
}
|
||||
490
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs
Normal file
490
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs
Normal file
@@ -0,0 +1,490 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Formats.Tar;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Exporter.Json;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbFeedExporter : IFeedExporter
|
||||
{
|
||||
public const string ExporterName = "trivy-db";
|
||||
public const string ExporterId = "export:trivy-db";
|
||||
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
private readonly TrivyDbExportOptions _options;
|
||||
private readonly TrivyDbPackageBuilder _packageBuilder;
|
||||
private readonly TrivyDbOciWriter _ociWriter;
|
||||
private readonly ExportStateManager _stateManager;
|
||||
private readonly TrivyDbExportPlanner _exportPlanner;
|
||||
private readonly ITrivyDbBuilder _builder;
|
||||
private readonly ITrivyDbOrasPusher _orasPusher;
|
||||
private readonly ILogger<TrivyDbFeedExporter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _exporterVersion;
|
||||
|
||||
public TrivyDbFeedExporter(
|
||||
IAdvisoryStore advisoryStore,
|
||||
IJsonExportPathResolver pathResolver,
|
||||
IOptions<TrivyDbExportOptions> options,
|
||||
TrivyDbPackageBuilder packageBuilder,
|
||||
TrivyDbOciWriter ociWriter,
|
||||
ExportStateManager stateManager,
|
||||
TrivyDbExportPlanner exportPlanner,
|
||||
ITrivyDbBuilder builder,
|
||||
ITrivyDbOrasPusher orasPusher,
|
||||
ILogger<TrivyDbFeedExporter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder));
|
||||
_ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter));
|
||||
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
|
||||
_exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner));
|
||||
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
|
||||
_orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter));
|
||||
}
|
||||
|
||||
public string Name => ExporterName;
|
||||
|
||||
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var exportedAt = _timeProvider.GetUtcNow();
|
||||
var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture);
|
||||
var reference = $"{_options.ReferencePrefix}:{exportId}";
|
||||
|
||||
_logger.LogInformation("Starting Trivy DB export {ExportId}", exportId);
|
||||
|
||||
var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver);
|
||||
var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
|
||||
var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)",
|
||||
exportId,
|
||||
jsonResult.AdvisoryCount,
|
||||
jsonResult.TotalBytes);
|
||||
|
||||
var manifest = jsonResult.Files
|
||||
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult);
|
||||
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
|
||||
var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest);
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Skip)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.",
|
||||
exportId,
|
||||
plan.BaseExportId ?? "(none)");
|
||||
|
||||
if (!_options.KeepWorkingTree)
|
||||
{
|
||||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Delta)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.",
|
||||
exportId,
|
||||
plan.ChangedFiles.Count);
|
||||
}
|
||||
|
||||
var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt);
|
||||
|
||||
try
|
||||
{
|
||||
var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest(
|
||||
metadataBytes,
|
||||
builderResult.ArchivePath,
|
||||
builderResult.ArchiveDigest,
|
||||
builderResult.ArchiveLength,
|
||||
exportedAt,
|
||||
exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture)));
|
||||
|
||||
var destination = _options.GetExportRoot(exportId);
|
||||
string? baseLayout = null;
|
||||
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(plan.BaseExportId))
|
||||
{
|
||||
baseLayout = _options.GetExportRoot(plan.BaseExportId);
|
||||
}
|
||||
|
||||
var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode))
|
||||
{
|
||||
await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Trivy DB export {ExportId} wrote manifest {ManifestDigest}",
|
||||
exportId,
|
||||
ociResult.ManifestDigest);
|
||||
|
||||
var resetBaseline = plan.ResetBaseline
|
||||
|| existingState is null
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
|
||||
|
||||
if (existingState is not null
|
||||
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
|
||||
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
|
||||
{
|
||||
resetBaseline = true;
|
||||
}
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Full || resetBaseline)
|
||||
{
|
||||
await _stateManager.StoreFullExportAsync(
|
||||
ExporterId,
|
||||
exportId,
|
||||
ociResult.ManifestDigest,
|
||||
cursor: treeDigest,
|
||||
targetRepository: _options.TargetRepository,
|
||||
exporterVersion: _exporterVersion,
|
||||
resetBaseline: resetBaseline,
|
||||
manifest: plan.Manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _stateManager.StoreDeltaExportAsync(
|
||||
ExporterId,
|
||||
deltaDigest: treeDigest,
|
||||
cursor: treeDigest,
|
||||
exporterVersion: _exporterVersion,
|
||||
manifest: plan.Manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await CreateOfflineBundleAsync(destination, exportId, exportedAt, plan.Mode, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDeleteDirectory(builderResult.WorkingDirectory);
|
||||
}
|
||||
|
||||
if (!_options.KeepWorkingTree)
|
||||
{
|
||||
TryDeleteDirectory(jsonResult.ExportDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<Advisory>> LoadAdvisoriesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var advisories = new List<Advisory>();
|
||||
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey));
|
||||
return advisories;
|
||||
}
|
||||
|
||||
private byte[] CreateMetadataJson(
|
||||
TrivyDbExportPlan plan,
|
||||
ReadOnlyMemory<byte> builderMetadata,
|
||||
string treeDigest,
|
||||
JsonExportResult result,
|
||||
DateTimeOffset exportedAt)
|
||||
{
|
||||
var metadata = new TrivyMetadata
|
||||
{
|
||||
GeneratedAt = exportedAt.UtcDateTime,
|
||||
AdvisoryCount = result.AdvisoryCount,
|
||||
TreeDigest = treeDigest,
|
||||
TreeBytes = result.TotalBytes,
|
||||
ExporterVersion = _exporterVersion,
|
||||
Builder = ParseBuilderMetadata(builderMetadata.Span),
|
||||
Mode = plan.Mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => "full",
|
||||
TrivyDbExportMode.Delta => "delta",
|
||||
TrivyDbExportMode.Skip => "skip",
|
||||
_ => "unknown",
|
||||
},
|
||||
BaseExportId = plan.BaseExportId,
|
||||
BaseManifestDigest = plan.BaseManifestDigest,
|
||||
ResetBaseline = plan.ResetBaseline,
|
||||
Delta = plan.Mode == TrivyDbExportMode.Delta
|
||||
? new DeltaMetadata(
|
||||
plan.ChangedFiles.Select(static file => new DeltaFileMetadata(file.Path, file.Length, file.Digest)).ToArray(),
|
||||
plan.RemovedPaths.ToArray())
|
||||
: null,
|
||||
};
|
||||
|
||||
return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
});
|
||||
}
|
||||
|
||||
private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<BuilderMetadata>(payload, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
});
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, TrivyDbExportMode mode, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_options.OfflineBundle.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ShouldIncludeInBundle(mode))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath;
|
||||
var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName)
|
||||
? $"{exportId}.offline.tar.gz"
|
||||
: _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal);
|
||||
|
||||
var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!);
|
||||
|
||||
if (File.Exists(bundlePath))
|
||||
{
|
||||
File.Delete(bundlePath);
|
||||
}
|
||||
|
||||
var normalizedRoot = Path.GetFullPath(layoutPath);
|
||||
var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories)
|
||||
.Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/")
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories)
|
||||
.Select(file => NormalizeTarPath(normalizedRoot, file))
|
||||
.OrderBy(static path => path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
await using (var archiveStream = new FileStream(
|
||||
bundlePath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true))
|
||||
await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false))
|
||||
{
|
||||
var timestamp = exportedAt.UtcDateTime;
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
var entry = new PaxTarEntry(TarEntryType.Directory, directory)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
|
||||
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
|
||||
UnixFileMode.OtherRead | UnixFileMode.OtherExecute,
|
||||
};
|
||||
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
|
||||
foreach (var relativePath in files)
|
||||
{
|
||||
var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar));
|
||||
var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath)
|
||||
{
|
||||
ModificationTime = timestamp,
|
||||
Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite |
|
||||
UnixFileMode.GroupRead |
|
||||
UnixFileMode.OtherRead,
|
||||
};
|
||||
|
||||
await using var source = new FileStream(
|
||||
fullPath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
entry.DataStream = source;
|
||||
writer.WriteEntry(entry);
|
||||
}
|
||||
}
|
||||
|
||||
await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false);
|
||||
var length = new FileInfo(bundlePath).Length;
|
||||
_logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest);
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort cleanup – ignore failures.
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
archivePath,
|
||||
FileMode.Open,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 8,
|
||||
options: FileOptions.Asynchronous);
|
||||
|
||||
if (stream.Length < 10)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
stream.Position = 4;
|
||||
var zeros = new byte[4];
|
||||
await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeSha256Async(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static string NormalizeTarPath(string root, string fullPath)
|
||||
{
|
||||
var relative = Path.GetRelativePath(root, fullPath);
|
||||
var normalized = relative.Replace(Path.DirectorySeparatorChar, '/');
|
||||
return string.IsNullOrEmpty(normalized) ? "." : normalized;
|
||||
}
|
||||
|
||||
private bool ShouldPublishToOras(TrivyDbExportMode mode)
|
||||
{
|
||||
var overrides = TrivyDbExportOverrideScope.Current;
|
||||
|
||||
return mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => overrides?.PublishFull ?? _options.Oras.PublishFull,
|
||||
TrivyDbExportMode.Delta => overrides?.PublishDelta ?? _options.Oras.PublishDelta,
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
|
||||
private bool ShouldIncludeInBundle(TrivyDbExportMode mode)
|
||||
{
|
||||
var overrides = TrivyDbExportOverrideScope.Current;
|
||||
|
||||
return mode switch
|
||||
{
|
||||
TrivyDbExportMode.Full => overrides?.IncludeFull ?? _options.OfflineBundle.IncludeFull,
|
||||
TrivyDbExportMode.Delta => overrides?.IncludeDelta ?? _options.OfflineBundle.IncludeDelta,
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
|
||||
private sealed record DeltaMetadata(
|
||||
IReadOnlyList<DeltaFileMetadata> ChangedFiles,
|
||||
IReadOnlyList<string> RemovedPaths);
|
||||
|
||||
private sealed record DeltaFileMetadata(string Path, long Length, string Digest);
|
||||
|
||||
private sealed class TrivyMetadata
|
||||
{
|
||||
public DateTime GeneratedAt { get; set; }
|
||||
|
||||
public int AdvisoryCount { get; set; }
|
||||
|
||||
public string TreeDigest { get; set; } = string.Empty;
|
||||
|
||||
public long TreeBytes { get; set; }
|
||||
|
||||
public string ExporterVersion { get; set; } = string.Empty;
|
||||
|
||||
public BuilderMetadata? Builder { get; set; }
|
||||
|
||||
public string Mode { get; set; } = "full";
|
||||
|
||||
public string? BaseExportId { get; set; }
|
||||
|
||||
public string? BaseManifestDigest { get; set; }
|
||||
|
||||
public bool ResetBaseline { get; set; }
|
||||
|
||||
public DeltaMetadata? Delta { get; set; }
|
||||
}
|
||||
|
||||
private sealed class BuilderMetadata
|
||||
{
|
||||
[JsonPropertyName("Version")]
|
||||
public int Version { get; set; }
|
||||
|
||||
public DateTime NextUpdate { get; set; }
|
||||
|
||||
public DateTime UpdatedAt { get; set; }
|
||||
|
||||
public DateTime? DownloadedAt { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public static class TrivyDbMediaTypes
|
||||
{
|
||||
public const string OciManifest = "application/vnd.oci.image.manifest.v1+json";
|
||||
public const string OciImageIndex = "application/vnd.oci.image.index.v1+json";
|
||||
public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json";
|
||||
public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip";
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record TrivyDbOciWriteResult(
|
||||
string RootDirectory,
|
||||
string ManifestDigest,
|
||||
IReadOnlyCollection<string> BlobDigests);
|
||||
375
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriter.cs
Normal file
375
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriter.cs
Normal file
@@ -0,0 +1,375 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
/// <summary>
|
||||
/// Writes a Trivy DB package to an OCI image layout directory with deterministic content.
|
||||
/// </summary>
|
||||
public sealed class TrivyDbOciWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}");
|
||||
|
||||
public async Task<TrivyDbOciWriteResult> WriteAsync(
|
||||
TrivyDbPackage package,
|
||||
string destination,
|
||||
string reference,
|
||||
TrivyDbExportPlan plan,
|
||||
string? baseLayoutPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (package is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(package));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(destination))
|
||||
{
|
||||
throw new ArgumentException("Destination directory must be provided.", nameof(destination));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(reference))
|
||||
{
|
||||
throw new ArgumentException("Reference tag must be provided.", nameof(reference));
|
||||
}
|
||||
|
||||
if (plan is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(plan));
|
||||
}
|
||||
|
||||
var root = Path.GetFullPath(destination);
|
||||
if (Directory.Exists(root))
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(root);
|
||||
var timestamp = package.Config.GeneratedAt.UtcDateTime;
|
||||
|
||||
await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var blobsRoot = Path.Combine(root, "blobs", "sha256");
|
||||
Directory.CreateDirectory(blobsRoot);
|
||||
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp);
|
||||
Directory.SetLastWriteTimeUtc(blobsRoot, timestamp);
|
||||
|
||||
var writtenDigests = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var pair in package.Blobs)
|
||||
{
|
||||
if (writtenDigests.Contains(pair.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
if (reused)
|
||||
{
|
||||
writtenDigests.Add(pair.Key);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (writtenDigests.Add(pair.Key))
|
||||
{
|
||||
await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions);
|
||||
var manifestDigest = ComputeDigest(manifestBytes);
|
||||
if (!writtenDigests.Contains(manifestDigest))
|
||||
{
|
||||
var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
if (!reused)
|
||||
{
|
||||
await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
writtenDigests.Add(manifestDigest);
|
||||
}
|
||||
|
||||
var manifestDescriptor = new OciDescriptor(
|
||||
TrivyDbMediaTypes.OciManifest,
|
||||
manifestDigest,
|
||||
manifestBytes.LongLength,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["org.opencontainers.image.ref.name"] = reference,
|
||||
});
|
||||
var index = new OciIndex(2, new[] { manifestDescriptor });
|
||||
var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions);
|
||||
await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath))
|
||||
{
|
||||
var reuseDigests = await TryReuseBaseBlobsAsync(
|
||||
blobsRoot,
|
||||
timestamp,
|
||||
writtenDigests,
|
||||
plan,
|
||||
baseLayoutPath,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
foreach (var digest in reuseDigests)
|
||||
{
|
||||
writtenDigests.Add(digest);
|
||||
}
|
||||
}
|
||||
|
||||
Directory.SetLastWriteTimeUtc(root, timestamp);
|
||||
|
||||
var blobDigests = writtenDigests.ToArray();
|
||||
Array.Sort(blobDigests, StringComparer.Ordinal);
|
||||
return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests);
|
||||
}
|
||||
|
||||
private static async Task WriteFileAsync(string path, ReadOnlyMemory<byte> bytes, DateTime utcTimestamp, CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(path);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
|
||||
}
|
||||
|
||||
await using var destination = new FileStream(
|
||||
path,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(path, utcTimestamp);
|
||||
}
|
||||
|
||||
private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken)
|
||||
{
|
||||
var fileName = ResolveDigestFileName(digest);
|
||||
var path = Path.Combine(blobsRoot, fileName);
|
||||
var directory = Path.GetDirectoryName(path);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
Directory.SetLastWriteTimeUtc(directory, utcTimestamp);
|
||||
}
|
||||
|
||||
await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var destination = new FileStream(
|
||||
path,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
|
||||
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(path, utcTimestamp);
|
||||
}
|
||||
|
||||
private static string ResolveDigestFileName(string digest)
|
||||
{
|
||||
if (!digest.StartsWith("sha256:", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'.");
|
||||
}
|
||||
|
||||
var hex = digest[7..];
|
||||
if (hex.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Digest hex component cannot be empty.");
|
||||
}
|
||||
|
||||
return hex;
|
||||
}
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = System.Security.Cryptography.SHA256.HashData(payload);
|
||||
var hex = Convert.ToHexString(hash);
|
||||
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
|
||||
buffer[0] = 's';
|
||||
buffer[1] = 'h';
|
||||
buffer[2] = 'a';
|
||||
buffer[3] = '2';
|
||||
buffer[4] = '5';
|
||||
buffer[5] = '6';
|
||||
buffer[6] = ':';
|
||||
for (var i = 0; i < hex.Length; i++)
|
||||
{
|
||||
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
|
||||
}
|
||||
|
||||
return new string(buffer);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyCollection<string>> TryReuseBaseBlobsAsync(
|
||||
string destinationBlobsRoot,
|
||||
DateTime timestamp,
|
||||
HashSet<string> written,
|
||||
TrivyDbExportPlan plan,
|
||||
string baseLayoutPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var baseRoot = Path.GetFullPath(baseLayoutPath);
|
||||
if (!Directory.Exists(baseRoot))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest);
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
await using var stream = new FileStream(
|
||||
manifestPath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var root = document.RootElement;
|
||||
|
||||
var digests = new SortedSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
plan.BaseManifestDigest,
|
||||
};
|
||||
|
||||
if (root.TryGetProperty("config", out var configNode))
|
||||
{
|
||||
var digest = configNode.GetProperty("digest").GetString();
|
||||
if (!string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
digests.Add(digest);
|
||||
}
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("layers", out var layersNode))
|
||||
{
|
||||
foreach (var layer in layersNode.EnumerateArray())
|
||||
{
|
||||
var digest = layer.GetProperty("digest").GetString();
|
||||
if (!string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
digests.Add(digest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var copied = new List<string>();
|
||||
foreach (var digest in digests)
|
||||
{
|
||||
if (written.Contains(digest))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourcePath = ResolveBlobPath(baseRoot, digest);
|
||||
if (!File.Exists(sourcePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
await using var source = new FileStream(
|
||||
sourcePath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var destination = new FileStream(
|
||||
destinationPath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(destinationPath, timestamp);
|
||||
copied.Add(digest);
|
||||
}
|
||||
|
||||
if (copied.Count > 0)
|
||||
{
|
||||
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
|
||||
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
|
||||
}
|
||||
|
||||
return copied;
|
||||
}
|
||||
|
||||
private static string ResolveBlobPath(string layoutRoot, string digest)
|
||||
{
|
||||
var fileName = ResolveDigestFileName(digest);
|
||||
return Path.Combine(layoutRoot, "blobs", "sha256", fileName);
|
||||
}
|
||||
|
||||
private static async Task<bool> TryReuseExistingBlobAsync(
|
||||
string? baseLayoutPath,
|
||||
string digest,
|
||||
string destinationBlobsRoot,
|
||||
DateTime timestamp,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(baseLayoutPath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var baseRoot = Path.GetFullPath(baseLayoutPath);
|
||||
var sourcePath = ResolveBlobPath(baseRoot, digest);
|
||||
if (!File.Exists(sourcePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
await using var source = new FileStream(
|
||||
sourcePath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var destination = new FileStream(
|
||||
destinationPath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 81920,
|
||||
options: FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(destinationPath, timestamp);
|
||||
Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp);
|
||||
Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
209
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOrasPusher.cs
Normal file
209
src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOrasPusher.cs
Normal file
@@ -0,0 +1,209 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher
|
||||
{
|
||||
private readonly TrivyDbExportOptions _options;
|
||||
private readonly ILogger<TrivyDbOrasPusher> _logger;
|
||||
|
||||
public TrivyDbOrasPusher(IOptions<TrivyDbExportOptions> options, ILogger<TrivyDbOrasPusher> logger)
|
||||
{
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken)
|
||||
{
|
||||
var orasOptions = _options.Oras;
|
||||
if (!orasOptions.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(reference))
|
||||
{
|
||||
throw new InvalidOperationException("ORAS push requested but reference is empty.");
|
||||
}
|
||||
|
||||
if (!Directory.Exists(layoutPath))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist.");
|
||||
}
|
||||
|
||||
var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath;
|
||||
var tag = ResolveTag(reference, exportId);
|
||||
var layoutReference = $"{layoutPath}:{tag}";
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = executable,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
|
||||
startInfo.ArgumentList.Add("cp");
|
||||
startInfo.ArgumentList.Add("--from-oci-layout");
|
||||
startInfo.ArgumentList.Add(layoutReference);
|
||||
if (orasOptions.SkipTlsVerify)
|
||||
{
|
||||
startInfo.ArgumentList.Add("--insecure");
|
||||
}
|
||||
if (orasOptions.UseHttp)
|
||||
{
|
||||
startInfo.ArgumentList.Add("--plain-http");
|
||||
}
|
||||
|
||||
if (orasOptions.AdditionalArguments is { Count: > 0 })
|
||||
{
|
||||
foreach (var arg in orasOptions.AdditionalArguments)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(arg))
|
||||
{
|
||||
startInfo.ArgumentList.Add(arg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
startInfo.ArgumentList.Add(reference);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory))
|
||||
{
|
||||
startInfo.WorkingDirectory = orasOptions.WorkingDirectory;
|
||||
}
|
||||
|
||||
if (!orasOptions.InheritEnvironment)
|
||||
{
|
||||
startInfo.Environment.Clear();
|
||||
}
|
||||
|
||||
if (orasOptions.Environment is { Count: > 0 })
|
||||
{
|
||||
foreach (var kvp in orasOptions.Environment)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(kvp.Key))
|
||||
{
|
||||
startInfo.Environment[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
var stdout = new StringBuilder();
|
||||
var stderr = new StringBuilder();
|
||||
var stdoutCompletion = new TaskCompletionSource<object?>();
|
||||
var stderrCompletion = new TaskCompletionSource<object?>();
|
||||
|
||||
process.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stdoutCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stdout.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
process.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null)
|
||||
{
|
||||
stderrCompletion.TrySetResult(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
stderr.AppendLine(e.Data);
|
||||
}
|
||||
};
|
||||
|
||||
_logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable);
|
||||
|
||||
try
|
||||
{
|
||||
if (!process.Start())
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to start '{executable}'.", ex);
|
||||
}
|
||||
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
using var registration = cancellationToken.Register(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
|
||||
#if NET8_0_OR_GREATER
|
||||
await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false);
|
||||
#else
|
||||
await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false);
|
||||
#endif
|
||||
|
||||
await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
_logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString());
|
||||
throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}.");
|
||||
}
|
||||
|
||||
if (stdout.Length > 0)
|
||||
{
|
||||
_logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString());
|
||||
}
|
||||
|
||||
if (stderr.Length > 0)
|
||||
{
|
||||
_logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private static string ResolveTag(string reference, string fallback)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var atIndex = reference.IndexOf('@');
|
||||
if (atIndex >= 0)
|
||||
{
|
||||
reference = reference[..atIndex];
|
||||
}
|
||||
|
||||
var slashIndex = reference.LastIndexOf('/');
|
||||
var colonIndex = reference.LastIndexOf(':');
|
||||
if (colonIndex > slashIndex && colonIndex >= 0)
|
||||
{
|
||||
return reference[(colonIndex + 1)..];
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record TrivyDbPackage(
|
||||
OciManifest Manifest,
|
||||
TrivyConfigDocument Config,
|
||||
IReadOnlyDictionary<string, TrivyDbBlob> Blobs,
|
||||
ReadOnlyMemory<byte> MetadataJson);
|
||||
@@ -0,0 +1,116 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed class TrivyDbPackageBuilder
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request)
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(request));
|
||||
}
|
||||
|
||||
if (request.MetadataJson.IsEmpty)
|
||||
{
|
||||
throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath))
|
||||
{
|
||||
throw new ArgumentException("Database archive path must be provided.", nameof(request));
|
||||
}
|
||||
|
||||
if (!File.Exists(request.DatabaseArchivePath))
|
||||
{
|
||||
throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.DatabaseDigest))
|
||||
{
|
||||
throw new ArgumentException("Database archive digest must be provided.", nameof(request));
|
||||
}
|
||||
|
||||
if (request.DatabaseLength < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength));
|
||||
}
|
||||
|
||||
var metadataBytes = request.MetadataJson;
|
||||
var generatedAt = request.GeneratedAt.ToUniversalTime();
|
||||
var configDocument = new TrivyConfigDocument(
|
||||
TrivyDbMediaTypes.TrivyConfig,
|
||||
generatedAt,
|
||||
request.DatabaseVersion,
|
||||
request.DatabaseDigest,
|
||||
request.DatabaseLength);
|
||||
|
||||
var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions);
|
||||
var configDigest = ComputeDigest(configBytes);
|
||||
|
||||
var configDescriptor = new OciDescriptor(
|
||||
TrivyDbMediaTypes.TrivyConfig,
|
||||
configDigest,
|
||||
configBytes.LongLength,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["org.opencontainers.image.title"] = "config.json",
|
||||
});
|
||||
|
||||
var layerDescriptor = new OciDescriptor(
|
||||
TrivyDbMediaTypes.TrivyLayer,
|
||||
request.DatabaseDigest,
|
||||
request.DatabaseLength,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["org.opencontainers.image.title"] = "db.tar.gz",
|
||||
});
|
||||
|
||||
var manifest = new OciManifest(
|
||||
2,
|
||||
TrivyDbMediaTypes.OciManifest,
|
||||
configDescriptor,
|
||||
ImmutableArray.Create(layerDescriptor));
|
||||
|
||||
var blobs = new SortedDictionary<string, TrivyDbBlob>(StringComparer.Ordinal)
|
||||
{
|
||||
[configDigest] = TrivyDbBlob.FromBytes(configBytes),
|
||||
[request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength),
|
||||
};
|
||||
|
||||
return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes);
|
||||
}
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = SHA256.HashData(payload);
|
||||
var hex = Convert.ToHexString(hash);
|
||||
Span<char> buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex
|
||||
buffer[0] = 's';
|
||||
buffer[1] = 'h';
|
||||
buffer[2] = 'a';
|
||||
buffer[3] = '2';
|
||||
buffer[4] = '5';
|
||||
buffer[5] = '6';
|
||||
buffer[6] = ':';
|
||||
for (var i = 0; i < hex.Length; i++)
|
||||
{
|
||||
buffer[7 + i] = char.ToLowerInvariant(hex[i]);
|
||||
}
|
||||
|
||||
return new string(buffer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.TrivyDb;
|
||||
|
||||
public sealed record TrivyDbPackageRequest(
|
||||
ReadOnlyMemory<byte> MetadataJson,
|
||||
string DatabaseArchivePath,
|
||||
string DatabaseDigest,
|
||||
long DatabaseLength,
|
||||
DateTimeOffset GeneratedAt,
|
||||
string DatabaseVersion);
|
||||
Reference in New Issue
Block a user