Resolve Concelier/Excititor merge conflicts
This commit is contained in:
28
src/StellaOps.Concelier.Exporter.Json/AGENTS.md
Normal file
28
src/StellaOps.Concelier.Exporter.Json/AGENTS.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving.
|
||||
## Scope
|
||||
- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable).
|
||||
- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601.
|
||||
- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites.
|
||||
- Packaging: output directory under exports/json/<timestamp> with reproducible naming; optionally symlink latest.
|
||||
- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads.
|
||||
## Participants
|
||||
- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests.
|
||||
- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job.
|
||||
- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured.
|
||||
## Interfaces & contracts
|
||||
- Job kind: export:json (JsonExportJob).
|
||||
- Determinism: same inputs -> identical file bytes; hash snapshot persisted.
|
||||
- Provenance: include minimal provenance fields when helpful; keep identity stable.
|
||||
## In/Out of scope
|
||||
In: JSON rendering and layout; incremental/deterministic writes.
|
||||
Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter).
|
||||
## Observability & security expectations
|
||||
- Metrics: export.json.records, bytes, duration, delta.changed.
|
||||
- Logs: target path, record counts, digest; no sensitive data.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.Exporter.Json.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public static class ExportDigestCalculator
|
||||
{
|
||||
public static string ComputeTreeDigest(JsonExportResult result)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
|
||||
using var sha256 = SHA256.Create();
|
||||
var buffer = new byte[128 * 1024];
|
||||
|
||||
foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal))
|
||||
{
|
||||
var normalized = file.Replace("\\", "/");
|
||||
var pathBytes = Encoding.UTF8.GetBytes(normalized);
|
||||
_ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0);
|
||||
|
||||
var fullPath = ResolveFullPath(result.ExportDirectory, normalized);
|
||||
using var stream = File.OpenRead(fullPath);
|
||||
int read;
|
||||
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
_ = sha256.TransformBlock(buffer, 0, read, null, 0);
|
||||
}
|
||||
}
|
||||
|
||||
_ = sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||
var hash = sha256.Hash ?? Array.Empty<byte>();
|
||||
var hex = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
return $"sha256:{hex}";
|
||||
}
|
||||
|
||||
private static string ResolveFullPath(string root, string normalizedRelativePath)
|
||||
{
|
||||
var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
var parts = new string[segments.Length + 1];
|
||||
parts[0] = root;
|
||||
for (var i = 0; i < segments.Length; i++)
|
||||
{
|
||||
parts[i + 1] = segments[i];
|
||||
}
|
||||
|
||||
return Path.Combine(parts);
|
||||
}
|
||||
}
|
||||
28
src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs
Normal file
28
src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs
Normal file
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public static class ExporterVersion
|
||||
{
|
||||
public static string GetVersion(Type anchor)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(anchor);
|
||||
var assembly = anchor.Assembly;
|
||||
|
||||
var informational = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
|
||||
if (!string.IsNullOrWhiteSpace(informational))
|
||||
{
|
||||
return informational;
|
||||
}
|
||||
|
||||
var fileVersion = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version;
|
||||
if (!string.IsNullOrWhiteSpace(fileVersion))
|
||||
{
|
||||
return fileVersion!;
|
||||
}
|
||||
|
||||
var version = assembly.GetName().Version;
|
||||
return version?.ToString() ?? "0.0.0";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public interface IJsonExportPathResolver
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns the relative path (using platform directory separators) for the supplied advisory.
|
||||
/// Path must not include the leading export root.
|
||||
/// </summary>
|
||||
string GetRelativePath(Advisory advisory);
|
||||
}
|
||||
37
src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs
Normal file
37
src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs
Normal file
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Metadata describing a single file produced by the JSON exporter.
|
||||
/// </summary>
|
||||
public sealed class JsonExportFile
|
||||
{
|
||||
public JsonExportFile(string relativePath, long length, string digest)
|
||||
{
|
||||
RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath));
|
||||
if (relativePath.Length == 0)
|
||||
{
|
||||
throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath));
|
||||
}
|
||||
|
||||
if (length < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(length));
|
||||
}
|
||||
|
||||
Digest = digest ?? throw new ArgumentNullException(nameof(digest));
|
||||
if (digest.Length == 0)
|
||||
{
|
||||
throw new ArgumentException("Digest cannot be empty.", nameof(digest));
|
||||
}
|
||||
|
||||
Length = length;
|
||||
}
|
||||
|
||||
public string RelativePath { get; }
|
||||
|
||||
public long Length { get; }
|
||||
|
||||
public string Digest { get; }
|
||||
}
|
||||
30
src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs
Normal file
30
src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs
Normal file
@@ -0,0 +1,30 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExportJob : IJob
|
||||
{
|
||||
public const string JobKind = "export:json";
|
||||
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10);
|
||||
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly JsonFeedExporter _exporter;
|
||||
private readonly ILogger<JsonExportJob> _logger;
|
||||
|
||||
public JsonExportJob(JsonFeedExporter exporter, ILogger<JsonExportJob> logger)
|
||||
{
|
||||
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Executing JSON export job {RunId}", context.RunId);
|
||||
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Completed JSON export job {RunId}", context.RunId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
internal static class JsonExportManifestWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = true,
|
||||
};
|
||||
|
||||
public static async Task WriteAsync(
|
||||
JsonExportResult result,
|
||||
string digest,
|
||||
string exporterVersion,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
ArgumentException.ThrowIfNullOrEmpty(digest);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
|
||||
|
||||
var exportId = Path.GetFileName(result.ExportDirectory);
|
||||
var files = result.Files
|
||||
.Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
var manifest = new JsonExportManifest(
|
||||
exportId,
|
||||
result.ExportedAt.UtcDateTime,
|
||||
digest,
|
||||
result.AdvisoryCount,
|
||||
result.TotalBytes,
|
||||
files.Length,
|
||||
files,
|
||||
exporterVersion);
|
||||
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions);
|
||||
var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json");
|
||||
await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime);
|
||||
}
|
||||
|
||||
private sealed record JsonExportManifest(
|
||||
[property: JsonPropertyOrder(1)] string ExportId,
|
||||
[property: JsonPropertyOrder(2)] DateTime GeneratedAt,
|
||||
[property: JsonPropertyOrder(3)] string Digest,
|
||||
[property: JsonPropertyOrder(4)] int AdvisoryCount,
|
||||
[property: JsonPropertyOrder(5)] long TotalBytes,
|
||||
[property: JsonPropertyOrder(6)] int FileCount,
|
||||
[property: JsonPropertyOrder(7)] IReadOnlyList<JsonExportManifestFile> Files,
|
||||
[property: JsonPropertyOrder(8)] string ExporterVersion);
|
||||
|
||||
private sealed record JsonExportManifestFile(
|
||||
[property: JsonPropertyOrder(1)] string Path,
|
||||
[property: JsonPropertyOrder(2)] long Bytes,
|
||||
[property: JsonPropertyOrder(3)] string Digest);
|
||||
}
|
||||
115
src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs
Normal file
115
src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs
Normal file
@@ -0,0 +1,115 @@
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for JSON exporter output paths and determinism controls.
|
||||
/// </summary>
|
||||
public sealed class JsonExportOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Root directory where exports are written. Default "exports/json".
|
||||
/// </summary>
|
||||
public string OutputRoot { get; set; } = Path.Combine("exports", "json");
|
||||
|
||||
/// <summary>
|
||||
/// Format string applied to the export timestamp to produce the directory name.
|
||||
/// </summary>
|
||||
public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
|
||||
|
||||
/// <summary>
|
||||
/// Optional static name for the symlink (or directory junction) pointing at the most recent export.
|
||||
/// </summary>
|
||||
public string LatestSymlinkName { get; set; } = "latest";
|
||||
|
||||
/// <summary>
|
||||
/// When true, attempts to re-point <see cref="LatestSymlinkName"/> after a successful export.
|
||||
/// </summary>
|
||||
public bool MaintainLatestSymlink { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional repository identifier recorded alongside export state metadata.
|
||||
/// </summary>
|
||||
public string? TargetRepository { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Mirror distribution configuration producing aggregate bundles for downstream mirrors.
|
||||
/// </summary>
|
||||
public JsonMirrorOptions Mirror { get; set; } = new();
|
||||
|
||||
public sealed class JsonMirrorOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Indicates whether mirror bundle generation is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Directory name (relative to the export root) where mirror artefacts are written.
|
||||
/// </summary>
|
||||
public string DirectoryName { get; set; } = "mirror";
|
||||
|
||||
/// <summary>
|
||||
/// Domains exposed to downstream mirrors.
|
||||
/// </summary>
|
||||
public IList<JsonMirrorDomainOptions> Domains { get; } = new List<JsonMirrorDomainOptions>();
|
||||
|
||||
/// <summary>
|
||||
/// Signing configuration for mirror bundles.
|
||||
/// </summary>
|
||||
public JsonMirrorSigningOptions Signing { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class JsonMirrorDomainOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Stable identifier for the mirror domain (used in URLs and directory names).
|
||||
/// </summary>
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional human-readable label for UI surfaces.
|
||||
/// </summary>
|
||||
public string? DisplayName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional advisory scheme filters (e.g. CVE, GHSA). Empty collection selects all schemes.
|
||||
/// </summary>
|
||||
public IList<string> IncludeSchemes { get; } = new List<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Optional provenance source filters (e.g. nvd, ghsa). Empty collection selects all sources.
|
||||
/// </summary>
|
||||
public IList<string> IncludeSources { get; } = new List<string>();
|
||||
}
|
||||
|
||||
public sealed class JsonMirrorSigningOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Indicates whether bundles should be signed. Defaults to disabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Signing algorithm identifier (defaults to ES256).
|
||||
/// </summary>
|
||||
public string Algorithm { get; set; } = SignatureAlgorithms.Es256;
|
||||
|
||||
/// <summary>
|
||||
/// Active signing key identifier.
|
||||
/// </summary>
|
||||
public string KeyId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Path to the private key (PEM) used for signing mirror bundles.
|
||||
/// </summary>
|
||||
public string KeyPath { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional crypto provider hint. When omitted the registry resolves an appropriate provider.
|
||||
/// </summary>
|
||||
public string? Provider { get; set; }
|
||||
}
|
||||
}
|
||||
55
src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs
Normal file
55
src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExportResult
|
||||
{
|
||||
public JsonExportResult(
|
||||
string exportDirectory,
|
||||
DateTimeOffset exportedAt,
|
||||
IEnumerable<JsonExportFile> files,
|
||||
int advisoryCount,
|
||||
long totalBytes,
|
||||
IEnumerable<Advisory>? advisories = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(exportDirectory))
|
||||
{
|
||||
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
|
||||
}
|
||||
|
||||
var list = (files ?? throw new ArgumentNullException(nameof(files)))
|
||||
.Where(static file => file is not null)
|
||||
.ToImmutableArray();
|
||||
|
||||
var advisoryList = (advisories ?? Array.Empty<Advisory>())
|
||||
.Where(static advisory => advisory is not null)
|
||||
.ToImmutableArray();
|
||||
|
||||
ExportDirectory = exportDirectory;
|
||||
ExportedAt = exportedAt;
|
||||
TotalBytes = totalBytes;
|
||||
|
||||
Files = list;
|
||||
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
|
||||
Advisories = advisoryList;
|
||||
AdvisoryCount = advisoryList.IsDefaultOrEmpty ? advisoryCount : advisoryList.Length;
|
||||
}
|
||||
|
||||
public string ExportDirectory { get; }
|
||||
|
||||
public DateTimeOffset ExportedAt { get; }
|
||||
|
||||
public ImmutableArray<JsonExportFile> Files { get; }
|
||||
|
||||
public ImmutableArray<string> FilePaths { get; }
|
||||
|
||||
public ImmutableArray<Advisory> Advisories { get; }
|
||||
|
||||
public int AdvisoryCount { get; }
|
||||
|
||||
public long TotalBytes { get; }
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering.
|
||||
/// </summary>
|
||||
public sealed class JsonExportSnapshotBuilder
|
||||
{
|
||||
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
|
||||
private readonly JsonExportOptions _options;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
|
||||
public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
}
|
||||
|
||||
public Task<JsonExportResult> WriteAsync(
|
||||
IReadOnlyCollection<Advisory> advisories,
|
||||
DateTimeOffset exportedAt,
|
||||
string? exportName = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (advisories is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisories));
|
||||
}
|
||||
|
||||
return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<JsonExportResult> WriteAsync(
|
||||
IAsyncEnumerable<Advisory> advisories,
|
||||
DateTimeOffset exportedAt,
|
||||
string? exportName = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (advisories is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisories));
|
||||
}
|
||||
|
||||
var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
|
||||
if (string.IsNullOrWhiteSpace(exportDirectoryName))
|
||||
{
|
||||
throw new InvalidOperationException("Export directory name resolved to an empty string.");
|
||||
}
|
||||
|
||||
var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot));
|
||||
TrySetDirectoryTimestamp(exportRoot, exportedAt);
|
||||
var exportDirectory = Path.Combine(exportRoot, exportDirectoryName);
|
||||
|
||||
if (Directory.Exists(exportDirectory))
|
||||
{
|
||||
Directory.Delete(exportDirectory, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(exportDirectory);
|
||||
TrySetDirectoryTimestamp(exportDirectory, exportedAt);
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var files = new List<JsonExportFile>();
|
||||
var advisoryList = new List<Advisory>();
|
||||
long totalBytes = 0L;
|
||||
|
||||
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var entry = Resolve(advisory);
|
||||
if (!seen.Add(entry.RelativePath))
|
||||
{
|
||||
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
|
||||
}
|
||||
|
||||
advisoryList.Add(entry.Advisory);
|
||||
|
||||
var destination = Combine(exportDirectory, entry.Segments);
|
||||
var destinationDirectory = Path.GetDirectoryName(destination);
|
||||
if (!string.IsNullOrEmpty(destinationDirectory))
|
||||
{
|
||||
EnsureDirectoryExists(destinationDirectory);
|
||||
TrySetDirectoryTimestamp(destinationDirectory, exportedAt);
|
||||
}
|
||||
var payload = SnapshotSerializer.ToSnapshot(entry.Advisory);
|
||||
var bytes = Utf8NoBom.GetBytes(payload);
|
||||
|
||||
await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime);
|
||||
|
||||
var digest = ComputeDigest(bytes);
|
||||
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
|
||||
totalBytes += bytes.LongLength;
|
||||
}
|
||||
|
||||
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
|
||||
|
||||
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryList.Count, totalBytes, advisoryList);
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<Advisory> EnumerateAsync(
|
||||
IEnumerable<Advisory> advisories,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
yield return advisory;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
|
||||
private static string EnsureDirectoryExists(string directory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
{
|
||||
throw new ArgumentException("Directory path must be provided.", nameof(directory));
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
private static string Combine(string root, IReadOnlyList<string> segments)
|
||||
{
|
||||
var parts = new string[segments.Count + 1];
|
||||
parts[0] = root;
|
||||
for (var i = 0; i < segments.Count; i++)
|
||||
{
|
||||
parts[i + 1] = segments[i];
|
||||
}
|
||||
|
||||
return Path.Combine(parts);
|
||||
}
|
||||
|
||||
private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Ignore failure to set timestamps; not critical for content determinism.
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
// Ignore permission issues when setting timestamps.
|
||||
}
|
||||
catch (PlatformNotSupportedException)
|
||||
{
|
||||
// Some platforms may not support this operation.
|
||||
}
|
||||
}
|
||||
|
||||
private PathResolution Resolve(Advisory advisory)
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisory));
|
||||
}
|
||||
|
||||
var normalized = CanonicalJsonSerializer.Normalize(advisory);
|
||||
var relativePath = _pathResolver.GetRelativePath(normalized);
|
||||
var segments = NormalizeRelativePath(relativePath);
|
||||
var normalizedPath = string.Join('/', segments);
|
||||
return new PathResolution(normalized, normalizedPath, segments);
|
||||
}
|
||||
|
||||
private static string[] NormalizeRelativePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver returned an empty path.");
|
||||
}
|
||||
|
||||
if (Path.IsPathRooted(relativePath))
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported.");
|
||||
}
|
||||
|
||||
var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (pieces.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver produced no path segments.");
|
||||
}
|
||||
|
||||
var sanitized = new string[pieces.Length];
|
||||
for (var i = 0; i < pieces.Length; i++)
|
||||
{
|
||||
var segment = pieces[i];
|
||||
if (segment == "." || segment == "..")
|
||||
{
|
||||
throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments.");
|
||||
}
|
||||
|
||||
sanitized[i] = SanitizeSegment(segment);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
private static string SanitizeSegment(string segment)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
Span<char> buffer = stackalloc char[segment.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in segment)
|
||||
{
|
||||
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
|
||||
{
|
||||
buffer[count++] = '_';
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer[count++] = ch;
|
||||
}
|
||||
}
|
||||
|
||||
var sanitized = new string(buffer[..count]).Trim();
|
||||
return string.IsNullOrEmpty(sanitized) ? "_" : sanitized;
|
||||
}
|
||||
|
||||
private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList<string> Segments);
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = SHA256.HashData(payload);
|
||||
var hex = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
return $"sha256:{hex}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:exporters:json";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
|
||||
services.TryAddSingleton<ExportStateManager>();
|
||||
|
||||
services.AddOptions<JsonExportOptions>()
|
||||
.Bind(configuration.GetSection(ConfigurationSection))
|
||||
.PostConfigure(static options =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.OutputRoot))
|
||||
{
|
||||
options.OutputRoot = Path.Combine("exports", "json");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
|
||||
{
|
||||
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Mirror.DirectoryName))
|
||||
{
|
||||
options.Mirror.DirectoryName = "mirror";
|
||||
}
|
||||
});
|
||||
|
||||
services.AddSingleton<JsonFeedExporter>();
|
||||
services.AddTransient<JsonExportJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
if (!options.Definitions.ContainsKey(JsonExportJob.JobKind))
|
||||
{
|
||||
options.Definitions[JsonExportJob.JobKind] = new JobDefinition(
|
||||
JsonExportJob.JobKind,
|
||||
typeof(JsonExportJob),
|
||||
JsonExportJob.DefaultTimeout,
|
||||
JsonExportJob.DefaultLeaseDuration,
|
||||
null,
|
||||
true);
|
||||
}
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
23
src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs
Normal file
23
src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs
Normal file
@@ -0,0 +1,23 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExporterPlugin : IExporterPlugin
|
||||
{
|
||||
public string Name => JsonFeedExporter.ExporterName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetService<IAdvisoryStore>() is not null;
|
||||
}
|
||||
|
||||
public IFeedExporter Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<JsonFeedExporter>(services);
|
||||
}
|
||||
}
|
||||
205
src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs
Normal file
205
src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs
Normal file
@@ -0,0 +1,205 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Core.Events;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonFeedExporter : IFeedExporter
|
||||
{
|
||||
public const string ExporterName = "json";
|
||||
public const string ExporterId = "export:json";
|
||||
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly JsonExportOptions _options;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
private readonly ExportStateManager _stateManager;
|
||||
private readonly ILogger<JsonFeedExporter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _exporterVersion;
|
||||
private readonly IAdvisoryEventLog _eventLog;
|
||||
|
||||
public JsonFeedExporter(
|
||||
IAdvisoryStore advisoryStore,
|
||||
IOptions<JsonExportOptions> options,
|
||||
IJsonExportPathResolver pathResolver,
|
||||
ExportStateManager stateManager,
|
||||
IAdvisoryEventLog eventLog,
|
||||
ILogger<JsonFeedExporter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
|
||||
_eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
|
||||
}
|
||||
|
||||
public string Name => ExporterName;
|
||||
|
||||
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var exportedAt = _timeProvider.GetUtcNow();
|
||||
var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
|
||||
var exportRoot = Path.GetFullPath(_options.OutputRoot);
|
||||
|
||||
_logger.LogInformation("Starting JSON export {ExportId}", exportId);
|
||||
|
||||
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
|
||||
var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
|
||||
var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
|
||||
_logger.LogInformation(
|
||||
"JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
|
||||
exportId,
|
||||
result.Files.Length,
|
||||
result.TotalBytes,
|
||||
result.AdvisoryCount,
|
||||
digest);
|
||||
|
||||
var manifest = result.Files
|
||||
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
if (existingState is not null
|
||||
&& existingState.Files.Count > 0
|
||||
&& string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId);
|
||||
TryDeleteDirectory(result.ExportDirectory);
|
||||
return;
|
||||
}
|
||||
|
||||
var resetBaseline = existingState is null
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
|
||||
|
||||
if (existingState is not null
|
||||
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
|
||||
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
|
||||
{
|
||||
resetBaseline = true;
|
||||
}
|
||||
|
||||
await _stateManager.StoreFullExportAsync(
|
||||
ExporterId,
|
||||
exportId,
|
||||
digest,
|
||||
cursor: digest,
|
||||
targetRepository: _options.TargetRepository,
|
||||
exporterVersion: _exporterVersion,
|
||||
resetBaseline: resetBaseline,
|
||||
manifest: manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.MaintainLatestSymlink)
|
||||
{
|
||||
TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<Advisory>> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var keys = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
|
||||
{
|
||||
keys.Add(advisory.AdvisoryKey.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
var advisories = new List<Advisory>(keys.Count);
|
||||
foreach (var key in keys)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false);
|
||||
if (!replay.Statements.IsDefaultOrEmpty)
|
||||
{
|
||||
advisories.Add(replay.Statements[0].Advisory);
|
||||
}
|
||||
}
|
||||
|
||||
return advisories;
|
||||
}
|
||||
|
||||
private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName);
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(latestPath) || File.Exists(latestPath))
|
||||
{
|
||||
TryRemoveExistingPointer(latestPath);
|
||||
}
|
||||
|
||||
Directory.CreateSymbolicLink(latestPath, exportDirectory);
|
||||
_logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryRemoveExistingPointer(string latestPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var attributes = File.GetAttributes(latestPath);
|
||||
if (attributes.HasFlag(FileAttributes.Directory))
|
||||
{
|
||||
Directory.Delete(latestPath, recursive: false);
|
||||
}
|
||||
else
|
||||
{
|
||||
File.Delete(latestPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryDeleteDirectory(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(path))
|
||||
{
|
||||
Directory.Delete(path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path);
|
||||
}
|
||||
}
|
||||
}
|
||||
622
src/StellaOps.Concelier.Exporter.Json/JsonMirrorBundleWriter.cs
Normal file
622
src/StellaOps.Concelier.Exporter.Json/JsonMirrorBundleWriter.cs
Normal file
@@ -0,0 +1,622 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
internal static class JsonMirrorBundleWriter
|
||||
{
|
||||
private const int SchemaVersion = 1;
|
||||
private const string BundleFileName = "bundle.json";
|
||||
private const string BundleSignatureFileName = "bundle.json.jws";
|
||||
private const string ManifestFileName = "manifest.json";
|
||||
private const string IndexFileName = "index.json";
|
||||
private const string SignatureMediaType = "application/vnd.stellaops.concelier.mirror-bundle+jws";
|
||||
private const string DefaultMirrorDirectoryName = "mirror";
|
||||
|
||||
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
|
||||
|
||||
private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.General)
|
||||
{
|
||||
PropertyNamingPolicy = null,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
public static async Task<JsonExportResult> WriteAsync(
|
||||
JsonExportResult result,
|
||||
JsonExportOptions options,
|
||||
IServiceProvider services,
|
||||
TimeProvider timeProvider,
|
||||
ILogger logger,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
var mirrorOptions = options.Mirror ?? new JsonExportOptions.JsonMirrorOptions();
|
||||
if (!mirrorOptions.Enabled || mirrorOptions.Domains.Count == 0)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var exportedAtUtc = result.ExportedAt.UtcDateTime;
|
||||
var mirrorDirectoryName = string.IsNullOrWhiteSpace(mirrorOptions.DirectoryName)
|
||||
? DefaultMirrorDirectoryName
|
||||
: mirrorOptions.DirectoryName.Trim();
|
||||
|
||||
var mirrorRoot = Path.Combine(result.ExportDirectory, mirrorDirectoryName);
|
||||
Directory.CreateDirectory(mirrorRoot);
|
||||
TrySetDirectoryTimestamp(mirrorRoot, exportedAtUtc);
|
||||
|
||||
var advisories = result.Advisories.IsDefaultOrEmpty
|
||||
? Array.Empty<Advisory>()
|
||||
: result.Advisories
|
||||
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var signingContext = PrepareSigningContext(mirrorOptions.Signing, services, timeProvider, logger);
|
||||
var additionalFiles = new List<JsonExportFile>();
|
||||
var domainEntries = new List<MirrorIndexDomainEntry>();
|
||||
|
||||
foreach (var domainOption in mirrorOptions.Domains)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (domainOption is null)
|
||||
{
|
||||
logger.LogWarning("Encountered null mirror domain configuration; skipping.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var domainId = (domainOption.Id ?? string.Empty).Trim();
|
||||
if (domainId.Length == 0)
|
||||
{
|
||||
logger.LogWarning("Skipping mirror domain with empty id.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var schemeFilter = CreateFilterSet(domainOption.IncludeSchemes);
|
||||
var sourceFilter = CreateFilterSet(domainOption.IncludeSources);
|
||||
var domainAdvisories = advisories
|
||||
.Where(advisory => MatchesFilters(advisory, schemeFilter, sourceFilter))
|
||||
.ToArray();
|
||||
|
||||
var sources = BuildSourceSummaries(domainAdvisories);
|
||||
var domainDisplayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
|
||||
? domainId
|
||||
: domainOption.DisplayName!.Trim();
|
||||
|
||||
var domainDirectory = Path.Combine(mirrorRoot, domainId);
|
||||
Directory.CreateDirectory(domainDirectory);
|
||||
TrySetDirectoryTimestamp(domainDirectory, exportedAtUtc);
|
||||
|
||||
var bundleDocument = new MirrorDomainBundleDocument(
|
||||
SchemaVersion,
|
||||
result.ExportedAt,
|
||||
options.TargetRepository,
|
||||
domainId,
|
||||
domainDisplayName,
|
||||
domainAdvisories.Length,
|
||||
domainAdvisories,
|
||||
sources);
|
||||
|
||||
var bundleBytes = Serialize(bundleDocument);
|
||||
var bundlePath = Path.Combine(domainDirectory, BundleFileName);
|
||||
await WriteFileAsync(bundlePath, bundleBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var bundleRelativePath = ToRelativePath(result.ExportDirectory, bundlePath);
|
||||
var bundleDigest = ComputeDigest(bundleBytes);
|
||||
var bundleLength = (long)bundleBytes.LongLength;
|
||||
additionalFiles.Add(new JsonExportFile(bundleRelativePath, bundleLength, bundleDigest));
|
||||
|
||||
MirrorSignatureDescriptor? signatureDescriptor = null;
|
||||
if (signingContext is not null)
|
||||
{
|
||||
var (signatureValue, signedAt) = await CreateSignatureAsync(
|
||||
signingContext,
|
||||
bundleBytes,
|
||||
timeProvider,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var signatureBytes = Utf8NoBom.GetBytes(signatureValue);
|
||||
var signaturePath = Path.Combine(domainDirectory, BundleSignatureFileName);
|
||||
await WriteFileAsync(signaturePath, signatureBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var signatureRelativePath = ToRelativePath(result.ExportDirectory, signaturePath);
|
||||
var signatureDigest = ComputeDigest(signatureBytes);
|
||||
var signatureLength = (long)signatureBytes.LongLength;
|
||||
additionalFiles.Add(new JsonExportFile(signatureRelativePath, signatureLength, signatureDigest));
|
||||
|
||||
signatureDescriptor = new MirrorSignatureDescriptor(
|
||||
signatureRelativePath,
|
||||
signingContext.Algorithm,
|
||||
signingContext.KeyId,
|
||||
signingContext.Provider,
|
||||
signedAt);
|
||||
}
|
||||
|
||||
var bundleDescriptor = new MirrorFileDescriptor(bundleRelativePath, bundleLength, bundleDigest, signatureDescriptor);
|
||||
|
||||
var manifestDocument = new MirrorDomainManifestDocument(
|
||||
SchemaVersion,
|
||||
result.ExportedAt,
|
||||
domainId,
|
||||
domainDisplayName,
|
||||
domainAdvisories.Length,
|
||||
sources,
|
||||
bundleDescriptor);
|
||||
|
||||
var manifestBytes = Serialize(manifestDocument);
|
||||
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
|
||||
await WriteFileAsync(manifestPath, manifestBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var manifestRelativePath = ToRelativePath(result.ExportDirectory, manifestPath);
|
||||
var manifestDigest = ComputeDigest(manifestBytes);
|
||||
var manifestLength = (long)manifestBytes.LongLength;
|
||||
additionalFiles.Add(new JsonExportFile(manifestRelativePath, manifestLength, manifestDigest));
|
||||
|
||||
var manifestDescriptor = new MirrorFileDescriptor(manifestRelativePath, manifestLength, manifestDigest, null);
|
||||
|
||||
domainEntries.Add(new MirrorIndexDomainEntry(
|
||||
domainId,
|
||||
domainDisplayName,
|
||||
domainAdvisories.Length,
|
||||
manifestDescriptor,
|
||||
bundleDescriptor,
|
||||
sources));
|
||||
}
|
||||
|
||||
domainEntries.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
|
||||
|
||||
var indexDocument = new MirrorIndexDocument(
|
||||
SchemaVersion,
|
||||
result.ExportedAt,
|
||||
options.TargetRepository,
|
||||
domainEntries);
|
||||
|
||||
var indexBytes = Serialize(indexDocument);
|
||||
var indexPath = Path.Combine(mirrorRoot, IndexFileName);
|
||||
await WriteFileAsync(indexPath, indexBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var indexRelativePath = ToRelativePath(result.ExportDirectory, indexPath);
|
||||
var indexDigest = ComputeDigest(indexBytes);
|
||||
var indexLength = (long)indexBytes.LongLength;
|
||||
additionalFiles.Add(new JsonExportFile(indexRelativePath, indexLength, indexDigest));
|
||||
|
||||
logger.LogInformation(
|
||||
"Generated {DomainCount} Concelier mirror domain bundle(s) under {MirrorRoot}.",
|
||||
domainEntries.Count,
|
||||
mirrorDirectoryName);
|
||||
|
||||
var combinedFiles = new List<JsonExportFile>(result.Files.Length + additionalFiles.Count);
|
||||
combinedFiles.AddRange(result.Files);
|
||||
combinedFiles.AddRange(additionalFiles);
|
||||
|
||||
var combinedTotalBytes = checked(result.TotalBytes + additionalFiles.Sum(static file => file.Length));
|
||||
|
||||
return new JsonExportResult(
|
||||
result.ExportDirectory,
|
||||
result.ExportedAt,
|
||||
combinedFiles,
|
||||
result.AdvisoryCount,
|
||||
combinedTotalBytes,
|
||||
result.Advisories);
|
||||
}
|
||||
|
||||
private static JsonMirrorSigningContext? PrepareSigningContext(
|
||||
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
|
||||
IServiceProvider services,
|
||||
TimeProvider timeProvider,
|
||||
ILogger logger)
|
||||
{
|
||||
if (signingOptions is null || !signingOptions.Enabled)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var algorithm = string.IsNullOrWhiteSpace(signingOptions.Algorithm)
|
||||
? SignatureAlgorithms.Es256
|
||||
: signingOptions.Algorithm.Trim();
|
||||
var keyId = (signingOptions.KeyId ?? string.Empty).Trim();
|
||||
if (keyId.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyId to be configured.");
|
||||
}
|
||||
|
||||
var registry = services.GetService<ICryptoProviderRegistry>()
|
||||
?? throw new InvalidOperationException("Mirror signing requires ICryptoProviderRegistry to be registered.");
|
||||
|
||||
var providerHint = signingOptions.Provider?.Trim();
|
||||
var keyReference = new CryptoKeyReference(keyId, providerHint);
|
||||
|
||||
CryptoSignerResolution resolved;
|
||||
try
|
||||
{
|
||||
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, providerHint);
|
||||
}
|
||||
catch (KeyNotFoundException)
|
||||
{
|
||||
var provider = ResolveProvider(registry, algorithm, providerHint);
|
||||
var signingKey = LoadSigningKey(signingOptions, provider, services, timeProvider, algorithm);
|
||||
provider.UpsertSigningKey(signingKey);
|
||||
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, provider.Name);
|
||||
}
|
||||
|
||||
logger.LogDebug(
|
||||
"Mirror signing configured with key {KeyId} via provider {Provider} using {Algorithm}.",
|
||||
resolved.Signer.KeyId,
|
||||
resolved.ProviderName,
|
||||
algorithm);
|
||||
|
||||
return new JsonMirrorSigningContext(resolved.Signer, algorithm, resolved.Signer.KeyId, resolved.ProviderName);
|
||||
}
|
||||
|
||||
private static ICryptoProvider ResolveProvider(ICryptoProviderRegistry registry, string algorithm, string? providerHint)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(providerHint) && registry.TryResolve(providerHint, out var hinted))
|
||||
{
|
||||
if (!hinted.Supports(CryptoCapability.Signing, algorithm))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Crypto provider '{providerHint}' does not support signing algorithm '{algorithm}'.");
|
||||
}
|
||||
|
||||
return hinted;
|
||||
}
|
||||
|
||||
return registry.ResolveOrThrow(CryptoCapability.Signing, algorithm);
|
||||
}
|
||||
|
||||
private static CryptoSigningKey LoadSigningKey(
|
||||
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
|
||||
ICryptoProvider provider,
|
||||
IServiceProvider services,
|
||||
TimeProvider timeProvider,
|
||||
string algorithm)
|
||||
{
|
||||
var keyPath = (signingOptions.KeyPath ?? string.Empty).Trim();
|
||||
if (keyPath.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyPath to be configured.");
|
||||
}
|
||||
|
||||
var environment = services.GetService<IHostEnvironment>();
|
||||
var basePath = environment?.ContentRootPath ?? AppContext.BaseDirectory;
|
||||
var resolvedPath = Path.IsPathRooted(keyPath)
|
||||
? keyPath
|
||||
: Path.GetFullPath(Path.Combine(basePath, keyPath));
|
||||
|
||||
if (!File.Exists(resolvedPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Mirror signing key '{signingOptions.KeyId}' not found.", resolvedPath);
|
||||
}
|
||||
|
||||
var pem = File.ReadAllText(resolvedPath);
|
||||
using var ecdsa = ECDsa.Create();
|
||||
try
|
||||
{
|
||||
ecdsa.ImportFromPem(pem);
|
||||
}
|
||||
catch (CryptographicException ex)
|
||||
{
|
||||
throw new InvalidOperationException("Failed to import mirror signing key. Ensure the PEM contains an EC private key.", ex);
|
||||
}
|
||||
|
||||
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
|
||||
return new CryptoSigningKey(
|
||||
new CryptoKeyReference(signingOptions.KeyId, provider.Name),
|
||||
algorithm,
|
||||
in parameters,
|
||||
timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
private static async Task<(string Value, DateTimeOffset SignedAt)> CreateSignatureAsync(
|
||||
JsonMirrorSigningContext context,
|
||||
ReadOnlyMemory<byte> payload,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var header = new Dictionary<string, object>
|
||||
{
|
||||
["alg"] = context.Algorithm,
|
||||
["kid"] = context.KeyId,
|
||||
["typ"] = SignatureMediaType,
|
||||
["b64"] = false,
|
||||
["crit"] = new[] { "b64" }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(context.Provider))
|
||||
{
|
||||
header["provider"] = context.Provider;
|
||||
}
|
||||
|
||||
var headerJson = JsonSerializer.Serialize(header, HeaderSerializerOptions);
|
||||
var protectedHeader = Base64UrlEncode(Utf8NoBom.GetBytes(headerJson));
|
||||
var signingInputLength = protectedHeader.Length + 1 + payload.Length;
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(signingInputLength);
|
||||
|
||||
try
|
||||
{
|
||||
var headerBytes = Encoding.ASCII.GetBytes(protectedHeader);
|
||||
Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length);
|
||||
buffer[headerBytes.Length] = (byte)'.';
|
||||
var payloadArray = payload.ToArray();
|
||||
Buffer.BlockCopy(payloadArray, 0, buffer, headerBytes.Length + 1, payloadArray.Length);
|
||||
|
||||
var signingInput = new ReadOnlyMemory<byte>(buffer, 0, signingInputLength);
|
||||
var signatureBytes = await context.Signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false);
|
||||
var encodedSignature = Base64UrlEncode(signatureBytes);
|
||||
var signedAt = timeProvider.GetUtcNow();
|
||||
return (string.Concat(protectedHeader, "..", encodedSignature), signedAt);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<JsonMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
|
||||
{
|
||||
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var provenance in advisory.Provenance)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(provenance.Source))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var source = provenance.Source.Trim();
|
||||
if (!builders.TryGetValue(source, out var accumulator))
|
||||
{
|
||||
accumulator = new SourceAccumulator();
|
||||
builders[source] = accumulator;
|
||||
}
|
||||
|
||||
accumulator.Record(provenance.RecordedAt);
|
||||
if (counted.Add(source))
|
||||
{
|
||||
accumulator.IncrementAdvisoryCount();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return builders
|
||||
.OrderBy(static pair => pair.Key, StringComparer.Ordinal)
|
||||
.Select(pair => new JsonMirrorSourceSummary(
|
||||
pair.Key,
|
||||
pair.Value.FirstRecordedAt,
|
||||
pair.Value.LastRecordedAt,
|
||||
pair.Value.AdvisoryCount))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static HashSet<string>? CreateFilterSet(IList<string>? values)
|
||||
{
|
||||
if (values is null || values.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
set.Add(value.Trim());
|
||||
}
|
||||
|
||||
return set.Count == 0 ? null : set;
|
||||
}
|
||||
|
||||
private static bool MatchesFilters(Advisory advisory, HashSet<string>? schemeFilter, HashSet<string>? sourceFilter)
|
||||
{
|
||||
if (schemeFilter is not null)
|
||||
{
|
||||
var scheme = ExtractScheme(advisory.AdvisoryKey);
|
||||
if (!schemeFilter.Contains(scheme))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceFilter is not null)
|
||||
{
|
||||
var hasSource = advisory.Provenance.Any(provenance =>
|
||||
!string.IsNullOrWhiteSpace(provenance.Source) &&
|
||||
sourceFilter.Contains(provenance.Source.Trim()));
|
||||
|
||||
if (!hasSource)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string ExtractScheme(string advisoryKey)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = advisoryKey.Trim();
|
||||
var separatorIndex = trimmed.IndexOf(':');
|
||||
return separatorIndex <= 0 ? trimmed : trimmed[..separatorIndex];
|
||||
}
|
||||
|
||||
private static byte[] Serialize<T>(T value)
|
||||
{
|
||||
var json = CanonicalJsonSerializer.SerializeIndented(value);
|
||||
return Utf8NoBom.GetBytes(json);
|
||||
}
|
||||
|
||||
private static async Task WriteFileAsync(string path, byte[] content, DateTime exportedAtUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
await File.WriteAllBytesAsync(path, content, cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(path, exportedAtUtc);
|
||||
}
|
||||
|
||||
private static string ToRelativePath(string root, string fullPath)
|
||||
{
|
||||
var relative = Path.GetRelativePath(root, fullPath);
|
||||
return relative.Replace(Path.DirectorySeparatorChar, '/');
|
||||
}
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = SHA256.HashData(payload);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private static void TrySetDirectoryTimestamp(string directory, DateTime exportedAtUtc)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.SetLastWriteTimeUtc(directory, exportedAtUtc);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
}
|
||||
catch (PlatformNotSupportedException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static string Base64UrlEncode(ReadOnlySpan<byte> value)
|
||||
{
|
||||
var encoded = Convert.ToBase64String(value);
|
||||
var builder = new StringBuilder(encoded.Length);
|
||||
foreach (var ch in encoded)
|
||||
{
|
||||
switch (ch)
|
||||
{
|
||||
case '+':
|
||||
builder.Append('-');
|
||||
break;
|
||||
case '/':
|
||||
builder.Append('_');
|
||||
break;
|
||||
case '=':
|
||||
break;
|
||||
default:
|
||||
builder.Append(ch);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private sealed record JsonMirrorSigningContext(ICryptoSigner Signer, string Algorithm, string KeyId, string Provider);
|
||||
|
||||
private sealed record MirrorIndexDocument(
|
||||
int SchemaVersion,
|
||||
DateTimeOffset GeneratedAt,
|
||||
string? TargetRepository,
|
||||
IReadOnlyList<MirrorIndexDomainEntry> Domains);
|
||||
|
||||
private sealed record MirrorIndexDomainEntry(
|
||||
string DomainId,
|
||||
string DisplayName,
|
||||
int AdvisoryCount,
|
||||
MirrorFileDescriptor Manifest,
|
||||
MirrorFileDescriptor Bundle,
|
||||
IReadOnlyList<JsonMirrorSourceSummary> Sources);
|
||||
|
||||
private sealed record MirrorDomainManifestDocument(
|
||||
int SchemaVersion,
|
||||
DateTimeOffset GeneratedAt,
|
||||
string DomainId,
|
||||
string DisplayName,
|
||||
int AdvisoryCount,
|
||||
IReadOnlyList<JsonMirrorSourceSummary> Sources,
|
||||
MirrorFileDescriptor Bundle);
|
||||
|
||||
private sealed record MirrorDomainBundleDocument(
|
||||
int SchemaVersion,
|
||||
DateTimeOffset GeneratedAt,
|
||||
string? TargetRepository,
|
||||
string DomainId,
|
||||
string DisplayName,
|
||||
int AdvisoryCount,
|
||||
IReadOnlyList<Advisory> Advisories,
|
||||
IReadOnlyList<JsonMirrorSourceSummary> Sources);
|
||||
|
||||
private sealed record MirrorFileDescriptor(
|
||||
string Path,
|
||||
long SizeBytes,
|
||||
string Digest,
|
||||
MirrorSignatureDescriptor? Signature);
|
||||
|
||||
private sealed record MirrorSignatureDescriptor(
|
||||
string Path,
|
||||
string Algorithm,
|
||||
string KeyId,
|
||||
string Provider,
|
||||
DateTimeOffset SignedAt);
|
||||
|
||||
private sealed record JsonMirrorSourceSummary(
|
||||
string Source,
|
||||
DateTimeOffset? FirstRecordedAt,
|
||||
DateTimeOffset? LastRecordedAt,
|
||||
int AdvisoryCount);
|
||||
|
||||
private sealed class SourceAccumulator
|
||||
{
|
||||
public DateTimeOffset? FirstRecordedAt { get; private set; }
|
||||
|
||||
public DateTimeOffset? LastRecordedAt { get; private set; }
|
||||
|
||||
public int AdvisoryCount { get; private set; }
|
||||
|
||||
public void Record(DateTimeOffset recordedAt)
|
||||
{
|
||||
var normalized = recordedAt.ToUniversalTime();
|
||||
if (FirstRecordedAt is null || normalized < FirstRecordedAt.Value)
|
||||
{
|
||||
FirstRecordedAt = normalized;
|
||||
}
|
||||
|
||||
if (LastRecordedAt is null || normalized > LastRecordedAt.Value)
|
||||
{
|
||||
LastRecordedAt = normalized;
|
||||
}
|
||||
}
|
||||
|
||||
public void IncrementAdvisoryCount()
|
||||
{
|
||||
AdvisoryCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
13
src/StellaOps.Concelier.Exporter.Json/TASKS.md
Normal file
13
src/StellaOps.Concelier.Exporter.Json/TASKS.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE – `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.|
|
||||
|Deterministic serializer|BE-Export|Models|DONE – Canonical serializer + snapshot builder emit stable JSON across runs.|
|
||||
|ExportState read/write|BE-Export|Storage.Mongo|DONE – `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.|
|
||||
|JsonExportJob wiring|BE-Export|Core|DONE – Job scheduler options now configurable via DI; JSON job registered with scheduler.|
|
||||
|Snapshot tests for file tree|QA|Exporters|DONE – Added resolver/exporter tests asserting tree layout and deterministic behavior.|
|
||||
|Parity smoke vs upstream vuln-list|QA|Exporters|DONE – `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.|
|
||||
|Stream advisories during export|BE-Export|Storage.Mongo|DONE – exporter + streaming-only test ensures single enumeration and per-file digest capture.|
|
||||
|Emit export manifest with digest metadata|BE-Export|Exporters|DONE – manifest now includes per-file digests/sizes alongside tree digest.|
|
||||
|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.|
|
||||
|CONCELIER-EXPORT-08-201 – Mirror bundle + domain manifest|Team Concelier Export|FEEDCORE-ENGINE-07-001|DONE (2025-10-19) – Mirror bundle writer emits domain aggregates + manifests with cosign-compatible JWS signatures; index/tests updated via `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj` (2025-10-19).|
|
||||
@@ -0,0 +1,456 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Identifiers;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Path resolver approximating the directory layout used by aquasecurity/vuln-list.
|
||||
/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks.
|
||||
/// </summary>
|
||||
public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver
|
||||
{
|
||||
private static readonly Regex CvePattern = new("^CVE-(?<year>\\d{4})-(?<id>\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex UsnPattern = new("^USN-(?<id>\\d+-\\d+)(?<suffix>[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex DebianPattern = new("^(?<prefix>DLA|DSA|ELA)-(?<id>\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex RedHatPattern = new("^RH(?<type>SA|BA|EA)-(?<rest>[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex AmazonPattern = new("^ALAS(?<channel>2|2022|2023)?-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex OraclePattern = new("^(?<kind>ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex PhotonPattern = new("^PHSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex RockyPattern = new("^RLSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex SusePattern = new("^SUSE-(?<kind>SU|RU|OU|SB)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Dictionary<string, string[]> SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["nvd"] = new[] { "nvd" },
|
||||
["ghsa"] = new[] { "ghsa" },
|
||||
["github"] = new[] { "ghsa" },
|
||||
["osv"] = new[] { "osv" },
|
||||
["redhat"] = new[] { "redhat", "oval" },
|
||||
["ubuntu"] = new[] { "ubuntu" },
|
||||
["debian"] = new[] { "debian" },
|
||||
["oracle"] = new[] { "oracle" },
|
||||
["photon"] = new[] { "photon" },
|
||||
["rocky"] = new[] { "rocky" },
|
||||
["suse"] = new[] { "suse" },
|
||||
["amazon"] = new[] { "amazon" },
|
||||
["aws"] = new[] { "amazon" },
|
||||
["alpine"] = new[] { "alpine" },
|
||||
["wolfi"] = new[] { "wolfi" },
|
||||
["chainguard"] = new[] { "chainguard" },
|
||||
["cert-fr"] = new[] { "cert", "fr" },
|
||||
["cert-in"] = new[] { "cert", "in" },
|
||||
["cert-cc"] = new[] { "cert", "cc" },
|
||||
["cert-bund"] = new[] { "cert", "bund" },
|
||||
["acsc"] = new[] { "cert", "au" },
|
||||
["cisa"] = new[] { "ics", "cisa" },
|
||||
["ics-cisa"] = new[] { "ics", "cisa" },
|
||||
["ics-kaspersky"] = new[] { "ics", "kaspersky" },
|
||||
["kaspersky"] = new[] { "ics", "kaspersky" },
|
||||
};
|
||||
|
||||
private static readonly Dictionary<string, string> GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["go"] = "go",
|
||||
["golang"] = "go",
|
||||
["npm"] = "npm",
|
||||
["maven"] = "maven",
|
||||
["pypi"] = "pip",
|
||||
["pip"] = "pip",
|
||||
["nuget"] = "nuget",
|
||||
["composer"] = "composer",
|
||||
["packagist"] = "composer",
|
||||
["rubygems"] = "rubygems",
|
||||
["gem"] = "rubygems",
|
||||
["swift"] = "swift",
|
||||
["cargo"] = "cargo",
|
||||
["hex"] = "hex",
|
||||
["pub"] = "pub",
|
||||
["github"] = "github",
|
||||
["docker"] = "container",
|
||||
};
|
||||
|
||||
public string GetRelativePath(Advisory advisory)
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisory));
|
||||
}
|
||||
|
||||
var identifier = SelectPreferredIdentifier(advisory);
|
||||
if (identifier.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Unable to derive identifier for advisory.");
|
||||
}
|
||||
|
||||
var layout = ResolveLayout(advisory, identifier);
|
||||
var segments = new string[layout.Segments.Length + 1];
|
||||
for (var i = 0; i < layout.Segments.Length; i++)
|
||||
{
|
||||
segments[i] = layout.Segments[i];
|
||||
}
|
||||
segments[^1] = layout.FileName;
|
||||
return Path.Combine(segments);
|
||||
}
|
||||
|
||||
private static Layout ResolveLayout(Advisory advisory, string identifier)
|
||||
{
|
||||
if (TryResolveCve(identifier, out var layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveGhsa(advisory, identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveUsn(identifier, out layout) ||
|
||||
TryResolveDebian(identifier, out layout) ||
|
||||
TryResolveRedHat(identifier, out layout) ||
|
||||
TryResolveAmazon(identifier, out layout) ||
|
||||
TryResolveOracle(identifier, out layout) ||
|
||||
TryResolvePhoton(identifier, out layout) ||
|
||||
TryResolveRocky(identifier, out layout) ||
|
||||
TryResolveSuse(identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveByProvenance(advisory, identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
return new Layout(new[] { "misc" }, CreateFileName(identifier));
|
||||
}
|
||||
|
||||
private static bool TryResolveCve(string identifier, out Layout layout)
|
||||
{
|
||||
var match = CvePattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
var year = match.Groups["year"].Value;
|
||||
layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout)
|
||||
{
|
||||
if (!GhsaPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath))
|
||||
{
|
||||
layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveUsn(string identifier, out Layout layout)
|
||||
{
|
||||
if (!UsnPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveDebian(string identifier, out Layout layout)
|
||||
{
|
||||
var match = DebianPattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveRedHat(string identifier, out Layout layout)
|
||||
{
|
||||
if (!RedHatPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveAmazon(string identifier, out Layout layout)
|
||||
{
|
||||
var match = AmazonPattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
var channel = match.Groups["channel"].Value;
|
||||
var subdirectory = channel switch
|
||||
{
|
||||
"2" => "2",
|
||||
"2023" => "2023",
|
||||
"2022" => "2022",
|
||||
_ => "1",
|
||||
};
|
||||
|
||||
layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveOracle(string identifier, out Layout layout)
|
||||
{
|
||||
if (!OraclePattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolvePhoton(string identifier, out Layout layout)
|
||||
{
|
||||
if (!PhotonPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveRocky(string identifier, out Layout layout)
|
||||
{
|
||||
if (!RockyPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveSuse(string identifier, out Layout layout)
|
||||
{
|
||||
if (!SusePattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout)
|
||||
{
|
||||
foreach (var source in EnumerateDistinctProvenanceSources(advisory))
|
||||
{
|
||||
if (SourceDirectoryMap.TryGetValue(source, out var segments))
|
||||
{
|
||||
layout = new Layout(segments, CreateFileName(identifier));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath)
|
||||
{
|
||||
foreach (var package in advisory.AffectedPackages)
|
||||
{
|
||||
if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (GhsaEcosystemMap.TryGetValue(type, out var mapped))
|
||||
{
|
||||
ecosystem = mapped;
|
||||
}
|
||||
else
|
||||
{
|
||||
ecosystem = type.ToLowerInvariant();
|
||||
}
|
||||
|
||||
packagePath = encodedPath;
|
||||
return true;
|
||||
}
|
||||
|
||||
ecosystem = "advisories";
|
||||
packagePath = "_";
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath)
|
||||
{
|
||||
type = string.Empty;
|
||||
encodedPath = string.Empty;
|
||||
|
||||
if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty
|
||||
? new[] { packageUrl.Name }
|
||||
: packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray();
|
||||
|
||||
type = packageUrl.Type;
|
||||
encodedPath = string.Join("%2F", segments);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateFileName(string identifier, bool uppercase = false)
|
||||
{
|
||||
var candidate = uppercase ? identifier.ToUpperInvariant() : identifier;
|
||||
return $"{SanitizeFileName(candidate)}.json";
|
||||
}
|
||||
|
||||
private static IEnumerable<string> EnumerateDistinctProvenanceSources(Advisory advisory)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var source in advisory.Provenance)
|
||||
{
|
||||
if (TryAddSource(source.Source))
|
||||
{
|
||||
yield return source.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var reference in advisory.References)
|
||||
{
|
||||
if (TryAddSource(reference.Provenance.Source))
|
||||
{
|
||||
yield return reference.Provenance.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var package in advisory.AffectedPackages)
|
||||
{
|
||||
foreach (var source in package.Provenance)
|
||||
{
|
||||
if (TryAddSource(source.Source))
|
||||
{
|
||||
yield return source.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var range in package.VersionRanges)
|
||||
{
|
||||
if (TryAddSource(range.Provenance.Source))
|
||||
{
|
||||
yield return range.Provenance.Source;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var metric in advisory.CvssMetrics)
|
||||
{
|
||||
if (TryAddSource(metric.Provenance.Source))
|
||||
{
|
||||
yield return metric.Provenance.Source;
|
||||
}
|
||||
}
|
||||
|
||||
bool TryAddSource(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return seen.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private static string SelectPreferredIdentifier(Advisory advisory)
|
||||
{
|
||||
if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred))
|
||||
{
|
||||
return preferred;
|
||||
}
|
||||
|
||||
foreach (var alias in advisory.Aliases)
|
||||
{
|
||||
if (TrySelectIdentifier(alias, out preferred))
|
||||
{
|
||||
return preferred;
|
||||
}
|
||||
}
|
||||
|
||||
return advisory.AdvisoryKey.Trim();
|
||||
}
|
||||
|
||||
private static bool TrySelectIdentifier(string value, out string identifier)
|
||||
{
|
||||
identifier = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed))
|
||||
{
|
||||
identifier = trimmed;
|
||||
return true;
|
||||
}
|
||||
|
||||
identifier = trimmed;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string name)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
Span<char> buffer = stackalloc char[name.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in name)
|
||||
{
|
||||
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
|
||||
{
|
||||
buffer[count++] = '_';
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer[count++] = ch;
|
||||
}
|
||||
}
|
||||
|
||||
var sanitized = new string(buffer[..count]).Trim();
|
||||
return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized;
|
||||
}
|
||||
|
||||
private readonly record struct Layout(string[] Segments, string FileName);
|
||||
}
|
||||
Reference in New Issue
Block a user