Resolve Concelier/Excititor merge conflicts

This commit is contained in:
master
2025-10-20 14:19:25 +03:00
2687 changed files with 212646 additions and 85913 deletions

View File

@@ -0,0 +1,28 @@
# AGENTS
## Role
Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving.
## Scope
- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable).
- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601.
- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites.
- Packaging: output directory under exports/json/<timestamp> with reproducible naming; optionally symlink latest.
- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads.
## Participants
- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests.
- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job.
- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured.
## Interfaces & contracts
- Job kind: export:json (JsonExportJob).
- Determinism: same inputs -> identical file bytes; hash snapshot persisted.
- Provenance: include minimal provenance fields when helpful; keep identity stable.
## In/Out of scope
In: JSON rendering and layout; incremental/deterministic writes.
Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter).
## Observability & security expectations
- Metrics: export.json.records, bytes, duration, delta.changed.
- Logs: target path, record counts, digest; no sensitive data.
## Tests
- Author and review coverage in `../StellaOps.Concelier.Exporter.Json.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.

View File

@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Concelier.Exporter.Json;
public static class ExportDigestCalculator
{
public static string ComputeTreeDigest(JsonExportResult result)
{
ArgumentNullException.ThrowIfNull(result);
using var sha256 = SHA256.Create();
var buffer = new byte[128 * 1024];
foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal))
{
var normalized = file.Replace("\\", "/");
var pathBytes = Encoding.UTF8.GetBytes(normalized);
_ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0);
var fullPath = ResolveFullPath(result.ExportDirectory, normalized);
using var stream = File.OpenRead(fullPath);
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
{
_ = sha256.TransformBlock(buffer, 0, read, null, 0);
}
}
_ = sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
var hash = sha256.Hash ?? Array.Empty<byte>();
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private static string ResolveFullPath(string root, string normalizedRelativePath)
{
var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
var parts = new string[segments.Length + 1];
parts[0] = root;
for (var i = 0; i < segments.Length; i++)
{
parts[i + 1] = segments[i];
}
return Path.Combine(parts);
}
}

View File

@@ -0,0 +1,28 @@
using System;
using System.Reflection;
namespace StellaOps.Concelier.Exporter.Json;
public static class ExporterVersion
{
public static string GetVersion(Type anchor)
{
ArgumentNullException.ThrowIfNull(anchor);
var assembly = anchor.Assembly;
var informational = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
if (!string.IsNullOrWhiteSpace(informational))
{
return informational;
}
var fileVersion = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version;
if (!string.IsNullOrWhiteSpace(fileVersion))
{
return fileVersion!;
}
var version = assembly.GetName().Version;
return version?.ToString() ?? "0.0.0";
}
}

View File

@@ -0,0 +1,12 @@
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
public interface IJsonExportPathResolver
{
/// <summary>
/// Returns the relative path (using platform directory separators) for the supplied advisory.
/// Path must not include the leading export root.
/// </summary>
string GetRelativePath(Advisory advisory);
}

View File

@@ -0,0 +1,37 @@
using System;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Metadata describing a single file produced by the JSON exporter.
/// </summary>
public sealed class JsonExportFile
{
public JsonExportFile(string relativePath, long length, string digest)
{
RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath));
if (relativePath.Length == 0)
{
throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Digest = digest ?? throw new ArgumentNullException(nameof(digest));
if (digest.Length == 0)
{
throw new ArgumentException("Digest cannot be empty.", nameof(digest));
}
Length = length;
}
public string RelativePath { get; }
public long Length { get; }
public string Digest { get; }
}

View File

@@ -0,0 +1,30 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportJob : IJob
{
public const string JobKind = "export:json";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5);
private readonly JsonFeedExporter _exporter;
private readonly ILogger<JsonExportJob> _logger;
public JsonExportJob(JsonFeedExporter exporter, ILogger<JsonExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing JSON export job {RunId}", context.RunId);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Completed JSON export job {RunId}", context.RunId);
}
}

View File

@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.Json;
internal static class JsonExportManifestWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = true,
};
public static async Task WriteAsync(
JsonExportResult result,
string digest,
string exporterVersion,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(result);
ArgumentException.ThrowIfNullOrEmpty(digest);
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
var exportId = Path.GetFileName(result.ExportDirectory);
var files = result.Files
.Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest))
.ToArray();
var manifest = new JsonExportManifest(
exportId,
result.ExportedAt.UtcDateTime,
digest,
result.AdvisoryCount,
result.TotalBytes,
files.Length,
files,
exporterVersion);
var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions);
var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json");
await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime);
}
private sealed record JsonExportManifest(
[property: JsonPropertyOrder(1)] string ExportId,
[property: JsonPropertyOrder(2)] DateTime GeneratedAt,
[property: JsonPropertyOrder(3)] string Digest,
[property: JsonPropertyOrder(4)] int AdvisoryCount,
[property: JsonPropertyOrder(5)] long TotalBytes,
[property: JsonPropertyOrder(6)] int FileCount,
[property: JsonPropertyOrder(7)] IReadOnlyList<JsonExportManifestFile> Files,
[property: JsonPropertyOrder(8)] string ExporterVersion);
private sealed record JsonExportManifestFile(
[property: JsonPropertyOrder(1)] string Path,
[property: JsonPropertyOrder(2)] long Bytes,
[property: JsonPropertyOrder(3)] string Digest);
}

View File

@@ -0,0 +1,115 @@
using System.Collections.Generic;
using System.IO;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Configuration for JSON exporter output paths and determinism controls.
/// </summary>
public sealed class JsonExportOptions
{
/// <summary>
/// Root directory where exports are written. Default "exports/json".
/// </summary>
public string OutputRoot { get; set; } = Path.Combine("exports", "json");
/// <summary>
/// Format string applied to the export timestamp to produce the directory name.
/// </summary>
public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
/// <summary>
/// Optional static name for the symlink (or directory junction) pointing at the most recent export.
/// </summary>
public string LatestSymlinkName { get; set; } = "latest";
/// <summary>
/// When true, attempts to re-point <see cref="LatestSymlinkName"/> after a successful export.
/// </summary>
public bool MaintainLatestSymlink { get; set; } = true;
/// <summary>
/// Optional repository identifier recorded alongside export state metadata.
/// </summary>
public string? TargetRepository { get; set; }
/// <summary>
/// Mirror distribution configuration producing aggregate bundles for downstream mirrors.
/// </summary>
public JsonMirrorOptions Mirror { get; set; } = new();
public sealed class JsonMirrorOptions
{
/// <summary>
/// Indicates whether mirror bundle generation is enabled.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Directory name (relative to the export root) where mirror artefacts are written.
/// </summary>
public string DirectoryName { get; set; } = "mirror";
/// <summary>
/// Domains exposed to downstream mirrors.
/// </summary>
public IList<JsonMirrorDomainOptions> Domains { get; } = new List<JsonMirrorDomainOptions>();
/// <summary>
/// Signing configuration for mirror bundles.
/// </summary>
public JsonMirrorSigningOptions Signing { get; set; } = new();
}
public sealed class JsonMirrorDomainOptions
{
/// <summary>
/// Stable identifier for the mirror domain (used in URLs and directory names).
/// </summary>
public string Id { get; set; } = string.Empty;
/// <summary>
/// Optional human-readable label for UI surfaces.
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Optional advisory scheme filters (e.g. CVE, GHSA). Empty collection selects all schemes.
/// </summary>
public IList<string> IncludeSchemes { get; } = new List<string>();
/// <summary>
/// Optional provenance source filters (e.g. nvd, ghsa). Empty collection selects all sources.
/// </summary>
public IList<string> IncludeSources { get; } = new List<string>();
}
public sealed class JsonMirrorSigningOptions
{
/// <summary>
/// Indicates whether bundles should be signed. Defaults to disabled.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Signing algorithm identifier (defaults to ES256).
/// </summary>
public string Algorithm { get; set; } = SignatureAlgorithms.Es256;
/// <summary>
/// Active signing key identifier.
/// </summary>
public string KeyId { get; set; } = string.Empty;
/// <summary>
/// Path to the private key (PEM) used for signing mirror bundles.
/// </summary>
public string KeyPath { get; set; } = string.Empty;
/// <summary>
/// Optional crypto provider hint. When omitted the registry resolves an appropriate provider.
/// </summary>
public string? Provider { get; set; }
}
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportResult
{
public JsonExportResult(
string exportDirectory,
DateTimeOffset exportedAt,
IEnumerable<JsonExportFile> files,
int advisoryCount,
long totalBytes,
IEnumerable<Advisory>? advisories = null)
{
if (string.IsNullOrWhiteSpace(exportDirectory))
{
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
}
var list = (files ?? throw new ArgumentNullException(nameof(files)))
.Where(static file => file is not null)
.ToImmutableArray();
var advisoryList = (advisories ?? Array.Empty<Advisory>())
.Where(static advisory => advisory is not null)
.ToImmutableArray();
ExportDirectory = exportDirectory;
ExportedAt = exportedAt;
TotalBytes = totalBytes;
Files = list;
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
Advisories = advisoryList;
AdvisoryCount = advisoryList.IsDefaultOrEmpty ? advisoryCount : advisoryList.Length;
}
public string ExportDirectory { get; }
public DateTimeOffset ExportedAt { get; }
public ImmutableArray<JsonExportFile> Files { get; }
public ImmutableArray<string> FilePaths { get; }
public ImmutableArray<Advisory> Advisories { get; }
public int AdvisoryCount { get; }
public long TotalBytes { get; }
}

View File

@@ -0,0 +1,241 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering.
/// </summary>
public sealed class JsonExportSnapshotBuilder
{
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
}
public Task<JsonExportResult> WriteAsync(
IReadOnlyCollection<Advisory> advisories,
DateTimeOffset exportedAt,
string? exportName = null,
CancellationToken cancellationToken = default)
{
if (advisories is null)
{
throw new ArgumentNullException(nameof(advisories));
}
return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken);
}
public async Task<JsonExportResult> WriteAsync(
IAsyncEnumerable<Advisory> advisories,
DateTimeOffset exportedAt,
string? exportName = null,
CancellationToken cancellationToken = default)
{
if (advisories is null)
{
throw new ArgumentNullException(nameof(advisories));
}
var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
if (string.IsNullOrWhiteSpace(exportDirectoryName))
{
throw new InvalidOperationException("Export directory name resolved to an empty string.");
}
var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot));
TrySetDirectoryTimestamp(exportRoot, exportedAt);
var exportDirectory = Path.Combine(exportRoot, exportDirectoryName);
if (Directory.Exists(exportDirectory))
{
Directory.Delete(exportDirectory, recursive: true);
}
Directory.CreateDirectory(exportDirectory);
TrySetDirectoryTimestamp(exportDirectory, exportedAt);
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var files = new List<JsonExportFile>();
var advisoryList = new List<Advisory>();
long totalBytes = 0L;
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = Resolve(advisory);
if (!seen.Add(entry.RelativePath))
{
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
}
advisoryList.Add(entry.Advisory);
var destination = Combine(exportDirectory, entry.Segments);
var destinationDirectory = Path.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory))
{
EnsureDirectoryExists(destinationDirectory);
TrySetDirectoryTimestamp(destinationDirectory, exportedAt);
}
var payload = SnapshotSerializer.ToSnapshot(entry.Advisory);
var bytes = Utf8NoBom.GetBytes(payload);
await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime);
var digest = ComputeDigest(bytes);
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
totalBytes += bytes.LongLength;
}
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryList.Count, totalBytes, advisoryList);
}
private static async IAsyncEnumerable<Advisory> EnumerateAsync(
IEnumerable<Advisory> advisories,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
foreach (var advisory in advisories)
{
cancellationToken.ThrowIfCancellationRequested();
yield return advisory;
await Task.Yield();
}
}
private static string EnsureDirectoryExists(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
{
throw new ArgumentException("Directory path must be provided.", nameof(directory));
}
Directory.CreateDirectory(directory);
return directory;
}
private static string Combine(string root, IReadOnlyList<string> segments)
{
var parts = new string[segments.Count + 1];
parts[0] = root;
for (var i = 0; i < segments.Count; i++)
{
parts[i + 1] = segments[i];
}
return Path.Combine(parts);
}
private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp)
{
try
{
Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime);
}
catch (IOException)
{
// Ignore failure to set timestamps; not critical for content determinism.
}
catch (UnauthorizedAccessException)
{
// Ignore permission issues when setting timestamps.
}
catch (PlatformNotSupportedException)
{
// Some platforms may not support this operation.
}
}
private PathResolution Resolve(Advisory advisory)
{
if (advisory is null)
{
throw new ArgumentNullException(nameof(advisory));
}
var normalized = CanonicalJsonSerializer.Normalize(advisory);
var relativePath = _pathResolver.GetRelativePath(normalized);
var segments = NormalizeRelativePath(relativePath);
var normalizedPath = string.Join('/', segments);
return new PathResolution(normalized, normalizedPath, segments);
}
private static string[] NormalizeRelativePath(string relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
throw new InvalidOperationException("Path resolver returned an empty path.");
}
if (Path.IsPathRooted(relativePath))
{
throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported.");
}
var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
if (pieces.Length == 0)
{
throw new InvalidOperationException("Path resolver produced no path segments.");
}
var sanitized = new string[pieces.Length];
for (var i = 0; i < pieces.Length; i++)
{
var segment = pieces[i];
if (segment == "." || segment == "..")
{
throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments.");
}
sanitized[i] = SanitizeSegment(segment);
}
return sanitized;
}
private static string SanitizeSegment(string segment)
{
var invalid = Path.GetInvalidFileNameChars();
Span<char> buffer = stackalloc char[segment.Length];
var count = 0;
foreach (var ch in segment)
{
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
{
buffer[count++] = '_';
}
else
{
buffer[count++] = ch;
}
}
var sanitized = new string(buffer[..count]).Trim();
return string.IsNullOrEmpty(sanitized) ? "_" : sanitized;
}
private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList<string> Segments);
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
}

View File

@@ -0,0 +1,64 @@
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:json";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<JsonExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
if (string.IsNullOrWhiteSpace(options.OutputRoot))
{
options.OutputRoot = Path.Combine("exports", "json");
}
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
{
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
}
if (string.IsNullOrWhiteSpace(options.Mirror.DirectoryName))
{
options.Mirror.DirectoryName = "mirror";
}
});
services.AddSingleton<JsonFeedExporter>();
services.AddTransient<JsonExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(JsonExportJob.JobKind))
{
options.Definitions[JsonExportJob.JobKind] = new JobDefinition(
JsonExportJob.JobKind,
typeof(JsonExportJob),
JsonExportJob.DefaultTimeout,
JsonExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
}

View File

@@ -0,0 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExporterPlugin : IExporterPlugin
{
public string Name => JsonFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<JsonFeedExporter>(services);
}
}

View File

@@ -0,0 +1,205 @@
using System;
using System.Globalization;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.Events;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonFeedExporter : IFeedExporter
{
public const string ExporterName = "json";
public const string ExporterId = "export:json";
private readonly IAdvisoryStore _advisoryStore;
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
private readonly ExportStateManager _stateManager;
private readonly ILogger<JsonFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
private readonly IAdvisoryEventLog _eventLog;
public JsonFeedExporter(
IAdvisoryStore advisoryStore,
IOptions<JsonExportOptions> options,
IJsonExportPathResolver pathResolver,
ExportStateManager stateManager,
IAdvisoryEventLog eventLog,
ILogger<JsonFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
var exportRoot = Path.GetFullPath(_options.OutputRoot);
_logger.LogInformation("Starting JSON export {ExportId}", exportId);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false);
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
_logger.LogInformation(
"JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
exportId,
result.Files.Length,
result.TotalBytes,
result.AdvisoryCount,
digest);
var manifest = result.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
if (existingState is not null
&& existingState.Files.Count > 0
&& string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal))
{
_logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId);
TryDeleteDirectory(result.ExportDirectory);
return;
}
var resetBaseline = existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
digest,
cursor: digest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);
if (_options.MaintainLatestSymlink)
{
TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
}
}
private async Task<IReadOnlyList<Advisory>> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken)
{
var keys = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
{
keys.Add(advisory.AdvisoryKey.Trim());
}
}
var advisories = new List<Advisory>(keys.Count);
foreach (var key in keys)
{
cancellationToken.ThrowIfCancellationRequested();
var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false);
if (!replay.Statements.IsDefaultOrEmpty)
{
advisories.Add(replay.Statements[0].Advisory);
}
}
return advisories;
}
private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
{
if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName))
{
return;
}
var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName);
try
{
if (Directory.Exists(latestPath) || File.Exists(latestPath))
{
TryRemoveExistingPointer(latestPath);
}
Directory.CreateSymbolicLink(latestPath, exportDirectory);
_logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException)
{
_logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath);
}
}
private void TryRemoveExistingPointer(string latestPath)
{
try
{
var attributes = File.GetAttributes(latestPath);
if (attributes.HasFlag(FileAttributes.Directory))
{
Directory.Delete(latestPath, recursive: false);
}
else
{
File.Delete(latestPath);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath);
}
}
private void TryDeleteDirectory(string path)
{
try
{
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path);
}
}
}

View File

@@ -0,0 +1,622 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Models;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Exporter.Json;
internal static class JsonMirrorBundleWriter
{
private const int SchemaVersion = 1;
private const string BundleFileName = "bundle.json";
private const string BundleSignatureFileName = "bundle.json.jws";
private const string ManifestFileName = "manifest.json";
private const string IndexFileName = "index.json";
private const string SignatureMediaType = "application/vnd.stellaops.concelier.mirror-bundle+jws";
private const string DefaultMirrorDirectoryName = "mirror";
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = null,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public static async Task<JsonExportResult> WriteAsync(
JsonExportResult result,
JsonExportOptions options,
IServiceProvider services,
TimeProvider timeProvider,
ILogger logger,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(result);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(logger);
var mirrorOptions = options.Mirror ?? new JsonExportOptions.JsonMirrorOptions();
if (!mirrorOptions.Enabled || mirrorOptions.Domains.Count == 0)
{
return result;
}
cancellationToken.ThrowIfCancellationRequested();
var exportedAtUtc = result.ExportedAt.UtcDateTime;
var mirrorDirectoryName = string.IsNullOrWhiteSpace(mirrorOptions.DirectoryName)
? DefaultMirrorDirectoryName
: mirrorOptions.DirectoryName.Trim();
var mirrorRoot = Path.Combine(result.ExportDirectory, mirrorDirectoryName);
Directory.CreateDirectory(mirrorRoot);
TrySetDirectoryTimestamp(mirrorRoot, exportedAtUtc);
var advisories = result.Advisories.IsDefaultOrEmpty
? Array.Empty<Advisory>()
: result.Advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
var signingContext = PrepareSigningContext(mirrorOptions.Signing, services, timeProvider, logger);
var additionalFiles = new List<JsonExportFile>();
var domainEntries = new List<MirrorIndexDomainEntry>();
foreach (var domainOption in mirrorOptions.Domains)
{
cancellationToken.ThrowIfCancellationRequested();
if (domainOption is null)
{
logger.LogWarning("Encountered null mirror domain configuration; skipping.");
continue;
}
var domainId = (domainOption.Id ?? string.Empty).Trim();
if (domainId.Length == 0)
{
logger.LogWarning("Skipping mirror domain with empty id.");
continue;
}
var schemeFilter = CreateFilterSet(domainOption.IncludeSchemes);
var sourceFilter = CreateFilterSet(domainOption.IncludeSources);
var domainAdvisories = advisories
.Where(advisory => MatchesFilters(advisory, schemeFilter, sourceFilter))
.ToArray();
var sources = BuildSourceSummaries(domainAdvisories);
var domainDisplayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
? domainId
: domainOption.DisplayName!.Trim();
var domainDirectory = Path.Combine(mirrorRoot, domainId);
Directory.CreateDirectory(domainDirectory);
TrySetDirectoryTimestamp(domainDirectory, exportedAtUtc);
var bundleDocument = new MirrorDomainBundleDocument(
SchemaVersion,
result.ExportedAt,
options.TargetRepository,
domainId,
domainDisplayName,
domainAdvisories.Length,
domainAdvisories,
sources);
var bundleBytes = Serialize(bundleDocument);
var bundlePath = Path.Combine(domainDirectory, BundleFileName);
await WriteFileAsync(bundlePath, bundleBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var bundleRelativePath = ToRelativePath(result.ExportDirectory, bundlePath);
var bundleDigest = ComputeDigest(bundleBytes);
var bundleLength = (long)bundleBytes.LongLength;
additionalFiles.Add(new JsonExportFile(bundleRelativePath, bundleLength, bundleDigest));
MirrorSignatureDescriptor? signatureDescriptor = null;
if (signingContext is not null)
{
var (signatureValue, signedAt) = await CreateSignatureAsync(
signingContext,
bundleBytes,
timeProvider,
cancellationToken)
.ConfigureAwait(false);
var signatureBytes = Utf8NoBom.GetBytes(signatureValue);
var signaturePath = Path.Combine(domainDirectory, BundleSignatureFileName);
await WriteFileAsync(signaturePath, signatureBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var signatureRelativePath = ToRelativePath(result.ExportDirectory, signaturePath);
var signatureDigest = ComputeDigest(signatureBytes);
var signatureLength = (long)signatureBytes.LongLength;
additionalFiles.Add(new JsonExportFile(signatureRelativePath, signatureLength, signatureDigest));
signatureDescriptor = new MirrorSignatureDescriptor(
signatureRelativePath,
signingContext.Algorithm,
signingContext.KeyId,
signingContext.Provider,
signedAt);
}
var bundleDescriptor = new MirrorFileDescriptor(bundleRelativePath, bundleLength, bundleDigest, signatureDescriptor);
var manifestDocument = new MirrorDomainManifestDocument(
SchemaVersion,
result.ExportedAt,
domainId,
domainDisplayName,
domainAdvisories.Length,
sources,
bundleDescriptor);
var manifestBytes = Serialize(manifestDocument);
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
await WriteFileAsync(manifestPath, manifestBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var manifestRelativePath = ToRelativePath(result.ExportDirectory, manifestPath);
var manifestDigest = ComputeDigest(manifestBytes);
var manifestLength = (long)manifestBytes.LongLength;
additionalFiles.Add(new JsonExportFile(manifestRelativePath, manifestLength, manifestDigest));
var manifestDescriptor = new MirrorFileDescriptor(manifestRelativePath, manifestLength, manifestDigest, null);
domainEntries.Add(new MirrorIndexDomainEntry(
domainId,
domainDisplayName,
domainAdvisories.Length,
manifestDescriptor,
bundleDescriptor,
sources));
}
domainEntries.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
var indexDocument = new MirrorIndexDocument(
SchemaVersion,
result.ExportedAt,
options.TargetRepository,
domainEntries);
var indexBytes = Serialize(indexDocument);
var indexPath = Path.Combine(mirrorRoot, IndexFileName);
await WriteFileAsync(indexPath, indexBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var indexRelativePath = ToRelativePath(result.ExportDirectory, indexPath);
var indexDigest = ComputeDigest(indexBytes);
var indexLength = (long)indexBytes.LongLength;
additionalFiles.Add(new JsonExportFile(indexRelativePath, indexLength, indexDigest));
logger.LogInformation(
"Generated {DomainCount} Concelier mirror domain bundle(s) under {MirrorRoot}.",
domainEntries.Count,
mirrorDirectoryName);
var combinedFiles = new List<JsonExportFile>(result.Files.Length + additionalFiles.Count);
combinedFiles.AddRange(result.Files);
combinedFiles.AddRange(additionalFiles);
var combinedTotalBytes = checked(result.TotalBytes + additionalFiles.Sum(static file => file.Length));
return new JsonExportResult(
result.ExportDirectory,
result.ExportedAt,
combinedFiles,
result.AdvisoryCount,
combinedTotalBytes,
result.Advisories);
}
private static JsonMirrorSigningContext? PrepareSigningContext(
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
IServiceProvider services,
TimeProvider timeProvider,
ILogger logger)
{
if (signingOptions is null || !signingOptions.Enabled)
{
return null;
}
var algorithm = string.IsNullOrWhiteSpace(signingOptions.Algorithm)
? SignatureAlgorithms.Es256
: signingOptions.Algorithm.Trim();
var keyId = (signingOptions.KeyId ?? string.Empty).Trim();
if (keyId.Length == 0)
{
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyId to be configured.");
}
var registry = services.GetService<ICryptoProviderRegistry>()
?? throw new InvalidOperationException("Mirror signing requires ICryptoProviderRegistry to be registered.");
var providerHint = signingOptions.Provider?.Trim();
var keyReference = new CryptoKeyReference(keyId, providerHint);
CryptoSignerResolution resolved;
try
{
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, providerHint);
}
catch (KeyNotFoundException)
{
var provider = ResolveProvider(registry, algorithm, providerHint);
var signingKey = LoadSigningKey(signingOptions, provider, services, timeProvider, algorithm);
provider.UpsertSigningKey(signingKey);
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, provider.Name);
}
logger.LogDebug(
"Mirror signing configured with key {KeyId} via provider {Provider} using {Algorithm}.",
resolved.Signer.KeyId,
resolved.ProviderName,
algorithm);
return new JsonMirrorSigningContext(resolved.Signer, algorithm, resolved.Signer.KeyId, resolved.ProviderName);
}
private static ICryptoProvider ResolveProvider(ICryptoProviderRegistry registry, string algorithm, string? providerHint)
{
if (!string.IsNullOrWhiteSpace(providerHint) && registry.TryResolve(providerHint, out var hinted))
{
if (!hinted.Supports(CryptoCapability.Signing, algorithm))
{
throw new InvalidOperationException(
$"Crypto provider '{providerHint}' does not support signing algorithm '{algorithm}'.");
}
return hinted;
}
return registry.ResolveOrThrow(CryptoCapability.Signing, algorithm);
}
private static CryptoSigningKey LoadSigningKey(
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
ICryptoProvider provider,
IServiceProvider services,
TimeProvider timeProvider,
string algorithm)
{
var keyPath = (signingOptions.KeyPath ?? string.Empty).Trim();
if (keyPath.Length == 0)
{
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyPath to be configured.");
}
var environment = services.GetService<IHostEnvironment>();
var basePath = environment?.ContentRootPath ?? AppContext.BaseDirectory;
var resolvedPath = Path.IsPathRooted(keyPath)
? keyPath
: Path.GetFullPath(Path.Combine(basePath, keyPath));
if (!File.Exists(resolvedPath))
{
throw new FileNotFoundException($"Mirror signing key '{signingOptions.KeyId}' not found.", resolvedPath);
}
var pem = File.ReadAllText(resolvedPath);
using var ecdsa = ECDsa.Create();
try
{
ecdsa.ImportFromPem(pem);
}
catch (CryptographicException ex)
{
throw new InvalidOperationException("Failed to import mirror signing key. Ensure the PEM contains an EC private key.", ex);
}
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
return new CryptoSigningKey(
new CryptoKeyReference(signingOptions.KeyId, provider.Name),
algorithm,
in parameters,
timeProvider.GetUtcNow());
}
private static async Task<(string Value, DateTimeOffset SignedAt)> CreateSignatureAsync(
JsonMirrorSigningContext context,
ReadOnlyMemory<byte> payload,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var header = new Dictionary<string, object>
{
["alg"] = context.Algorithm,
["kid"] = context.KeyId,
["typ"] = SignatureMediaType,
["b64"] = false,
["crit"] = new[] { "b64" }
};
if (!string.IsNullOrWhiteSpace(context.Provider))
{
header["provider"] = context.Provider;
}
var headerJson = JsonSerializer.Serialize(header, HeaderSerializerOptions);
var protectedHeader = Base64UrlEncode(Utf8NoBom.GetBytes(headerJson));
var signingInputLength = protectedHeader.Length + 1 + payload.Length;
var buffer = ArrayPool<byte>.Shared.Rent(signingInputLength);
try
{
var headerBytes = Encoding.ASCII.GetBytes(protectedHeader);
Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length);
buffer[headerBytes.Length] = (byte)'.';
var payloadArray = payload.ToArray();
Buffer.BlockCopy(payloadArray, 0, buffer, headerBytes.Length + 1, payloadArray.Length);
var signingInput = new ReadOnlyMemory<byte>(buffer, 0, signingInputLength);
var signatureBytes = await context.Signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false);
var encodedSignature = Base64UrlEncode(signatureBytes);
var signedAt = timeProvider.GetUtcNow();
return (string.Concat(protectedHeader, "..", encodedSignature), signedAt);
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
private static IReadOnlyList<JsonMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
{
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provenance in advisory.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
var source = provenance.Source.Trim();
if (!builders.TryGetValue(source, out var accumulator))
{
accumulator = new SourceAccumulator();
builders[source] = accumulator;
}
accumulator.Record(provenance.RecordedAt);
if (counted.Add(source))
{
accumulator.IncrementAdvisoryCount();
}
}
}
return builders
.OrderBy(static pair => pair.Key, StringComparer.Ordinal)
.Select(pair => new JsonMirrorSourceSummary(
pair.Key,
pair.Value.FirstRecordedAt,
pair.Value.LastRecordedAt,
pair.Value.AdvisoryCount))
.ToArray();
}
private static HashSet<string>? CreateFilterSet(IList<string>? values)
{
if (values is null || values.Count == 0)
{
return null;
}
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
set.Add(value.Trim());
}
return set.Count == 0 ? null : set;
}
private static bool MatchesFilters(Advisory advisory, HashSet<string>? schemeFilter, HashSet<string>? sourceFilter)
{
if (schemeFilter is not null)
{
var scheme = ExtractScheme(advisory.AdvisoryKey);
if (!schemeFilter.Contains(scheme))
{
return false;
}
}
if (sourceFilter is not null)
{
var hasSource = advisory.Provenance.Any(provenance =>
!string.IsNullOrWhiteSpace(provenance.Source) &&
sourceFilter.Contains(provenance.Source.Trim()));
if (!hasSource)
{
return false;
}
}
return true;
}
private static string ExtractScheme(string advisoryKey)
{
if (string.IsNullOrWhiteSpace(advisoryKey))
{
return string.Empty;
}
var trimmed = advisoryKey.Trim();
var separatorIndex = trimmed.IndexOf(':');
return separatorIndex <= 0 ? trimmed : trimmed[..separatorIndex];
}
private static byte[] Serialize<T>(T value)
{
var json = CanonicalJsonSerializer.SerializeIndented(value);
return Utf8NoBom.GetBytes(json);
}
private static async Task WriteFileAsync(string path, byte[] content, DateTime exportedAtUtc, CancellationToken cancellationToken)
{
await File.WriteAllBytesAsync(path, content, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, exportedAtUtc);
}
private static string ToRelativePath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static void TrySetDirectoryTimestamp(string directory, DateTime exportedAtUtc)
{
try
{
Directory.SetLastWriteTimeUtc(directory, exportedAtUtc);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
catch (PlatformNotSupportedException)
{
}
}
private static string Base64UrlEncode(ReadOnlySpan<byte> value)
{
var encoded = Convert.ToBase64String(value);
var builder = new StringBuilder(encoded.Length);
foreach (var ch in encoded)
{
switch (ch)
{
case '+':
builder.Append('-');
break;
case '/':
builder.Append('_');
break;
case '=':
break;
default:
builder.Append(ch);
break;
}
}
return builder.ToString();
}
private sealed record JsonMirrorSigningContext(ICryptoSigner Signer, string Algorithm, string KeyId, string Provider);
private sealed record MirrorIndexDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string? TargetRepository,
IReadOnlyList<MirrorIndexDomainEntry> Domains);
private sealed record MirrorIndexDomainEntry(
string DomainId,
string DisplayName,
int AdvisoryCount,
MirrorFileDescriptor Manifest,
MirrorFileDescriptor Bundle,
IReadOnlyList<JsonMirrorSourceSummary> Sources);
private sealed record MirrorDomainManifestDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string DomainId,
string DisplayName,
int AdvisoryCount,
IReadOnlyList<JsonMirrorSourceSummary> Sources,
MirrorFileDescriptor Bundle);
private sealed record MirrorDomainBundleDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string? TargetRepository,
string DomainId,
string DisplayName,
int AdvisoryCount,
IReadOnlyList<Advisory> Advisories,
IReadOnlyList<JsonMirrorSourceSummary> Sources);
private sealed record MirrorFileDescriptor(
string Path,
long SizeBytes,
string Digest,
MirrorSignatureDescriptor? Signature);
private sealed record MirrorSignatureDescriptor(
string Path,
string Algorithm,
string KeyId,
string Provider,
DateTimeOffset SignedAt);
private sealed record JsonMirrorSourceSummary(
string Source,
DateTimeOffset? FirstRecordedAt,
DateTimeOffset? LastRecordedAt,
int AdvisoryCount);
private sealed class SourceAccumulator
{
public DateTimeOffset? FirstRecordedAt { get; private set; }
public DateTimeOffset? LastRecordedAt { get; private set; }
public int AdvisoryCount { get; private set; }
public void Record(DateTimeOffset recordedAt)
{
var normalized = recordedAt.ToUniversalTime();
if (FirstRecordedAt is null || normalized < FirstRecordedAt.Value)
{
FirstRecordedAt = normalized;
}
if (LastRecordedAt is null || normalized > LastRecordedAt.Value)
{
LastRecordedAt = normalized;
}
}
public void IncrementAdvisoryCount()
{
AdvisoryCount++;
}
}
}

View File

@@ -0,0 +1,24 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
<ProjectReference Include="..\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,13 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.|
|Deterministic serializer|BE-Export|Models|DONE Canonical serializer + snapshot builder emit stable JSON across runs.|
|ExportState read/write|BE-Export|Storage.Mongo|DONE `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.|
|JsonExportJob wiring|BE-Export|Core|DONE Job scheduler options now configurable via DI; JSON job registered with scheduler.|
|Snapshot tests for file tree|QA|Exporters|DONE Added resolver/exporter tests asserting tree layout and deterministic behavior.|
|Parity smoke vs upstream vuln-list|QA|Exporters|DONE `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.|
|Stream advisories during export|BE-Export|Storage.Mongo|DONE exporter + streaming-only test ensures single enumeration and per-file digest capture.|
|Emit export manifest with digest metadata|BE-Export|Exporters|DONE manifest now includes per-file digests/sizes alongside tree digest.|
|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.|
|CONCELIER-EXPORT-08-201 Mirror bundle + domain manifest|Team Concelier Export|FEEDCORE-ENGINE-07-001|DONE (2025-10-19) Mirror bundle writer emits domain aggregates + manifests with cosign-compatible JWS signatures; index/tests updated via `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj` (2025-10-19).|

View File

@@ -0,0 +1,456 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Path resolver approximating the directory layout used by aquasecurity/vuln-list.
/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks.
/// </summary>
public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver
{
private static readonly Regex CvePattern = new("^CVE-(?<year>\\d{4})-(?<id>\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex UsnPattern = new("^USN-(?<id>\\d+-\\d+)(?<suffix>[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex DebianPattern = new("^(?<prefix>DLA|DSA|ELA)-(?<id>\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex RedHatPattern = new("^RH(?<type>SA|BA|EA)-(?<rest>[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex AmazonPattern = new("^ALAS(?<channel>2|2022|2023)?-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex OraclePattern = new("^(?<kind>ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex PhotonPattern = new("^PHSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex RockyPattern = new("^RLSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex SusePattern = new("^SUSE-(?<kind>SU|RU|OU|SB)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Dictionary<string, string[]> SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase)
{
["nvd"] = new[] { "nvd" },
["ghsa"] = new[] { "ghsa" },
["github"] = new[] { "ghsa" },
["osv"] = new[] { "osv" },
["redhat"] = new[] { "redhat", "oval" },
["ubuntu"] = new[] { "ubuntu" },
["debian"] = new[] { "debian" },
["oracle"] = new[] { "oracle" },
["photon"] = new[] { "photon" },
["rocky"] = new[] { "rocky" },
["suse"] = new[] { "suse" },
["amazon"] = new[] { "amazon" },
["aws"] = new[] { "amazon" },
["alpine"] = new[] { "alpine" },
["wolfi"] = new[] { "wolfi" },
["chainguard"] = new[] { "chainguard" },
["cert-fr"] = new[] { "cert", "fr" },
["cert-in"] = new[] { "cert", "in" },
["cert-cc"] = new[] { "cert", "cc" },
["cert-bund"] = new[] { "cert", "bund" },
["acsc"] = new[] { "cert", "au" },
["cisa"] = new[] { "ics", "cisa" },
["ics-cisa"] = new[] { "ics", "cisa" },
["ics-kaspersky"] = new[] { "ics", "kaspersky" },
["kaspersky"] = new[] { "ics", "kaspersky" },
};
private static readonly Dictionary<string, string> GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase)
{
["go"] = "go",
["golang"] = "go",
["npm"] = "npm",
["maven"] = "maven",
["pypi"] = "pip",
["pip"] = "pip",
["nuget"] = "nuget",
["composer"] = "composer",
["packagist"] = "composer",
["rubygems"] = "rubygems",
["gem"] = "rubygems",
["swift"] = "swift",
["cargo"] = "cargo",
["hex"] = "hex",
["pub"] = "pub",
["github"] = "github",
["docker"] = "container",
};
public string GetRelativePath(Advisory advisory)
{
if (advisory is null)
{
throw new ArgumentNullException(nameof(advisory));
}
var identifier = SelectPreferredIdentifier(advisory);
if (identifier.Length == 0)
{
throw new InvalidOperationException("Unable to derive identifier for advisory.");
}
var layout = ResolveLayout(advisory, identifier);
var segments = new string[layout.Segments.Length + 1];
for (var i = 0; i < layout.Segments.Length; i++)
{
segments[i] = layout.Segments[i];
}
segments[^1] = layout.FileName;
return Path.Combine(segments);
}
private static Layout ResolveLayout(Advisory advisory, string identifier)
{
if (TryResolveCve(identifier, out var layout))
{
return layout;
}
if (TryResolveGhsa(advisory, identifier, out layout))
{
return layout;
}
if (TryResolveUsn(identifier, out layout) ||
TryResolveDebian(identifier, out layout) ||
TryResolveRedHat(identifier, out layout) ||
TryResolveAmazon(identifier, out layout) ||
TryResolveOracle(identifier, out layout) ||
TryResolvePhoton(identifier, out layout) ||
TryResolveRocky(identifier, out layout) ||
TryResolveSuse(identifier, out layout))
{
return layout;
}
if (TryResolveByProvenance(advisory, identifier, out layout))
{
return layout;
}
return new Layout(new[] { "misc" }, CreateFileName(identifier));
}
private static bool TryResolveCve(string identifier, out Layout layout)
{
var match = CvePattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
var year = match.Groups["year"].Value;
layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout)
{
if (!GhsaPattern.IsMatch(identifier))
{
layout = default;
return false;
}
if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath))
{
layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true));
return true;
}
layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveUsn(string identifier, out Layout layout)
{
if (!UsnPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveDebian(string identifier, out Layout layout)
{
var match = DebianPattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveRedHat(string identifier, out Layout layout)
{
if (!RedHatPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveAmazon(string identifier, out Layout layout)
{
var match = AmazonPattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
var channel = match.Groups["channel"].Value;
var subdirectory = channel switch
{
"2" => "2",
"2023" => "2023",
"2022" => "2022",
_ => "1",
};
layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveOracle(string identifier, out Layout layout)
{
if (!OraclePattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolvePhoton(string identifier, out Layout layout)
{
if (!PhotonPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveRocky(string identifier, out Layout layout)
{
if (!RockyPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveSuse(string identifier, out Layout layout)
{
if (!SusePattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout)
{
foreach (var source in EnumerateDistinctProvenanceSources(advisory))
{
if (SourceDirectoryMap.TryGetValue(source, out var segments))
{
layout = new Layout(segments, CreateFileName(identifier));
return true;
}
}
layout = default;
return false;
}
private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath)
{
foreach (var package in advisory.AffectedPackages)
{
if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath))
{
continue;
}
if (GhsaEcosystemMap.TryGetValue(type, out var mapped))
{
ecosystem = mapped;
}
else
{
ecosystem = type.ToLowerInvariant();
}
packagePath = encodedPath;
return true;
}
ecosystem = "advisories";
packagePath = "_";
return false;
}
private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath)
{
type = string.Empty;
encodedPath = string.Empty;
if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl))
{
return false;
}
var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty
? new[] { packageUrl.Name }
: packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray();
type = packageUrl.Type;
encodedPath = string.Join("%2F", segments);
return true;
}
private static string CreateFileName(string identifier, bool uppercase = false)
{
var candidate = uppercase ? identifier.ToUpperInvariant() : identifier;
return $"{SanitizeFileName(candidate)}.json";
}
private static IEnumerable<string> EnumerateDistinctProvenanceSources(Advisory advisory)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var source in advisory.Provenance)
{
if (TryAddSource(source.Source))
{
yield return source.Source;
}
}
foreach (var reference in advisory.References)
{
if (TryAddSource(reference.Provenance.Source))
{
yield return reference.Provenance.Source;
}
}
foreach (var package in advisory.AffectedPackages)
{
foreach (var source in package.Provenance)
{
if (TryAddSource(source.Source))
{
yield return source.Source;
}
}
foreach (var range in package.VersionRanges)
{
if (TryAddSource(range.Provenance.Source))
{
yield return range.Provenance.Source;
}
}
}
foreach (var metric in advisory.CvssMetrics)
{
if (TryAddSource(metric.Provenance.Source))
{
yield return metric.Provenance.Source;
}
}
bool TryAddSource(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
return seen.Add(value);
}
}
private static string SelectPreferredIdentifier(Advisory advisory)
{
if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred))
{
return preferred;
}
foreach (var alias in advisory.Aliases)
{
if (TrySelectIdentifier(alias, out preferred))
{
return preferred;
}
}
return advisory.AdvisoryKey.Trim();
}
private static bool TrySelectIdentifier(string value, out string identifier)
{
identifier = string.Empty;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var trimmed = value.Trim();
if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed))
{
identifier = trimmed;
return true;
}
identifier = trimmed;
return false;
}
private static string SanitizeFileName(string name)
{
var invalid = Path.GetInvalidFileNameChars();
Span<char> buffer = stackalloc char[name.Length];
var count = 0;
foreach (var ch in name)
{
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
{
buffer[count++] = '_';
}
else
{
buffer[count++] = ch;
}
}
var sanitized = new string(buffer[..count]).Trim();
return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized;
}
private readonly record struct Layout(string[] Segments, string FileName);
}