Rename Feedser to Concelier

This commit is contained in:
master
2025-10-18 20:04:15 +03:00
parent dd66f58b00
commit 89ede53cc3
1208 changed files with 4370 additions and 4370 deletions

View File

@@ -0,0 +1,28 @@
# AGENTS
## Role
Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving.
## Scope
- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable).
- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601.
- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites.
- Packaging: output directory under exports/json/<timestamp> with reproducible naming; optionally symlink latest.
- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads.
## Participants
- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests.
- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job.
- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured.
## Interfaces & contracts
- Job kind: export:json (JsonExportJob).
- Determinism: same inputs -> identical file bytes; hash snapshot persisted.
- Provenance: include minimal provenance fields when helpful; keep identity stable.
## In/Out of scope
In: JSON rendering and layout; incremental/deterministic writes.
Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter).
## Observability & security expectations
- Metrics: export.json.records, bytes, duration, delta.changed.
- Logs: target path, record counts, digest; no sensitive data.
## Tests
- Author and review coverage in `../StellaOps.Concelier.Exporter.Json.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.

View File

@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Concelier.Exporter.Json;
public static class ExportDigestCalculator
{
public static string ComputeTreeDigest(JsonExportResult result)
{
ArgumentNullException.ThrowIfNull(result);
using var sha256 = SHA256.Create();
var buffer = new byte[128 * 1024];
foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal))
{
var normalized = file.Replace("\\", "/");
var pathBytes = Encoding.UTF8.GetBytes(normalized);
_ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0);
var fullPath = ResolveFullPath(result.ExportDirectory, normalized);
using var stream = File.OpenRead(fullPath);
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
{
_ = sha256.TransformBlock(buffer, 0, read, null, 0);
}
}
_ = sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
var hash = sha256.Hash ?? Array.Empty<byte>();
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
private static string ResolveFullPath(string root, string normalizedRelativePath)
{
var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
var parts = new string[segments.Length + 1];
parts[0] = root;
for (var i = 0; i < segments.Length; i++)
{
parts[i + 1] = segments[i];
}
return Path.Combine(parts);
}
}

View File

@@ -0,0 +1,28 @@
using System;
using System.Reflection;
namespace StellaOps.Concelier.Exporter.Json;
public static class ExporterVersion
{
public static string GetVersion(Type anchor)
{
ArgumentNullException.ThrowIfNull(anchor);
var assembly = anchor.Assembly;
var informational = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
if (!string.IsNullOrWhiteSpace(informational))
{
return informational;
}
var fileVersion = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version;
if (!string.IsNullOrWhiteSpace(fileVersion))
{
return fileVersion!;
}
var version = assembly.GetName().Version;
return version?.ToString() ?? "0.0.0";
}
}

View File

@@ -0,0 +1,12 @@
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
public interface IJsonExportPathResolver
{
/// <summary>
/// Returns the relative path (using platform directory separators) for the supplied advisory.
/// Path must not include the leading export root.
/// </summary>
string GetRelativePath(Advisory advisory);
}

View File

@@ -0,0 +1,37 @@
using System;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Metadata describing a single file produced by the JSON exporter.
/// </summary>
public sealed class JsonExportFile
{
public JsonExportFile(string relativePath, long length, string digest)
{
RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath));
if (relativePath.Length == 0)
{
throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath));
}
if (length < 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
Digest = digest ?? throw new ArgumentNullException(nameof(digest));
if (digest.Length == 0)
{
throw new ArgumentException("Digest cannot be empty.", nameof(digest));
}
Length = length;
}
public string RelativePath { get; }
public long Length { get; }
public string Digest { get; }
}

View File

@@ -0,0 +1,30 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportJob : IJob
{
public const string JobKind = "export:json";
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10);
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5);
private readonly JsonFeedExporter _exporter;
private readonly ILogger<JsonExportJob> _logger;
public JsonExportJob(JsonFeedExporter exporter, ILogger<JsonExportJob> logger)
{
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("Executing JSON export job {RunId}", context.RunId);
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Completed JSON export job {RunId}", context.RunId);
}
}

View File

@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Exporter.Json;
internal static class JsonExportManifestWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = true,
};
public static async Task WriteAsync(
JsonExportResult result,
string digest,
string exporterVersion,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(result);
ArgumentException.ThrowIfNullOrEmpty(digest);
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
var exportId = Path.GetFileName(result.ExportDirectory);
var files = result.Files
.Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest))
.ToArray();
var manifest = new JsonExportManifest(
exportId,
result.ExportedAt.UtcDateTime,
digest,
result.AdvisoryCount,
result.TotalBytes,
files.Length,
files,
exporterVersion);
var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions);
var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json");
await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime);
}
private sealed record JsonExportManifest(
[property: JsonPropertyOrder(1)] string ExportId,
[property: JsonPropertyOrder(2)] DateTime GeneratedAt,
[property: JsonPropertyOrder(3)] string Digest,
[property: JsonPropertyOrder(4)] int AdvisoryCount,
[property: JsonPropertyOrder(5)] long TotalBytes,
[property: JsonPropertyOrder(6)] int FileCount,
[property: JsonPropertyOrder(7)] IReadOnlyList<JsonExportManifestFile> Files,
[property: JsonPropertyOrder(8)] string ExporterVersion);
private sealed record JsonExportManifestFile(
[property: JsonPropertyOrder(1)] string Path,
[property: JsonPropertyOrder(2)] long Bytes,
[property: JsonPropertyOrder(3)] string Digest);
}

View File

@@ -0,0 +1,34 @@
using System.IO;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Configuration for JSON exporter output paths and determinism controls.
/// </summary>
public sealed class JsonExportOptions
{
/// <summary>
/// Root directory where exports are written. Default "exports/json".
/// </summary>
public string OutputRoot { get; set; } = Path.Combine("exports", "json");
/// <summary>
/// Format string applied to the export timestamp to produce the directory name.
/// </summary>
public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
/// <summary>
/// Optional static name for the symlink (or directory junction) pointing at the most recent export.
/// </summary>
public string LatestSymlinkName { get; set; } = "latest";
/// <summary>
/// When true, attempts to re-point <see cref="LatestSymlinkName"/> after a successful export.
/// </summary>
public bool MaintainLatestSymlink { get; set; } = true;
/// <summary>
/// Optional repository identifier recorded alongside export state metadata.
/// </summary>
public string? TargetRepository { get; set; }
}

View File

@@ -0,0 +1,46 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportResult
{
public JsonExportResult(
string exportDirectory,
DateTimeOffset exportedAt,
IEnumerable<JsonExportFile> files,
int advisoryCount,
long totalBytes)
{
if (string.IsNullOrWhiteSpace(exportDirectory))
{
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
}
ExportDirectory = exportDirectory;
ExportedAt = exportedAt;
AdvisoryCount = advisoryCount;
TotalBytes = totalBytes;
var list = (files ?? throw new ArgumentNullException(nameof(files)))
.Where(static file => file is not null)
.ToImmutableArray();
Files = list;
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
}
public string ExportDirectory { get; }
public DateTimeOffset ExportedAt { get; }
public ImmutableArray<JsonExportFile> Files { get; }
public ImmutableArray<string> FilePaths { get; }
public int AdvisoryCount { get; }
public long TotalBytes { get; }
}

View File

@@ -0,0 +1,239 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering.
/// </summary>
public sealed class JsonExportSnapshotBuilder
{
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
}
public Task<JsonExportResult> WriteAsync(
IReadOnlyCollection<Advisory> advisories,
DateTimeOffset exportedAt,
string? exportName = null,
CancellationToken cancellationToken = default)
{
if (advisories is null)
{
throw new ArgumentNullException(nameof(advisories));
}
return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken);
}
public async Task<JsonExportResult> WriteAsync(
IAsyncEnumerable<Advisory> advisories,
DateTimeOffset exportedAt,
string? exportName = null,
CancellationToken cancellationToken = default)
{
if (advisories is null)
{
throw new ArgumentNullException(nameof(advisories));
}
var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
if (string.IsNullOrWhiteSpace(exportDirectoryName))
{
throw new InvalidOperationException("Export directory name resolved to an empty string.");
}
var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot));
TrySetDirectoryTimestamp(exportRoot, exportedAt);
var exportDirectory = Path.Combine(exportRoot, exportDirectoryName);
if (Directory.Exists(exportDirectory))
{
Directory.Delete(exportDirectory, recursive: true);
}
Directory.CreateDirectory(exportDirectory);
TrySetDirectoryTimestamp(exportDirectory, exportedAt);
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var files = new List<JsonExportFile>();
long totalBytes = 0L;
var advisoryCount = 0;
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
advisoryCount++;
var entry = Resolve(advisory);
if (!seen.Add(entry.RelativePath))
{
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
}
var destination = Combine(exportDirectory, entry.Segments);
var destinationDirectory = Path.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory))
{
EnsureDirectoryExists(destinationDirectory);
TrySetDirectoryTimestamp(destinationDirectory, exportedAt);
}
var payload = SnapshotSerializer.ToSnapshot(entry.Advisory);
var bytes = Utf8NoBom.GetBytes(payload);
await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime);
var digest = ComputeDigest(bytes);
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
totalBytes += bytes.LongLength;
}
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes);
}
private static async IAsyncEnumerable<Advisory> EnumerateAsync(
IEnumerable<Advisory> advisories,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
foreach (var advisory in advisories)
{
cancellationToken.ThrowIfCancellationRequested();
yield return advisory;
await Task.Yield();
}
}
private static string EnsureDirectoryExists(string directory)
{
if (string.IsNullOrWhiteSpace(directory))
{
throw new ArgumentException("Directory path must be provided.", nameof(directory));
}
Directory.CreateDirectory(directory);
return directory;
}
private static string Combine(string root, IReadOnlyList<string> segments)
{
var parts = new string[segments.Count + 1];
parts[0] = root;
for (var i = 0; i < segments.Count; i++)
{
parts[i + 1] = segments[i];
}
return Path.Combine(parts);
}
private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp)
{
try
{
Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime);
}
catch (IOException)
{
// Ignore failure to set timestamps; not critical for content determinism.
}
catch (UnauthorizedAccessException)
{
// Ignore permission issues when setting timestamps.
}
catch (PlatformNotSupportedException)
{
// Some platforms may not support this operation.
}
}
private PathResolution Resolve(Advisory advisory)
{
if (advisory is null)
{
throw new ArgumentNullException(nameof(advisory));
}
var relativePath = _pathResolver.GetRelativePath(advisory);
var segments = NormalizeRelativePath(relativePath);
var normalized = string.Join('/', segments);
return new PathResolution(advisory, normalized, segments);
}
private static string[] NormalizeRelativePath(string relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath))
{
throw new InvalidOperationException("Path resolver returned an empty path.");
}
if (Path.IsPathRooted(relativePath))
{
throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported.");
}
var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
if (pieces.Length == 0)
{
throw new InvalidOperationException("Path resolver produced no path segments.");
}
var sanitized = new string[pieces.Length];
for (var i = 0; i < pieces.Length; i++)
{
var segment = pieces[i];
if (segment == "." || segment == "..")
{
throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments.");
}
sanitized[i] = SanitizeSegment(segment);
}
return sanitized;
}
private static string SanitizeSegment(string segment)
{
var invalid = Path.GetInvalidFileNameChars();
Span<char> buffer = stackalloc char[segment.Length];
var count = 0;
foreach (var ch in segment)
{
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
{
buffer[count++] = '_';
}
else
{
buffer[count++] = ch;
}
}
var sanitized = new string(buffer[..count]).Trim();
return string.IsNullOrEmpty(sanitized) ? "_" : sanitized;
}
private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList<string> Segments);
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
var hex = Convert.ToHexString(hash).ToLowerInvariant();
return $"sha256:{hex}";
}
}

View File

@@ -0,0 +1,59 @@
using System;
using System.IO;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Storage.Mongo.Exporting;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:exporters:json";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
services.TryAddSingleton<ExportStateManager>();
services.AddOptions<JsonExportOptions>()
.Bind(configuration.GetSection(ConfigurationSection))
.PostConfigure(static options =>
{
if (string.IsNullOrWhiteSpace(options.OutputRoot))
{
options.OutputRoot = Path.Combine("exports", "json");
}
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
{
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
}
});
services.AddSingleton<JsonFeedExporter>();
services.AddTransient<JsonExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
if (!options.Definitions.ContainsKey(JsonExportJob.JobKind))
{
options.Definitions[JsonExportJob.JobKind] = new JobDefinition(
JsonExportJob.JobKind,
typeof(JsonExportJob),
JsonExportJob.DefaultTimeout,
JsonExportJob.DefaultLeaseDuration,
null,
true);
}
});
return services;
}
}

View File

@@ -0,0 +1,23 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExporterPlugin : IExporterPlugin
{
public string Name => JsonFeedExporter.ExporterName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<IAdvisoryStore>() is not null;
}
public IFeedExporter Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<JsonFeedExporter>(services);
}
}

View File

@@ -0,0 +1,170 @@
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonFeedExporter : IFeedExporter
{
public const string ExporterName = "json";
public const string ExporterId = "export:json";
private readonly IAdvisoryStore _advisoryStore;
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
private readonly ExportStateManager _stateManager;
private readonly ILogger<JsonFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public JsonFeedExporter(
IAdvisoryStore advisoryStore,
IOptions<JsonExportOptions> options,
IJsonExportPathResolver pathResolver,
ExportStateManager stateManager,
ILogger<JsonFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
}
public string Name => ExporterName;
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var exportedAt = _timeProvider.GetUtcNow();
var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
var exportRoot = Path.GetFullPath(_options.OutputRoot);
_logger.LogInformation("Starting JSON export {ExportId}", exportId);
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
var advisoryStream = _advisoryStore.StreamAsync(cancellationToken);
var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
_logger.LogInformation(
"JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
exportId,
result.Files.Length,
result.TotalBytes,
result.AdvisoryCount,
digest);
var manifest = result.Files
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
.ToArray();
if (existingState is not null
&& existingState.Files.Count > 0
&& string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal))
{
_logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId);
TryDeleteDirectory(result.ExportDirectory);
return;
}
var resetBaseline = existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
digest,
cursor: digest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
manifest: manifest,
cancellationToken: cancellationToken).ConfigureAwait(false);
await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);
if (_options.MaintainLatestSymlink)
{
TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
}
}
private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
{
if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName))
{
return;
}
var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName);
try
{
if (Directory.Exists(latestPath) || File.Exists(latestPath))
{
TryRemoveExistingPointer(latestPath);
}
Directory.CreateSymbolicLink(latestPath, exportDirectory);
_logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException)
{
_logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath);
}
}
private void TryRemoveExistingPointer(string latestPath)
{
try
{
var attributes = File.GetAttributes(latestPath);
if (attributes.HasFlag(FileAttributes.Directory))
{
Directory.Delete(latestPath, recursive: false);
}
else
{
File.Delete(latestPath);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath);
}
}
private void TryDeleteDirectory(string path)
{
try
{
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
_logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path);
}
}
}

View File

@@ -0,0 +1,22 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,12 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.|
|Deterministic serializer|BE-Export|Models|DONE Canonical serializer + snapshot builder emit stable JSON across runs.|
|ExportState read/write|BE-Export|Storage.Mongo|DONE `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.|
|JsonExportJob wiring|BE-Export|Core|DONE Job scheduler options now configurable via DI; JSON job registered with scheduler.|
|Snapshot tests for file tree|QA|Exporters|DONE Added resolver/exporter tests asserting tree layout and deterministic behavior.|
|Parity smoke vs upstream vuln-list|QA|Exporters|DONE `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.|
|Stream advisories during export|BE-Export|Storage.Mongo|DONE exporter + streaming-only test ensures single enumeration and per-file digest capture.|
|Emit export manifest with digest metadata|BE-Export|Exporters|DONE manifest now includes per-file digests/sizes alongside tree digest.|
|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.|

View File

@@ -0,0 +1,456 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Normalization.Identifiers;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Path resolver approximating the directory layout used by aquasecurity/vuln-list.
/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks.
/// </summary>
public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver
{
private static readonly Regex CvePattern = new("^CVE-(?<year>\\d{4})-(?<id>\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex UsnPattern = new("^USN-(?<id>\\d+-\\d+)(?<suffix>[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex DebianPattern = new("^(?<prefix>DLA|DSA|ELA)-(?<id>\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex RedHatPattern = new("^RH(?<type>SA|BA|EA)-(?<rest>[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex AmazonPattern = new("^ALAS(?<channel>2|2022|2023)?-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex OraclePattern = new("^(?<kind>ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex PhotonPattern = new("^PHSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex RockyPattern = new("^RLSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Regex SusePattern = new("^SUSE-(?<kind>SU|RU|OU|SB)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly Dictionary<string, string[]> SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase)
{
["nvd"] = new[] { "nvd" },
["ghsa"] = new[] { "ghsa" },
["github"] = new[] { "ghsa" },
["osv"] = new[] { "osv" },
["redhat"] = new[] { "redhat", "oval" },
["ubuntu"] = new[] { "ubuntu" },
["debian"] = new[] { "debian" },
["oracle"] = new[] { "oracle" },
["photon"] = new[] { "photon" },
["rocky"] = new[] { "rocky" },
["suse"] = new[] { "suse" },
["amazon"] = new[] { "amazon" },
["aws"] = new[] { "amazon" },
["alpine"] = new[] { "alpine" },
["wolfi"] = new[] { "wolfi" },
["chainguard"] = new[] { "chainguard" },
["cert-fr"] = new[] { "cert", "fr" },
["cert-in"] = new[] { "cert", "in" },
["cert-cc"] = new[] { "cert", "cc" },
["cert-bund"] = new[] { "cert", "bund" },
["acsc"] = new[] { "cert", "au" },
["cisa"] = new[] { "ics", "cisa" },
["ics-cisa"] = new[] { "ics", "cisa" },
["ics-kaspersky"] = new[] { "ics", "kaspersky" },
["kaspersky"] = new[] { "ics", "kaspersky" },
};
private static readonly Dictionary<string, string> GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase)
{
["go"] = "go",
["golang"] = "go",
["npm"] = "npm",
["maven"] = "maven",
["pypi"] = "pip",
["pip"] = "pip",
["nuget"] = "nuget",
["composer"] = "composer",
["packagist"] = "composer",
["rubygems"] = "rubygems",
["gem"] = "rubygems",
["swift"] = "swift",
["cargo"] = "cargo",
["hex"] = "hex",
["pub"] = "pub",
["github"] = "github",
["docker"] = "container",
};
public string GetRelativePath(Advisory advisory)
{
if (advisory is null)
{
throw new ArgumentNullException(nameof(advisory));
}
var identifier = SelectPreferredIdentifier(advisory);
if (identifier.Length == 0)
{
throw new InvalidOperationException("Unable to derive identifier for advisory.");
}
var layout = ResolveLayout(advisory, identifier);
var segments = new string[layout.Segments.Length + 1];
for (var i = 0; i < layout.Segments.Length; i++)
{
segments[i] = layout.Segments[i];
}
segments[^1] = layout.FileName;
return Path.Combine(segments);
}
private static Layout ResolveLayout(Advisory advisory, string identifier)
{
if (TryResolveCve(identifier, out var layout))
{
return layout;
}
if (TryResolveGhsa(advisory, identifier, out layout))
{
return layout;
}
if (TryResolveUsn(identifier, out layout) ||
TryResolveDebian(identifier, out layout) ||
TryResolveRedHat(identifier, out layout) ||
TryResolveAmazon(identifier, out layout) ||
TryResolveOracle(identifier, out layout) ||
TryResolvePhoton(identifier, out layout) ||
TryResolveRocky(identifier, out layout) ||
TryResolveSuse(identifier, out layout))
{
return layout;
}
if (TryResolveByProvenance(advisory, identifier, out layout))
{
return layout;
}
return new Layout(new[] { "misc" }, CreateFileName(identifier));
}
private static bool TryResolveCve(string identifier, out Layout layout)
{
var match = CvePattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
var year = match.Groups["year"].Value;
layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout)
{
if (!GhsaPattern.IsMatch(identifier))
{
layout = default;
return false;
}
if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath))
{
layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true));
return true;
}
layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveUsn(string identifier, out Layout layout)
{
if (!UsnPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveDebian(string identifier, out Layout layout)
{
var match = DebianPattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveRedHat(string identifier, out Layout layout)
{
if (!RedHatPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveAmazon(string identifier, out Layout layout)
{
var match = AmazonPattern.Match(identifier);
if (!match.Success)
{
layout = default;
return false;
}
var channel = match.Groups["channel"].Value;
var subdirectory = channel switch
{
"2" => "2",
"2023" => "2023",
"2022" => "2022",
_ => "1",
};
layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveOracle(string identifier, out Layout layout)
{
if (!OraclePattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolvePhoton(string identifier, out Layout layout)
{
if (!PhotonPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveRocky(string identifier, out Layout layout)
{
if (!RockyPattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveSuse(string identifier, out Layout layout)
{
if (!SusePattern.IsMatch(identifier))
{
layout = default;
return false;
}
layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true));
return true;
}
private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout)
{
foreach (var source in EnumerateDistinctProvenanceSources(advisory))
{
if (SourceDirectoryMap.TryGetValue(source, out var segments))
{
layout = new Layout(segments, CreateFileName(identifier));
return true;
}
}
layout = default;
return false;
}
private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath)
{
foreach (var package in advisory.AffectedPackages)
{
if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath))
{
continue;
}
if (GhsaEcosystemMap.TryGetValue(type, out var mapped))
{
ecosystem = mapped;
}
else
{
ecosystem = type.ToLowerInvariant();
}
packagePath = encodedPath;
return true;
}
ecosystem = "advisories";
packagePath = "_";
return false;
}
private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath)
{
type = string.Empty;
encodedPath = string.Empty;
if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl))
{
return false;
}
var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty
? new[] { packageUrl.Name }
: packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray();
type = packageUrl.Type;
encodedPath = string.Join("%2F", segments);
return true;
}
private static string CreateFileName(string identifier, bool uppercase = false)
{
var candidate = uppercase ? identifier.ToUpperInvariant() : identifier;
return $"{SanitizeFileName(candidate)}.json";
}
private static IEnumerable<string> EnumerateDistinctProvenanceSources(Advisory advisory)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var source in advisory.Provenance)
{
if (TryAddSource(source.Source))
{
yield return source.Source;
}
}
foreach (var reference in advisory.References)
{
if (TryAddSource(reference.Provenance.Source))
{
yield return reference.Provenance.Source;
}
}
foreach (var package in advisory.AffectedPackages)
{
foreach (var source in package.Provenance)
{
if (TryAddSource(source.Source))
{
yield return source.Source;
}
}
foreach (var range in package.VersionRanges)
{
if (TryAddSource(range.Provenance.Source))
{
yield return range.Provenance.Source;
}
}
}
foreach (var metric in advisory.CvssMetrics)
{
if (TryAddSource(metric.Provenance.Source))
{
yield return metric.Provenance.Source;
}
}
bool TryAddSource(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
return seen.Add(value);
}
}
private static string SelectPreferredIdentifier(Advisory advisory)
{
if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred))
{
return preferred;
}
foreach (var alias in advisory.Aliases)
{
if (TrySelectIdentifier(alias, out preferred))
{
return preferred;
}
}
return advisory.AdvisoryKey.Trim();
}
private static bool TrySelectIdentifier(string value, out string identifier)
{
identifier = string.Empty;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var trimmed = value.Trim();
if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed))
{
identifier = trimmed;
return true;
}
identifier = trimmed;
return false;
}
private static string SanitizeFileName(string name)
{
var invalid = Path.GetInvalidFileNameChars();
Span<char> buffer = stackalloc char[name.Length];
var count = 0;
foreach (var ch in name)
{
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
{
buffer[count++] = '_';
}
else
{
buffer[count++] = ch;
}
}
var sanitized = new string(buffer[..count]).Trim();
return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized;
}
private readonly record struct Layout(string[] Segments, string FileName);
}