Rename Feedser to Concelier
This commit is contained in:
28
src/StellaOps.Concelier.Exporter.Json/AGENTS.md
Normal file
28
src/StellaOps.Concelier.Exporter.Json/AGENTS.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving.
|
||||
## Scope
|
||||
- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable).
|
||||
- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601.
|
||||
- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites.
|
||||
- Packaging: output directory under exports/json/<timestamp> with reproducible naming; optionally symlink latest.
|
||||
- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads.
|
||||
## Participants
|
||||
- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests.
|
||||
- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job.
|
||||
- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured.
|
||||
## Interfaces & contracts
|
||||
- Job kind: export:json (JsonExportJob).
|
||||
- Determinism: same inputs -> identical file bytes; hash snapshot persisted.
|
||||
- Provenance: include minimal provenance fields when helpful; keep identity stable.
|
||||
## In/Out of scope
|
||||
In: JSON rendering and layout; incremental/deterministic writes.
|
||||
Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter).
|
||||
## Observability & security expectations
|
||||
- Metrics: export.json.records, bytes, duration, delta.changed.
|
||||
- Logs: target path, record counts, digest; no sensitive data.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.Exporter.Json.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public static class ExportDigestCalculator
|
||||
{
|
||||
public static string ComputeTreeDigest(JsonExportResult result)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
|
||||
using var sha256 = SHA256.Create();
|
||||
var buffer = new byte[128 * 1024];
|
||||
|
||||
foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal))
|
||||
{
|
||||
var normalized = file.Replace("\\", "/");
|
||||
var pathBytes = Encoding.UTF8.GetBytes(normalized);
|
||||
_ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0);
|
||||
|
||||
var fullPath = ResolveFullPath(result.ExportDirectory, normalized);
|
||||
using var stream = File.OpenRead(fullPath);
|
||||
int read;
|
||||
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
_ = sha256.TransformBlock(buffer, 0, read, null, 0);
|
||||
}
|
||||
}
|
||||
|
||||
_ = sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||
var hash = sha256.Hash ?? Array.Empty<byte>();
|
||||
var hex = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
return $"sha256:{hex}";
|
||||
}
|
||||
|
||||
private static string ResolveFullPath(string root, string normalizedRelativePath)
|
||||
{
|
||||
var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
var parts = new string[segments.Length + 1];
|
||||
parts[0] = root;
|
||||
for (var i = 0; i < segments.Length; i++)
|
||||
{
|
||||
parts[i + 1] = segments[i];
|
||||
}
|
||||
|
||||
return Path.Combine(parts);
|
||||
}
|
||||
}
|
||||
28
src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs
Normal file
28
src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs
Normal file
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public static class ExporterVersion
|
||||
{
|
||||
public static string GetVersion(Type anchor)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(anchor);
|
||||
var assembly = anchor.Assembly;
|
||||
|
||||
var informational = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
|
||||
if (!string.IsNullOrWhiteSpace(informational))
|
||||
{
|
||||
return informational;
|
||||
}
|
||||
|
||||
var fileVersion = assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version;
|
||||
if (!string.IsNullOrWhiteSpace(fileVersion))
|
||||
{
|
||||
return fileVersion!;
|
||||
}
|
||||
|
||||
var version = assembly.GetName().Version;
|
||||
return version?.ToString() ?? "0.0.0";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public interface IJsonExportPathResolver
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns the relative path (using platform directory separators) for the supplied advisory.
|
||||
/// Path must not include the leading export root.
|
||||
/// </summary>
|
||||
string GetRelativePath(Advisory advisory);
|
||||
}
|
||||
37
src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs
Normal file
37
src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs
Normal file
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Metadata describing a single file produced by the JSON exporter.
|
||||
/// </summary>
|
||||
public sealed class JsonExportFile
|
||||
{
|
||||
public JsonExportFile(string relativePath, long length, string digest)
|
||||
{
|
||||
RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath));
|
||||
if (relativePath.Length == 0)
|
||||
{
|
||||
throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath));
|
||||
}
|
||||
|
||||
if (length < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(length));
|
||||
}
|
||||
|
||||
Digest = digest ?? throw new ArgumentNullException(nameof(digest));
|
||||
if (digest.Length == 0)
|
||||
{
|
||||
throw new ArgumentException("Digest cannot be empty.", nameof(digest));
|
||||
}
|
||||
|
||||
Length = length;
|
||||
}
|
||||
|
||||
public string RelativePath { get; }
|
||||
|
||||
public long Length { get; }
|
||||
|
||||
public string Digest { get; }
|
||||
}
|
||||
30
src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs
Normal file
30
src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs
Normal file
@@ -0,0 +1,30 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExportJob : IJob
|
||||
{
|
||||
public const string JobKind = "export:json";
|
||||
public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10);
|
||||
public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly JsonFeedExporter _exporter;
|
||||
private readonly ILogger<JsonExportJob> _logger;
|
||||
|
||||
public JsonExportJob(JsonFeedExporter exporter, ILogger<JsonExportJob> logger)
|
||||
{
|
||||
_exporter = exporter ?? throw new ArgumentNullException(nameof(exporter));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Executing JSON export job {RunId}", context.RunId);
|
||||
await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Completed JSON export job {RunId}", context.RunId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
internal static class JsonExportManifestWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = true,
|
||||
};
|
||||
|
||||
public static async Task WriteAsync(
|
||||
JsonExportResult result,
|
||||
string digest,
|
||||
string exporterVersion,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
ArgumentException.ThrowIfNullOrEmpty(digest);
|
||||
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
|
||||
|
||||
var exportId = Path.GetFileName(result.ExportDirectory);
|
||||
var files = result.Files
|
||||
.Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
var manifest = new JsonExportManifest(
|
||||
exportId,
|
||||
result.ExportedAt.UtcDateTime,
|
||||
digest,
|
||||
result.AdvisoryCount,
|
||||
result.TotalBytes,
|
||||
files.Length,
|
||||
files,
|
||||
exporterVersion);
|
||||
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions);
|
||||
var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json");
|
||||
await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime);
|
||||
}
|
||||
|
||||
private sealed record JsonExportManifest(
|
||||
[property: JsonPropertyOrder(1)] string ExportId,
|
||||
[property: JsonPropertyOrder(2)] DateTime GeneratedAt,
|
||||
[property: JsonPropertyOrder(3)] string Digest,
|
||||
[property: JsonPropertyOrder(4)] int AdvisoryCount,
|
||||
[property: JsonPropertyOrder(5)] long TotalBytes,
|
||||
[property: JsonPropertyOrder(6)] int FileCount,
|
||||
[property: JsonPropertyOrder(7)] IReadOnlyList<JsonExportManifestFile> Files,
|
||||
[property: JsonPropertyOrder(8)] string ExporterVersion);
|
||||
|
||||
private sealed record JsonExportManifestFile(
|
||||
[property: JsonPropertyOrder(1)] string Path,
|
||||
[property: JsonPropertyOrder(2)] long Bytes,
|
||||
[property: JsonPropertyOrder(3)] string Digest);
|
||||
}
|
||||
34
src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs
Normal file
34
src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs
Normal file
@@ -0,0 +1,34 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for JSON exporter output paths and determinism controls.
|
||||
/// </summary>
|
||||
public sealed class JsonExportOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Root directory where exports are written. Default "exports/json".
|
||||
/// </summary>
|
||||
public string OutputRoot { get; set; } = Path.Combine("exports", "json");
|
||||
|
||||
/// <summary>
|
||||
/// Format string applied to the export timestamp to produce the directory name.
|
||||
/// </summary>
|
||||
public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'";
|
||||
|
||||
/// <summary>
|
||||
/// Optional static name for the symlink (or directory junction) pointing at the most recent export.
|
||||
/// </summary>
|
||||
public string LatestSymlinkName { get; set; } = "latest";
|
||||
|
||||
/// <summary>
|
||||
/// When true, attempts to re-point <see cref="LatestSymlinkName"/> after a successful export.
|
||||
/// </summary>
|
||||
public bool MaintainLatestSymlink { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Optional repository identifier recorded alongside export state metadata.
|
||||
/// </summary>
|
||||
public string? TargetRepository { get; set; }
|
||||
}
|
||||
46
src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs
Normal file
46
src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExportResult
|
||||
{
|
||||
public JsonExportResult(
|
||||
string exportDirectory,
|
||||
DateTimeOffset exportedAt,
|
||||
IEnumerable<JsonExportFile> files,
|
||||
int advisoryCount,
|
||||
long totalBytes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(exportDirectory))
|
||||
{
|
||||
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
|
||||
}
|
||||
|
||||
ExportDirectory = exportDirectory;
|
||||
ExportedAt = exportedAt;
|
||||
AdvisoryCount = advisoryCount;
|
||||
TotalBytes = totalBytes;
|
||||
|
||||
var list = (files ?? throw new ArgumentNullException(nameof(files)))
|
||||
.Where(static file => file is not null)
|
||||
.ToImmutableArray();
|
||||
|
||||
Files = list;
|
||||
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
|
||||
}
|
||||
|
||||
public string ExportDirectory { get; }
|
||||
|
||||
public DateTimeOffset ExportedAt { get; }
|
||||
|
||||
public ImmutableArray<JsonExportFile> Files { get; }
|
||||
|
||||
public ImmutableArray<string> FilePaths { get; }
|
||||
|
||||
public int AdvisoryCount { get; }
|
||||
|
||||
public long TotalBytes { get; }
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering.
|
||||
/// </summary>
|
||||
public sealed class JsonExportSnapshotBuilder
|
||||
{
|
||||
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
|
||||
private readonly JsonExportOptions _options;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
|
||||
public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
}
|
||||
|
||||
public Task<JsonExportResult> WriteAsync(
|
||||
IReadOnlyCollection<Advisory> advisories,
|
||||
DateTimeOffset exportedAt,
|
||||
string? exportName = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (advisories is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisories));
|
||||
}
|
||||
|
||||
return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<JsonExportResult> WriteAsync(
|
||||
IAsyncEnumerable<Advisory> advisories,
|
||||
DateTimeOffset exportedAt,
|
||||
string? exportName = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (advisories is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisories));
|
||||
}
|
||||
|
||||
var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
|
||||
if (string.IsNullOrWhiteSpace(exportDirectoryName))
|
||||
{
|
||||
throw new InvalidOperationException("Export directory name resolved to an empty string.");
|
||||
}
|
||||
|
||||
var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot));
|
||||
TrySetDirectoryTimestamp(exportRoot, exportedAt);
|
||||
var exportDirectory = Path.Combine(exportRoot, exportDirectoryName);
|
||||
|
||||
if (Directory.Exists(exportDirectory))
|
||||
{
|
||||
Directory.Delete(exportDirectory, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(exportDirectory);
|
||||
TrySetDirectoryTimestamp(exportDirectory, exportedAt);
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var files = new List<JsonExportFile>();
|
||||
long totalBytes = 0L;
|
||||
var advisoryCount = 0;
|
||||
|
||||
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
advisoryCount++;
|
||||
var entry = Resolve(advisory);
|
||||
if (!seen.Add(entry.RelativePath))
|
||||
{
|
||||
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
|
||||
}
|
||||
|
||||
var destination = Combine(exportDirectory, entry.Segments);
|
||||
var destinationDirectory = Path.GetDirectoryName(destination);
|
||||
if (!string.IsNullOrEmpty(destinationDirectory))
|
||||
{
|
||||
EnsureDirectoryExists(destinationDirectory);
|
||||
TrySetDirectoryTimestamp(destinationDirectory, exportedAt);
|
||||
}
|
||||
var payload = SnapshotSerializer.ToSnapshot(entry.Advisory);
|
||||
var bytes = Utf8NoBom.GetBytes(payload);
|
||||
|
||||
await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false);
|
||||
File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime);
|
||||
|
||||
var digest = ComputeDigest(bytes);
|
||||
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
|
||||
totalBytes += bytes.LongLength;
|
||||
}
|
||||
|
||||
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
|
||||
|
||||
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes);
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<Advisory> EnumerateAsync(
|
||||
IEnumerable<Advisory> advisories,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
yield return advisory;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
|
||||
private static string EnsureDirectoryExists(string directory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
{
|
||||
throw new ArgumentException("Directory path must be provided.", nameof(directory));
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
private static string Combine(string root, IReadOnlyList<string> segments)
|
||||
{
|
||||
var parts = new string[segments.Count + 1];
|
||||
parts[0] = root;
|
||||
for (var i = 0; i < segments.Count; i++)
|
||||
{
|
||||
parts[i + 1] = segments[i];
|
||||
}
|
||||
|
||||
return Path.Combine(parts);
|
||||
}
|
||||
|
||||
private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Ignore failure to set timestamps; not critical for content determinism.
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
// Ignore permission issues when setting timestamps.
|
||||
}
|
||||
catch (PlatformNotSupportedException)
|
||||
{
|
||||
// Some platforms may not support this operation.
|
||||
}
|
||||
}
|
||||
|
||||
private PathResolution Resolve(Advisory advisory)
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisory));
|
||||
}
|
||||
|
||||
var relativePath = _pathResolver.GetRelativePath(advisory);
|
||||
var segments = NormalizeRelativePath(relativePath);
|
||||
var normalized = string.Join('/', segments);
|
||||
return new PathResolution(advisory, normalized, segments);
|
||||
}
|
||||
|
||||
private static string[] NormalizeRelativePath(string relativePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver returned an empty path.");
|
||||
}
|
||||
|
||||
if (Path.IsPathRooted(relativePath))
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported.");
|
||||
}
|
||||
|
||||
var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (pieces.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Path resolver produced no path segments.");
|
||||
}
|
||||
|
||||
var sanitized = new string[pieces.Length];
|
||||
for (var i = 0; i < pieces.Length; i++)
|
||||
{
|
||||
var segment = pieces[i];
|
||||
if (segment == "." || segment == "..")
|
||||
{
|
||||
throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments.");
|
||||
}
|
||||
|
||||
sanitized[i] = SanitizeSegment(segment);
|
||||
}
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
private static string SanitizeSegment(string segment)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
Span<char> buffer = stackalloc char[segment.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in segment)
|
||||
{
|
||||
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
|
||||
{
|
||||
buffer[count++] = '_';
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer[count++] = ch;
|
||||
}
|
||||
}
|
||||
|
||||
var sanitized = new string(buffer[..count]).Trim();
|
||||
return string.IsNullOrEmpty(sanitized) ? "_" : sanitized;
|
||||
}
|
||||
|
||||
private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList<string> Segments);
|
||||
|
||||
private static string ComputeDigest(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
var hash = SHA256.HashData(payload);
|
||||
var hex = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
return $"sha256:{hex}";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:exporters:json";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.TryAddSingleton<IJsonExportPathResolver, VulnListJsonExportPathResolver>();
|
||||
services.TryAddSingleton<ExportStateManager>();
|
||||
|
||||
services.AddOptions<JsonExportOptions>()
|
||||
.Bind(configuration.GetSection(ConfigurationSection))
|
||||
.PostConfigure(static options =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.OutputRoot))
|
||||
{
|
||||
options.OutputRoot = Path.Combine("exports", "json");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
|
||||
{
|
||||
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
|
||||
}
|
||||
});
|
||||
|
||||
services.AddSingleton<JsonFeedExporter>();
|
||||
services.AddTransient<JsonExportJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
if (!options.Definitions.ContainsKey(JsonExportJob.JobKind))
|
||||
{
|
||||
options.Definitions[JsonExportJob.JobKind] = new JobDefinition(
|
||||
JsonExportJob.JobKind,
|
||||
typeof(JsonExportJob),
|
||||
JsonExportJob.DefaultTimeout,
|
||||
JsonExportJob.DefaultLeaseDuration,
|
||||
null,
|
||||
true);
|
||||
}
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
23
src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs
Normal file
23
src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs
Normal file
@@ -0,0 +1,23 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonExporterPlugin : IExporterPlugin
|
||||
{
|
||||
public string Name => JsonFeedExporter.ExporterName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetService<IAdvisoryStore>() is not null;
|
||||
}
|
||||
|
||||
public IFeedExporter Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<JsonFeedExporter>(services);
|
||||
}
|
||||
}
|
||||
170
src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs
Normal file
170
src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs
Normal file
@@ -0,0 +1,170 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Exporting;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
public sealed class JsonFeedExporter : IFeedExporter
|
||||
{
|
||||
public const string ExporterName = "json";
|
||||
public const string ExporterId = "export:json";
|
||||
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly JsonExportOptions _options;
|
||||
private readonly IJsonExportPathResolver _pathResolver;
|
||||
private readonly ExportStateManager _stateManager;
|
||||
private readonly ILogger<JsonFeedExporter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly string _exporterVersion;
|
||||
|
||||
public JsonFeedExporter(
|
||||
IAdvisoryStore advisoryStore,
|
||||
IOptions<JsonExportOptions> options,
|
||||
IJsonExportPathResolver pathResolver,
|
||||
ExportStateManager stateManager,
|
||||
ILogger<JsonFeedExporter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
|
||||
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
|
||||
}
|
||||
|
||||
public string Name => ExporterName;
|
||||
|
||||
public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var exportedAt = _timeProvider.GetUtcNow();
|
||||
var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
|
||||
var exportRoot = Path.GetFullPath(_options.OutputRoot);
|
||||
|
||||
_logger.LogInformation("Starting JSON export {ExportId}", exportId);
|
||||
|
||||
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
|
||||
var advisoryStream = _advisoryStore.StreamAsync(cancellationToken);
|
||||
var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
|
||||
_logger.LogInformation(
|
||||
"JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
|
||||
exportId,
|
||||
result.Files.Length,
|
||||
result.TotalBytes,
|
||||
result.AdvisoryCount,
|
||||
digest);
|
||||
|
||||
var manifest = result.Files
|
||||
.Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
|
||||
.ToArray();
|
||||
|
||||
if (existingState is not null
|
||||
&& existingState.Files.Count > 0
|
||||
&& string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId);
|
||||
TryDeleteDirectory(result.ExportDirectory);
|
||||
return;
|
||||
}
|
||||
|
||||
var resetBaseline = existingState is null
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|
||||
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
|
||||
|
||||
if (existingState is not null
|
||||
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
|
||||
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
|
||||
{
|
||||
resetBaseline = true;
|
||||
}
|
||||
|
||||
await _stateManager.StoreFullExportAsync(
|
||||
ExporterId,
|
||||
exportId,
|
||||
digest,
|
||||
cursor: digest,
|
||||
targetRepository: _options.TargetRepository,
|
||||
exporterVersion: _exporterVersion,
|
||||
resetBaseline: resetBaseline,
|
||||
manifest: manifest,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (_options.MaintainLatestSymlink)
|
||||
{
|
||||
TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName);
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(latestPath) || File.Exists(latestPath))
|
||||
{
|
||||
TryRemoveExistingPointer(latestPath);
|
||||
}
|
||||
|
||||
Directory.CreateSymbolicLink(latestPath, exportDirectory);
|
||||
_logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryRemoveExistingPointer(string latestPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var attributes = File.GetAttributes(latestPath);
|
||||
if (attributes.HasFlag(FileAttributes.Directory))
|
||||
{
|
||||
Directory.Delete(latestPath, recursive: false);
|
||||
}
|
||||
else
|
||||
{
|
||||
File.Delete(latestPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryDeleteDirectory(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(path))
|
||||
{
|
||||
Directory.Delete(path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
12
src/StellaOps.Concelier.Exporter.Json/TASKS.md
Normal file
12
src/StellaOps.Concelier.Exporter.Json/TASKS.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE – `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.|
|
||||
|Deterministic serializer|BE-Export|Models|DONE – Canonical serializer + snapshot builder emit stable JSON across runs.|
|
||||
|ExportState read/write|BE-Export|Storage.Mongo|DONE – `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.|
|
||||
|JsonExportJob wiring|BE-Export|Core|DONE – Job scheduler options now configurable via DI; JSON job registered with scheduler.|
|
||||
|Snapshot tests for file tree|QA|Exporters|DONE – Added resolver/exporter tests asserting tree layout and deterministic behavior.|
|
||||
|Parity smoke vs upstream vuln-list|QA|Exporters|DONE – `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.|
|
||||
|Stream advisories during export|BE-Export|Storage.Mongo|DONE – exporter + streaming-only test ensures single enumeration and per-file digest capture.|
|
||||
|Emit export manifest with digest metadata|BE-Export|Exporters|DONE – manifest now includes per-file digests/sizes alongside tree digest.|
|
||||
|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.|
|
||||
@@ -0,0 +1,456 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Identifiers;
|
||||
|
||||
namespace StellaOps.Concelier.Exporter.Json;
|
||||
|
||||
/// <summary>
|
||||
/// Path resolver approximating the directory layout used by aquasecurity/vuln-list.
|
||||
/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks.
|
||||
/// </summary>
|
||||
public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver
|
||||
{
|
||||
private static readonly Regex CvePattern = new("^CVE-(?<year>\\d{4})-(?<id>\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex UsnPattern = new("^USN-(?<id>\\d+-\\d+)(?<suffix>[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex DebianPattern = new("^(?<prefix>DLA|DSA|ELA)-(?<id>\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex RedHatPattern = new("^RH(?<type>SA|BA|EA)-(?<rest>[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex AmazonPattern = new("^ALAS(?<channel>2|2022|2023)?-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex OraclePattern = new("^(?<kind>ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex PhotonPattern = new("^PHSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex RockyPattern = new("^RLSA-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex SusePattern = new("^SUSE-(?<kind>SU|RU|OU|SB)-(?<rest>[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Dictionary<string, string[]> SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["nvd"] = new[] { "nvd" },
|
||||
["ghsa"] = new[] { "ghsa" },
|
||||
["github"] = new[] { "ghsa" },
|
||||
["osv"] = new[] { "osv" },
|
||||
["redhat"] = new[] { "redhat", "oval" },
|
||||
["ubuntu"] = new[] { "ubuntu" },
|
||||
["debian"] = new[] { "debian" },
|
||||
["oracle"] = new[] { "oracle" },
|
||||
["photon"] = new[] { "photon" },
|
||||
["rocky"] = new[] { "rocky" },
|
||||
["suse"] = new[] { "suse" },
|
||||
["amazon"] = new[] { "amazon" },
|
||||
["aws"] = new[] { "amazon" },
|
||||
["alpine"] = new[] { "alpine" },
|
||||
["wolfi"] = new[] { "wolfi" },
|
||||
["chainguard"] = new[] { "chainguard" },
|
||||
["cert-fr"] = new[] { "cert", "fr" },
|
||||
["cert-in"] = new[] { "cert", "in" },
|
||||
["cert-cc"] = new[] { "cert", "cc" },
|
||||
["cert-bund"] = new[] { "cert", "bund" },
|
||||
["acsc"] = new[] { "cert", "au" },
|
||||
["cisa"] = new[] { "ics", "cisa" },
|
||||
["ics-cisa"] = new[] { "ics", "cisa" },
|
||||
["ics-kaspersky"] = new[] { "ics", "kaspersky" },
|
||||
["kaspersky"] = new[] { "ics", "kaspersky" },
|
||||
};
|
||||
|
||||
private static readonly Dictionary<string, string> GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["go"] = "go",
|
||||
["golang"] = "go",
|
||||
["npm"] = "npm",
|
||||
["maven"] = "maven",
|
||||
["pypi"] = "pip",
|
||||
["pip"] = "pip",
|
||||
["nuget"] = "nuget",
|
||||
["composer"] = "composer",
|
||||
["packagist"] = "composer",
|
||||
["rubygems"] = "rubygems",
|
||||
["gem"] = "rubygems",
|
||||
["swift"] = "swift",
|
||||
["cargo"] = "cargo",
|
||||
["hex"] = "hex",
|
||||
["pub"] = "pub",
|
||||
["github"] = "github",
|
||||
["docker"] = "container",
|
||||
};
|
||||
|
||||
public string GetRelativePath(Advisory advisory)
|
||||
{
|
||||
if (advisory is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(advisory));
|
||||
}
|
||||
|
||||
var identifier = SelectPreferredIdentifier(advisory);
|
||||
if (identifier.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Unable to derive identifier for advisory.");
|
||||
}
|
||||
|
||||
var layout = ResolveLayout(advisory, identifier);
|
||||
var segments = new string[layout.Segments.Length + 1];
|
||||
for (var i = 0; i < layout.Segments.Length; i++)
|
||||
{
|
||||
segments[i] = layout.Segments[i];
|
||||
}
|
||||
segments[^1] = layout.FileName;
|
||||
return Path.Combine(segments);
|
||||
}
|
||||
|
||||
private static Layout ResolveLayout(Advisory advisory, string identifier)
|
||||
{
|
||||
if (TryResolveCve(identifier, out var layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveGhsa(advisory, identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveUsn(identifier, out layout) ||
|
||||
TryResolveDebian(identifier, out layout) ||
|
||||
TryResolveRedHat(identifier, out layout) ||
|
||||
TryResolveAmazon(identifier, out layout) ||
|
||||
TryResolveOracle(identifier, out layout) ||
|
||||
TryResolvePhoton(identifier, out layout) ||
|
||||
TryResolveRocky(identifier, out layout) ||
|
||||
TryResolveSuse(identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
if (TryResolveByProvenance(advisory, identifier, out layout))
|
||||
{
|
||||
return layout;
|
||||
}
|
||||
|
||||
return new Layout(new[] { "misc" }, CreateFileName(identifier));
|
||||
}
|
||||
|
||||
private static bool TryResolveCve(string identifier, out Layout layout)
|
||||
{
|
||||
var match = CvePattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
var year = match.Groups["year"].Value;
|
||||
layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout)
|
||||
{
|
||||
if (!GhsaPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath))
|
||||
{
|
||||
layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveUsn(string identifier, out Layout layout)
|
||||
{
|
||||
if (!UsnPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveDebian(string identifier, out Layout layout)
|
||||
{
|
||||
var match = DebianPattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveRedHat(string identifier, out Layout layout)
|
||||
{
|
||||
if (!RedHatPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveAmazon(string identifier, out Layout layout)
|
||||
{
|
||||
var match = AmazonPattern.Match(identifier);
|
||||
if (!match.Success)
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
var channel = match.Groups["channel"].Value;
|
||||
var subdirectory = channel switch
|
||||
{
|
||||
"2" => "2",
|
||||
"2023" => "2023",
|
||||
"2022" => "2022",
|
||||
_ => "1",
|
||||
};
|
||||
|
||||
layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveOracle(string identifier, out Layout layout)
|
||||
{
|
||||
if (!OraclePattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolvePhoton(string identifier, out Layout layout)
|
||||
{
|
||||
if (!PhotonPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveRocky(string identifier, out Layout layout)
|
||||
{
|
||||
if (!RockyPattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveSuse(string identifier, out Layout layout)
|
||||
{
|
||||
if (!SusePattern.IsMatch(identifier))
|
||||
{
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout)
|
||||
{
|
||||
foreach (var source in EnumerateDistinctProvenanceSources(advisory))
|
||||
{
|
||||
if (SourceDirectoryMap.TryGetValue(source, out var segments))
|
||||
{
|
||||
layout = new Layout(segments, CreateFileName(identifier));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
layout = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath)
|
||||
{
|
||||
foreach (var package in advisory.AffectedPackages)
|
||||
{
|
||||
if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (GhsaEcosystemMap.TryGetValue(type, out var mapped))
|
||||
{
|
||||
ecosystem = mapped;
|
||||
}
|
||||
else
|
||||
{
|
||||
ecosystem = type.ToLowerInvariant();
|
||||
}
|
||||
|
||||
packagePath = encodedPath;
|
||||
return true;
|
||||
}
|
||||
|
||||
ecosystem = "advisories";
|
||||
packagePath = "_";
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath)
|
||||
{
|
||||
type = string.Empty;
|
||||
encodedPath = string.Empty;
|
||||
|
||||
if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty
|
||||
? new[] { packageUrl.Name }
|
||||
: packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray();
|
||||
|
||||
type = packageUrl.Type;
|
||||
encodedPath = string.Join("%2F", segments);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string CreateFileName(string identifier, bool uppercase = false)
|
||||
{
|
||||
var candidate = uppercase ? identifier.ToUpperInvariant() : identifier;
|
||||
return $"{SanitizeFileName(candidate)}.json";
|
||||
}
|
||||
|
||||
private static IEnumerable<string> EnumerateDistinctProvenanceSources(Advisory advisory)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var source in advisory.Provenance)
|
||||
{
|
||||
if (TryAddSource(source.Source))
|
||||
{
|
||||
yield return source.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var reference in advisory.References)
|
||||
{
|
||||
if (TryAddSource(reference.Provenance.Source))
|
||||
{
|
||||
yield return reference.Provenance.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var package in advisory.AffectedPackages)
|
||||
{
|
||||
foreach (var source in package.Provenance)
|
||||
{
|
||||
if (TryAddSource(source.Source))
|
||||
{
|
||||
yield return source.Source;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var range in package.VersionRanges)
|
||||
{
|
||||
if (TryAddSource(range.Provenance.Source))
|
||||
{
|
||||
yield return range.Provenance.Source;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var metric in advisory.CvssMetrics)
|
||||
{
|
||||
if (TryAddSource(metric.Provenance.Source))
|
||||
{
|
||||
yield return metric.Provenance.Source;
|
||||
}
|
||||
}
|
||||
|
||||
bool TryAddSource(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return seen.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private static string SelectPreferredIdentifier(Advisory advisory)
|
||||
{
|
||||
if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred))
|
||||
{
|
||||
return preferred;
|
||||
}
|
||||
|
||||
foreach (var alias in advisory.Aliases)
|
||||
{
|
||||
if (TrySelectIdentifier(alias, out preferred))
|
||||
{
|
||||
return preferred;
|
||||
}
|
||||
}
|
||||
|
||||
return advisory.AdvisoryKey.Trim();
|
||||
}
|
||||
|
||||
private static bool TrySelectIdentifier(string value, out string identifier)
|
||||
{
|
||||
identifier = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed))
|
||||
{
|
||||
identifier = trimmed;
|
||||
return true;
|
||||
}
|
||||
|
||||
identifier = trimmed;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string name)
|
||||
{
|
||||
var invalid = Path.GetInvalidFileNameChars();
|
||||
Span<char> buffer = stackalloc char[name.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in name)
|
||||
{
|
||||
if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0)
|
||||
{
|
||||
buffer[count++] = '_';
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer[count++] = ch;
|
||||
}
|
||||
}
|
||||
|
||||
var sanitized = new string(buffer[..count]).Trim();
|
||||
return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized;
|
||||
}
|
||||
|
||||
private readonly record struct Layout(string[] Segments, string FileName);
|
||||
}
|
||||
Reference in New Issue
Block a user