Add channel test providers for Email, Slack, Teams, and Webhook

- Implemented EmailChannelTestProvider to generate email preview payloads.
- Implemented SlackChannelTestProvider to create Slack message previews.
- Implemented TeamsChannelTestProvider for generating Teams Adaptive Card previews.
- Implemented WebhookChannelTestProvider to create webhook payloads.
- Added INotifyChannelTestProvider interface for channel-specific preview generation.
- Created ChannelTestPreviewContracts for request and response models.
- Developed NotifyChannelTestService to handle test send requests and generate previews.
- Added rate limit policies for test sends and delivery history.
- Implemented unit tests for service registration and binding.
- Updated project files to include necessary dependencies and configurations.
This commit is contained in:
master
2025-10-19 23:29:34 +03:00
parent a811f7ac47
commit a07f46231b
239 changed files with 17245 additions and 3155 deletions

View File

@@ -1,9 +1,11 @@
using System.IO;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Configuration for JSON exporter output paths and determinism controls.
using System.Collections.Generic;
using System.IO;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Exporter.Json;
/// <summary>
/// Configuration for JSON exporter output paths and determinism controls.
/// </summary>
public sealed class JsonExportOptions
{
@@ -27,8 +29,87 @@ public sealed class JsonExportOptions
/// </summary>
public bool MaintainLatestSymlink { get; set; } = true;
/// <summary>
/// Optional repository identifier recorded alongside export state metadata.
/// </summary>
public string? TargetRepository { get; set; }
}
/// <summary>
/// Optional repository identifier recorded alongside export state metadata.
/// </summary>
public string? TargetRepository { get; set; }
/// <summary>
/// Mirror distribution configuration producing aggregate bundles for downstream mirrors.
/// </summary>
public JsonMirrorOptions Mirror { get; set; } = new();
public sealed class JsonMirrorOptions
{
/// <summary>
/// Indicates whether mirror bundle generation is enabled.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Directory name (relative to the export root) where mirror artefacts are written.
/// </summary>
public string DirectoryName { get; set; } = "mirror";
/// <summary>
/// Domains exposed to downstream mirrors.
/// </summary>
public IList<JsonMirrorDomainOptions> Domains { get; } = new List<JsonMirrorDomainOptions>();
/// <summary>
/// Signing configuration for mirror bundles.
/// </summary>
public JsonMirrorSigningOptions Signing { get; set; } = new();
}
public sealed class JsonMirrorDomainOptions
{
/// <summary>
/// Stable identifier for the mirror domain (used in URLs and directory names).
/// </summary>
public string Id { get; set; } = string.Empty;
/// <summary>
/// Optional human-readable label for UI surfaces.
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Optional advisory scheme filters (e.g. CVE, GHSA). Empty collection selects all schemes.
/// </summary>
public IList<string> IncludeSchemes { get; } = new List<string>();
/// <summary>
/// Optional provenance source filters (e.g. nvd, ghsa). Empty collection selects all sources.
/// </summary>
public IList<string> IncludeSources { get; } = new List<string>();
}
public sealed class JsonMirrorSigningOptions
{
/// <summary>
/// Indicates whether bundles should be signed. Defaults to disabled.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Signing algorithm identifier (defaults to ES256).
/// </summary>
public string Algorithm { get; set; } = SignatureAlgorithms.Es256;
/// <summary>
/// Active signing key identifier.
/// </summary>
public string KeyId { get; set; } = string.Empty;
/// <summary>
/// Path to the private key (PEM) used for signing mirror bundles.
/// </summary>
public string KeyPath { get; set; } = string.Empty;
/// <summary>
/// Optional crypto provider hint. When omitted the registry resolves an appropriate provider.
/// </summary>
public string? Provider { get; set; }
}
}

View File

@@ -1,46 +1,55 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportResult
{
public JsonExportResult(
string exportDirectory,
DateTimeOffset exportedAt,
IEnumerable<JsonExportFile> files,
int advisoryCount,
long totalBytes)
{
if (string.IsNullOrWhiteSpace(exportDirectory))
{
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
}
ExportDirectory = exportDirectory;
ExportedAt = exportedAt;
AdvisoryCount = advisoryCount;
TotalBytes = totalBytes;
var list = (files ?? throw new ArgumentNullException(nameof(files)))
.Where(static file => file is not null)
.ToImmutableArray();
Files = list;
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
}
public string ExportDirectory { get; }
public DateTimeOffset ExportedAt { get; }
public ImmutableArray<JsonExportFile> Files { get; }
public ImmutableArray<string> FilePaths { get; }
public int AdvisoryCount { get; }
public long TotalBytes { get; }
}
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Exporter.Json;
public sealed class JsonExportResult
{
public JsonExportResult(
string exportDirectory,
DateTimeOffset exportedAt,
IEnumerable<JsonExportFile> files,
int advisoryCount,
long totalBytes,
IEnumerable<Advisory>? advisories = null)
{
if (string.IsNullOrWhiteSpace(exportDirectory))
{
throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory));
}
var list = (files ?? throw new ArgumentNullException(nameof(files)))
.Where(static file => file is not null)
.ToImmutableArray();
var advisoryList = (advisories ?? Array.Empty<Advisory>())
.Where(static advisory => advisory is not null)
.ToImmutableArray();
ExportDirectory = exportDirectory;
ExportedAt = exportedAt;
TotalBytes = totalBytes;
Files = list;
FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray();
Advisories = advisoryList;
AdvisoryCount = advisoryList.IsDefaultOrEmpty ? advisoryCount : advisoryList.Length;
}
public string ExportDirectory { get; }
public DateTimeOffset ExportedAt { get; }
public ImmutableArray<JsonExportFile> Files { get; }
public ImmutableArray<string> FilePaths { get; }
public ImmutableArray<Advisory> Advisories { get; }
public int AdvisoryCount { get; }
public long TotalBytes { get; }
}

View File

@@ -67,26 +67,27 @@ public sealed class JsonExportSnapshotBuilder
Directory.CreateDirectory(exportDirectory);
TrySetDirectoryTimestamp(exportDirectory, exportedAt);
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var files = new List<JsonExportFile>();
long totalBytes = 0L;
var advisoryCount = 0;
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
advisoryCount++;
var entry = Resolve(advisory);
if (!seen.Add(entry.RelativePath))
{
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
}
var destination = Combine(exportDirectory, entry.Segments);
var destinationDirectory = Path.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory))
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var files = new List<JsonExportFile>();
var advisoryList = new List<Advisory>();
long totalBytes = 0L;
await foreach (var advisory in advisories.WithCancellation(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
var entry = Resolve(advisory);
if (!seen.Add(entry.RelativePath))
{
throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'.");
}
advisoryList.Add(entry.Advisory);
var destination = Combine(exportDirectory, entry.Segments);
var destinationDirectory = Path.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory))
{
EnsureDirectoryExists(destinationDirectory);
TrySetDirectoryTimestamp(destinationDirectory, exportedAt);
}
@@ -97,14 +98,14 @@ public sealed class JsonExportSnapshotBuilder
File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime);
var digest = ComputeDigest(bytes);
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
totalBytes += bytes.LongLength;
}
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes);
}
files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest));
totalBytes += bytes.LongLength;
}
files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath));
return new JsonExportResult(exportDirectory, exportedAt, files, advisoryList.Count, totalBytes, advisoryList);
}
private static async IAsyncEnumerable<Advisory> EnumerateAsync(
IEnumerable<Advisory> advisories,
@@ -168,10 +169,11 @@ public sealed class JsonExportSnapshotBuilder
throw new ArgumentNullException(nameof(advisory));
}
var relativePath = _pathResolver.GetRelativePath(advisory);
var segments = NormalizeRelativePath(relativePath);
var normalized = string.Join('/', segments);
return new PathResolution(advisory, normalized, segments);
var normalized = CanonicalJsonSerializer.Normalize(advisory);
var relativePath = _pathResolver.GetRelativePath(normalized);
var segments = NormalizeRelativePath(relativePath);
var normalizedPath = string.Join('/', segments);
return new PathResolution(normalized, normalizedPath, segments);
}
private static string[] NormalizeRelativePath(string relativePath)

View File

@@ -31,14 +31,19 @@ public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectio
options.OutputRoot = Path.Combine("exports", "json");
}
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
{
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
}
});
services.AddSingleton<JsonFeedExporter>();
services.AddTransient<JsonExportJob>();
if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat))
{
options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'";
}
if (string.IsNullOrWhiteSpace(options.Mirror.DirectoryName))
{
options.Mirror.DirectoryName = "mirror";
}
});
services.AddSingleton<JsonFeedExporter>();
services.AddTransient<JsonExportJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{

View File

@@ -1,12 +1,16 @@
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.Events;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Exporter.Json;
@@ -16,29 +20,32 @@ public sealed class JsonFeedExporter : IFeedExporter
public const string ExporterId = "export:json";
private readonly IAdvisoryStore _advisoryStore;
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
private readonly ExportStateManager _stateManager;
private readonly ILogger<JsonFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
public JsonFeedExporter(
IAdvisoryStore advisoryStore,
IOptions<JsonExportOptions> options,
IJsonExportPathResolver pathResolver,
ExportStateManager stateManager,
ILogger<JsonFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
}
private readonly JsonExportOptions _options;
private readonly IJsonExportPathResolver _pathResolver;
private readonly ExportStateManager _stateManager;
private readonly ILogger<JsonFeedExporter> _logger;
private readonly TimeProvider _timeProvider;
private readonly string _exporterVersion;
private readonly IAdvisoryEventLog _eventLog;
public JsonFeedExporter(
IAdvisoryStore advisoryStore,
IOptions<JsonExportOptions> options,
IJsonExportPathResolver pathResolver,
ExportStateManager stateManager,
IAdvisoryEventLog eventLog,
ILogger<JsonFeedExporter> logger,
TimeProvider? timeProvider = null)
{
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
_stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
_eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
}
public string Name => ExporterName;
@@ -52,11 +59,12 @@ public sealed class JsonFeedExporter : IFeedExporter
var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
var advisoryStream = _advisoryStore.StreamAsync(cancellationToken);
var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false);
var digest = ExportDigestCalculator.ComputeTreeDigest(result);
_logger.LogInformation(
"JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
exportId,
@@ -106,7 +114,34 @@ public sealed class JsonFeedExporter : IFeedExporter
{
TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
}
}
}
private async Task<IReadOnlyList<Advisory>> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken)
{
var keys = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
{
keys.Add(advisory.AdvisoryKey.Trim());
}
}
var advisories = new List<Advisory>(keys.Count);
foreach (var key in keys)
{
cancellationToken.ThrowIfCancellationRequested();
var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false);
if (!replay.Statements.IsDefaultOrEmpty)
{
advisories.Add(replay.Statements[0].Advisory);
}
}
return advisories;
}
private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
{

View File

@@ -0,0 +1,622 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Models;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Exporter.Json;
internal static class JsonMirrorBundleWriter
{
private const int SchemaVersion = 1;
private const string BundleFileName = "bundle.json";
private const string BundleSignatureFileName = "bundle.json.jws";
private const string ManifestFileName = "manifest.json";
private const string IndexFileName = "index.json";
private const string SignatureMediaType = "application/vnd.stellaops.concelier.mirror-bundle+jws";
private const string DefaultMirrorDirectoryName = "mirror";
private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = null,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false,
};
public static async Task<JsonExportResult> WriteAsync(
JsonExportResult result,
JsonExportOptions options,
IServiceProvider services,
TimeProvider timeProvider,
ILogger logger,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(result);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(timeProvider);
ArgumentNullException.ThrowIfNull(logger);
var mirrorOptions = options.Mirror ?? new JsonExportOptions.JsonMirrorOptions();
if (!mirrorOptions.Enabled || mirrorOptions.Domains.Count == 0)
{
return result;
}
cancellationToken.ThrowIfCancellationRequested();
var exportedAtUtc = result.ExportedAt.UtcDateTime;
var mirrorDirectoryName = string.IsNullOrWhiteSpace(mirrorOptions.DirectoryName)
? DefaultMirrorDirectoryName
: mirrorOptions.DirectoryName.Trim();
var mirrorRoot = Path.Combine(result.ExportDirectory, mirrorDirectoryName);
Directory.CreateDirectory(mirrorRoot);
TrySetDirectoryTimestamp(mirrorRoot, exportedAtUtc);
var advisories = result.Advisories.IsDefaultOrEmpty
? Array.Empty<Advisory>()
: result.Advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
var signingContext = PrepareSigningContext(mirrorOptions.Signing, services, timeProvider, logger);
var additionalFiles = new List<JsonExportFile>();
var domainEntries = new List<MirrorIndexDomainEntry>();
foreach (var domainOption in mirrorOptions.Domains)
{
cancellationToken.ThrowIfCancellationRequested();
if (domainOption is null)
{
logger.LogWarning("Encountered null mirror domain configuration; skipping.");
continue;
}
var domainId = (domainOption.Id ?? string.Empty).Trim();
if (domainId.Length == 0)
{
logger.LogWarning("Skipping mirror domain with empty id.");
continue;
}
var schemeFilter = CreateFilterSet(domainOption.IncludeSchemes);
var sourceFilter = CreateFilterSet(domainOption.IncludeSources);
var domainAdvisories = advisories
.Where(advisory => MatchesFilters(advisory, schemeFilter, sourceFilter))
.ToArray();
var sources = BuildSourceSummaries(domainAdvisories);
var domainDisplayName = string.IsNullOrWhiteSpace(domainOption.DisplayName)
? domainId
: domainOption.DisplayName!.Trim();
var domainDirectory = Path.Combine(mirrorRoot, domainId);
Directory.CreateDirectory(domainDirectory);
TrySetDirectoryTimestamp(domainDirectory, exportedAtUtc);
var bundleDocument = new MirrorDomainBundleDocument(
SchemaVersion,
result.ExportedAt,
options.TargetRepository,
domainId,
domainDisplayName,
domainAdvisories.Length,
domainAdvisories,
sources);
var bundleBytes = Serialize(bundleDocument);
var bundlePath = Path.Combine(domainDirectory, BundleFileName);
await WriteFileAsync(bundlePath, bundleBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var bundleRelativePath = ToRelativePath(result.ExportDirectory, bundlePath);
var bundleDigest = ComputeDigest(bundleBytes);
var bundleLength = (long)bundleBytes.LongLength;
additionalFiles.Add(new JsonExportFile(bundleRelativePath, bundleLength, bundleDigest));
MirrorSignatureDescriptor? signatureDescriptor = null;
if (signingContext is not null)
{
var (signatureValue, signedAt) = await CreateSignatureAsync(
signingContext,
bundleBytes,
timeProvider,
cancellationToken)
.ConfigureAwait(false);
var signatureBytes = Utf8NoBom.GetBytes(signatureValue);
var signaturePath = Path.Combine(domainDirectory, BundleSignatureFileName);
await WriteFileAsync(signaturePath, signatureBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var signatureRelativePath = ToRelativePath(result.ExportDirectory, signaturePath);
var signatureDigest = ComputeDigest(signatureBytes);
var signatureLength = (long)signatureBytes.LongLength;
additionalFiles.Add(new JsonExportFile(signatureRelativePath, signatureLength, signatureDigest));
signatureDescriptor = new MirrorSignatureDescriptor(
signatureRelativePath,
signingContext.Algorithm,
signingContext.KeyId,
signingContext.Provider,
signedAt);
}
var bundleDescriptor = new MirrorFileDescriptor(bundleRelativePath, bundleLength, bundleDigest, signatureDescriptor);
var manifestDocument = new MirrorDomainManifestDocument(
SchemaVersion,
result.ExportedAt,
domainId,
domainDisplayName,
domainAdvisories.Length,
sources,
bundleDescriptor);
var manifestBytes = Serialize(manifestDocument);
var manifestPath = Path.Combine(domainDirectory, ManifestFileName);
await WriteFileAsync(manifestPath, manifestBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var manifestRelativePath = ToRelativePath(result.ExportDirectory, manifestPath);
var manifestDigest = ComputeDigest(manifestBytes);
var manifestLength = (long)manifestBytes.LongLength;
additionalFiles.Add(new JsonExportFile(manifestRelativePath, manifestLength, manifestDigest));
var manifestDescriptor = new MirrorFileDescriptor(manifestRelativePath, manifestLength, manifestDigest, null);
domainEntries.Add(new MirrorIndexDomainEntry(
domainId,
domainDisplayName,
domainAdvisories.Length,
manifestDescriptor,
bundleDescriptor,
sources));
}
domainEntries.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId));
var indexDocument = new MirrorIndexDocument(
SchemaVersion,
result.ExportedAt,
options.TargetRepository,
domainEntries);
var indexBytes = Serialize(indexDocument);
var indexPath = Path.Combine(mirrorRoot, IndexFileName);
await WriteFileAsync(indexPath, indexBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false);
var indexRelativePath = ToRelativePath(result.ExportDirectory, indexPath);
var indexDigest = ComputeDigest(indexBytes);
var indexLength = (long)indexBytes.LongLength;
additionalFiles.Add(new JsonExportFile(indexRelativePath, indexLength, indexDigest));
logger.LogInformation(
"Generated {DomainCount} Concelier mirror domain bundle(s) under {MirrorRoot}.",
domainEntries.Count,
mirrorDirectoryName);
var combinedFiles = new List<JsonExportFile>(result.Files.Length + additionalFiles.Count);
combinedFiles.AddRange(result.Files);
combinedFiles.AddRange(additionalFiles);
var combinedTotalBytes = checked(result.TotalBytes + additionalFiles.Sum(static file => file.Length));
return new JsonExportResult(
result.ExportDirectory,
result.ExportedAt,
combinedFiles,
result.AdvisoryCount,
combinedTotalBytes,
result.Advisories);
}
private static JsonMirrorSigningContext? PrepareSigningContext(
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
IServiceProvider services,
TimeProvider timeProvider,
ILogger logger)
{
if (signingOptions is null || !signingOptions.Enabled)
{
return null;
}
var algorithm = string.IsNullOrWhiteSpace(signingOptions.Algorithm)
? SignatureAlgorithms.Es256
: signingOptions.Algorithm.Trim();
var keyId = (signingOptions.KeyId ?? string.Empty).Trim();
if (keyId.Length == 0)
{
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyId to be configured.");
}
var registry = services.GetService<ICryptoProviderRegistry>()
?? throw new InvalidOperationException("Mirror signing requires ICryptoProviderRegistry to be registered.");
var providerHint = signingOptions.Provider?.Trim();
var keyReference = new CryptoKeyReference(keyId, providerHint);
CryptoSignerResolution resolved;
try
{
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, providerHint);
}
catch (KeyNotFoundException)
{
var provider = ResolveProvider(registry, algorithm, providerHint);
var signingKey = LoadSigningKey(signingOptions, provider, services, timeProvider, algorithm);
provider.UpsertSigningKey(signingKey);
resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, provider.Name);
}
logger.LogDebug(
"Mirror signing configured with key {KeyId} via provider {Provider} using {Algorithm}.",
resolved.Signer.KeyId,
resolved.ProviderName,
algorithm);
return new JsonMirrorSigningContext(resolved.Signer, algorithm, resolved.Signer.KeyId, resolved.ProviderName);
}
private static ICryptoProvider ResolveProvider(ICryptoProviderRegistry registry, string algorithm, string? providerHint)
{
if (!string.IsNullOrWhiteSpace(providerHint) && registry.TryResolve(providerHint, out var hinted))
{
if (!hinted.Supports(CryptoCapability.Signing, algorithm))
{
throw new InvalidOperationException(
$"Crypto provider '{providerHint}' does not support signing algorithm '{algorithm}'.");
}
return hinted;
}
return registry.ResolveOrThrow(CryptoCapability.Signing, algorithm);
}
private static CryptoSigningKey LoadSigningKey(
JsonExportOptions.JsonMirrorSigningOptions signingOptions,
ICryptoProvider provider,
IServiceProvider services,
TimeProvider timeProvider,
string algorithm)
{
var keyPath = (signingOptions.KeyPath ?? string.Empty).Trim();
if (keyPath.Length == 0)
{
throw new InvalidOperationException("Mirror signing requires mirror.signing.keyPath to be configured.");
}
var environment = services.GetService<IHostEnvironment>();
var basePath = environment?.ContentRootPath ?? AppContext.BaseDirectory;
var resolvedPath = Path.IsPathRooted(keyPath)
? keyPath
: Path.GetFullPath(Path.Combine(basePath, keyPath));
if (!File.Exists(resolvedPath))
{
throw new FileNotFoundException($"Mirror signing key '{signingOptions.KeyId}' not found.", resolvedPath);
}
var pem = File.ReadAllText(resolvedPath);
using var ecdsa = ECDsa.Create();
try
{
ecdsa.ImportFromPem(pem);
}
catch (CryptographicException ex)
{
throw new InvalidOperationException("Failed to import mirror signing key. Ensure the PEM contains an EC private key.", ex);
}
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
return new CryptoSigningKey(
new CryptoKeyReference(signingOptions.KeyId, provider.Name),
algorithm,
in parameters,
timeProvider.GetUtcNow());
}
private static async Task<(string Value, DateTimeOffset SignedAt)> CreateSignatureAsync(
JsonMirrorSigningContext context,
ReadOnlyMemory<byte> payload,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var header = new Dictionary<string, object>
{
["alg"] = context.Algorithm,
["kid"] = context.KeyId,
["typ"] = SignatureMediaType,
["b64"] = false,
["crit"] = new[] { "b64" }
};
if (!string.IsNullOrWhiteSpace(context.Provider))
{
header["provider"] = context.Provider;
}
var headerJson = JsonSerializer.Serialize(header, HeaderSerializerOptions);
var protectedHeader = Base64UrlEncode(Utf8NoBom.GetBytes(headerJson));
var signingInputLength = protectedHeader.Length + 1 + payload.Length;
var buffer = ArrayPool<byte>.Shared.Rent(signingInputLength);
try
{
var headerBytes = Encoding.ASCII.GetBytes(protectedHeader);
Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length);
buffer[headerBytes.Length] = (byte)'.';
var payloadArray = payload.ToArray();
Buffer.BlockCopy(payloadArray, 0, buffer, headerBytes.Length + 1, payloadArray.Length);
var signingInput = new ReadOnlyMemory<byte>(buffer, 0, signingInputLength);
var signatureBytes = await context.Signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false);
var encodedSignature = Base64UrlEncode(signatureBytes);
var signedAt = timeProvider.GetUtcNow();
return (string.Concat(protectedHeader, "..", encodedSignature), signedAt);
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
private static IReadOnlyList<JsonMirrorSourceSummary> BuildSourceSummaries(IReadOnlyList<Advisory> advisories)
{
var builders = new Dictionary<string, SourceAccumulator>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
var counted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var provenance in advisory.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
var source = provenance.Source.Trim();
if (!builders.TryGetValue(source, out var accumulator))
{
accumulator = new SourceAccumulator();
builders[source] = accumulator;
}
accumulator.Record(provenance.RecordedAt);
if (counted.Add(source))
{
accumulator.IncrementAdvisoryCount();
}
}
}
return builders
.OrderBy(static pair => pair.Key, StringComparer.Ordinal)
.Select(pair => new JsonMirrorSourceSummary(
pair.Key,
pair.Value.FirstRecordedAt,
pair.Value.LastRecordedAt,
pair.Value.AdvisoryCount))
.ToArray();
}
private static HashSet<string>? CreateFilterSet(IList<string>? values)
{
if (values is null || values.Count == 0)
{
return null;
}
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
set.Add(value.Trim());
}
return set.Count == 0 ? null : set;
}
private static bool MatchesFilters(Advisory advisory, HashSet<string>? schemeFilter, HashSet<string>? sourceFilter)
{
if (schemeFilter is not null)
{
var scheme = ExtractScheme(advisory.AdvisoryKey);
if (!schemeFilter.Contains(scheme))
{
return false;
}
}
if (sourceFilter is not null)
{
var hasSource = advisory.Provenance.Any(provenance =>
!string.IsNullOrWhiteSpace(provenance.Source) &&
sourceFilter.Contains(provenance.Source.Trim()));
if (!hasSource)
{
return false;
}
}
return true;
}
private static string ExtractScheme(string advisoryKey)
{
if (string.IsNullOrWhiteSpace(advisoryKey))
{
return string.Empty;
}
var trimmed = advisoryKey.Trim();
var separatorIndex = trimmed.IndexOf(':');
return separatorIndex <= 0 ? trimmed : trimmed[..separatorIndex];
}
private static byte[] Serialize<T>(T value)
{
var json = CanonicalJsonSerializer.SerializeIndented(value);
return Utf8NoBom.GetBytes(json);
}
private static async Task WriteFileAsync(string path, byte[] content, DateTime exportedAtUtc, CancellationToken cancellationToken)
{
await File.WriteAllBytesAsync(path, content, cancellationToken).ConfigureAwait(false);
File.SetLastWriteTimeUtc(path, exportedAtUtc);
}
private static string ToRelativePath(string root, string fullPath)
{
var relative = Path.GetRelativePath(root, fullPath);
return relative.Replace(Path.DirectorySeparatorChar, '/');
}
private static string ComputeDigest(ReadOnlySpan<byte> payload)
{
var hash = SHA256.HashData(payload);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static void TrySetDirectoryTimestamp(string directory, DateTime exportedAtUtc)
{
try
{
Directory.SetLastWriteTimeUtc(directory, exportedAtUtc);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
catch (PlatformNotSupportedException)
{
}
}
private static string Base64UrlEncode(ReadOnlySpan<byte> value)
{
var encoded = Convert.ToBase64String(value);
var builder = new StringBuilder(encoded.Length);
foreach (var ch in encoded)
{
switch (ch)
{
case '+':
builder.Append('-');
break;
case '/':
builder.Append('_');
break;
case '=':
break;
default:
builder.Append(ch);
break;
}
}
return builder.ToString();
}
private sealed record JsonMirrorSigningContext(ICryptoSigner Signer, string Algorithm, string KeyId, string Provider);
private sealed record MirrorIndexDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string? TargetRepository,
IReadOnlyList<MirrorIndexDomainEntry> Domains);
private sealed record MirrorIndexDomainEntry(
string DomainId,
string DisplayName,
int AdvisoryCount,
MirrorFileDescriptor Manifest,
MirrorFileDescriptor Bundle,
IReadOnlyList<JsonMirrorSourceSummary> Sources);
private sealed record MirrorDomainManifestDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string DomainId,
string DisplayName,
int AdvisoryCount,
IReadOnlyList<JsonMirrorSourceSummary> Sources,
MirrorFileDescriptor Bundle);
private sealed record MirrorDomainBundleDocument(
int SchemaVersion,
DateTimeOffset GeneratedAt,
string? TargetRepository,
string DomainId,
string DisplayName,
int AdvisoryCount,
IReadOnlyList<Advisory> Advisories,
IReadOnlyList<JsonMirrorSourceSummary> Sources);
private sealed record MirrorFileDescriptor(
string Path,
long SizeBytes,
string Digest,
MirrorSignatureDescriptor? Signature);
private sealed record MirrorSignatureDescriptor(
string Path,
string Algorithm,
string KeyId,
string Provider,
DateTimeOffset SignedAt);
private sealed record JsonMirrorSourceSummary(
string Source,
DateTimeOffset? FirstRecordedAt,
DateTimeOffset? LastRecordedAt,
int AdvisoryCount);
private sealed class SourceAccumulator
{
public DateTimeOffset? FirstRecordedAt { get; private set; }
public DateTimeOffset? LastRecordedAt { get; private set; }
public int AdvisoryCount { get; private set; }
public void Record(DateTimeOffset recordedAt)
{
var normalized = recordedAt.ToUniversalTime();
if (FirstRecordedAt is null || normalized < FirstRecordedAt.Value)
{
FirstRecordedAt = normalized;
}
if (LastRecordedAt is null || normalized > LastRecordedAt.Value)
{
LastRecordedAt = normalized;
}
}
public void IncrementAdvisoryCount()
{
AdvisoryCount++;
}
}
}

View File

@@ -10,13 +10,15 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
</ItemGroup>
</Project>
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" />
<ProjectReference Include="..\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -10,4 +10,4 @@
|Stream advisories during export|BE-Export|Storage.Mongo|DONE exporter + streaming-only test ensures single enumeration and per-file digest capture.|
|Emit export manifest with digest metadata|BE-Export|Exporters|DONE manifest now includes per-file digests/sizes alongside tree digest.|
|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.|
|CONCELIER-EXPORT-08-201 Mirror bundle + domain manifest|Team Concelier Export|FEEDCORE-ENGINE-07-001|TODO Produce per-domain aggregate bundles (JSON + manifest) with deterministic digests, include upstream source metadata, and publish index consumed by mirror endpoints/tests.|
|CONCELIER-EXPORT-08-201 Mirror bundle + domain manifest|Team Concelier Export|FEEDCORE-ENGINE-07-001|DONE (2025-10-19) Mirror bundle writer emits domain aggregates + manifests with cosign-compatible JWS signatures; index/tests updated via `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj` (2025-10-19).|