using System; using System.Globalization; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Concelier.Core.Events; using StellaOps.Concelier.Models; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Exporting; using StellaOps.Plugin; namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonFeedExporter : IFeedExporter { public const string ExporterName = "json"; public const string ExporterId = "export:json"; private readonly IAdvisoryStore _advisoryStore; private readonly JsonExportOptions _options; private readonly IJsonExportPathResolver _pathResolver; private readonly ExportStateManager _stateManager; private readonly ILogger _logger; private readonly TimeProvider _timeProvider; private readonly string _exporterVersion; private readonly IAdvisoryEventLog _eventLog; public JsonFeedExporter( IAdvisoryStore advisoryStore, IOptions options, IJsonExportPathResolver pathResolver, ExportStateManager stateManager, IAdvisoryEventLog eventLog, ILogger logger, TimeProvider? timeProvider = null) { _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); _eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter)); } public string Name => ExporterName; public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) { var exportedAt = _timeProvider.GetUtcNow(); var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture); var exportRoot = Path.GetFullPath(_options.OutputRoot); _logger.LogInformation("Starting JSON export {ExportId}", exportId); var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); var builder = new JsonExportSnapshotBuilder(_options, _pathResolver); var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false); var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false); result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false); var digest = ExportDigestCalculator.ComputeTreeDigest(result); _logger.LogInformation( "JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}", exportId, result.Files.Length, result.TotalBytes, result.AdvisoryCount, digest); var manifest = result.Files .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) .ToArray(); if (existingState is not null && existingState.Files.Count > 0 && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) { _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId); TryDeleteDirectory(result.ExportDirectory); return; } var resetBaseline = existingState is null || string.IsNullOrWhiteSpace(existingState.BaseExportId) || string.IsNullOrWhiteSpace(existingState.BaseDigest); if (existingState is not null && !string.IsNullOrWhiteSpace(_options.TargetRepository) && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) { resetBaseline = true; } await _stateManager.StoreFullExportAsync( ExporterId, exportId, digest, cursor: digest, targetRepository: _options.TargetRepository, exporterVersion: _exporterVersion, resetBaseline: resetBaseline, manifest: manifest, cancellationToken: cancellationToken).ConfigureAwait(false); await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); if (_options.MaintainLatestSymlink) { TryUpdateLatestSymlink(exportRoot, result.ExportDirectory); } } private async Task> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken) { var keys = new SortedSet(StringComparer.OrdinalIgnoreCase); await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) { keys.Add(advisory.AdvisoryKey.Trim()); } } var advisories = new List(keys.Count); foreach (var key in keys) { cancellationToken.ThrowIfCancellationRequested(); var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false); if (!replay.Statements.IsDefaultOrEmpty) { advisories.Add(replay.Statements[0].Advisory); } } return advisories; } private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory) { if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName)) { return; } var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName); try { if (Directory.Exists(latestPath) || File.Exists(latestPath)) { TryRemoveExistingPointer(latestPath); } Directory.CreateSymbolicLink(latestPath, exportDirectory); _logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) { _logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath); } } private void TryRemoveExistingPointer(string latestPath) { try { var attributes = File.GetAttributes(latestPath); if (attributes.HasFlag(FileAttributes.Directory)) { Directory.Delete(latestPath, recursive: false); } else { File.Delete(latestPath); } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { _logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath); } } private void TryDeleteDirectory(string path) { try { if (Directory.Exists(path)) { Directory.Delete(path, recursive: true); } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { _logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path); } } }