git.stella-ops.org/src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs

using System;
using System.Globalization;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.Events;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Exporting;
using StellaOps.Plugin;

namespace StellaOps.Concelier.Exporter.Json;

public sealed class JsonFeedExporter : IFeedExporter
{
    public const string ExporterName = "json";
    public const string ExporterId = "export:json";

    private readonly IAdvisoryStore _advisoryStore;
    private readonly JsonExportOptions _options;
    private readonly IJsonExportPathResolver _pathResolver;
    private readonly ExportStateManager _stateManager;
    private readonly ILogger<JsonFeedExporter> _logger;
    private readonly TimeProvider _timeProvider;
    private readonly string _exporterVersion;
    private readonly IAdvisoryEventLog _eventLog;

    public JsonFeedExporter(
        IAdvisoryStore advisoryStore,
        IOptions<JsonExportOptions> options,
        IJsonExportPathResolver pathResolver,
        ExportStateManager stateManager,
        IAdvisoryEventLog eventLog,
        ILogger<JsonFeedExporter> logger,
        TimeProvider? timeProvider = null)
    {
        _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
        _options = options?.Value ?? throw new ArgumentNullException(nameof(options));
        _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver));
        _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager));
        _eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _timeProvider = timeProvider ?? TimeProvider.System;
        _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter));
    }

    public string Name => ExporterName;

    public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        var exportedAt = _timeProvider.GetUtcNow();
        var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture);
        var exportRoot = Path.GetFullPath(_options.OutputRoot);

        _logger.LogInformation("Starting JSON export {ExportId}", exportId);

        var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false);

        var builder = new JsonExportSnapshotBuilder(_options, _pathResolver);
        var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false);
        var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false);
        result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false);

        var digest = ExportDigestCalculator.ComputeTreeDigest(result);
        _logger.LogInformation(
            "JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}",
            exportId,
            result.Files.Length,
            result.TotalBytes,
            result.AdvisoryCount,
            digest);

        var manifest = result.Files
            .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest))
            .ToArray();

        if (existingState is not null
            && existingState.Files.Count > 0
            && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal))
        {
            _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId);
            TryDeleteDirectory(result.ExportDirectory);
            return;
        }

        var resetBaseline = existingState is null
            || string.IsNullOrWhiteSpace(existingState.BaseExportId)
            || string.IsNullOrWhiteSpace(existingState.BaseDigest);

        if (existingState is not null
            && !string.IsNullOrWhiteSpace(_options.TargetRepository)
            && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
        {
            resetBaseline = true;
        }

        await _stateManager.StoreFullExportAsync(
            ExporterId,
            exportId,
            digest,
            cursor: digest,
            targetRepository: _options.TargetRepository,
            exporterVersion: _exporterVersion,
            resetBaseline: resetBaseline,
            manifest: manifest,
            cancellationToken: cancellationToken).ConfigureAwait(false);

        await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);

        if (_options.MaintainLatestSymlink)
        {
            TryUpdateLatestSymlink(exportRoot, result.ExportDirectory);
        }
    }

    private async Task<IReadOnlyList<Advisory>> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken)
    {
        var keys = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);

        await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken))
        {
            cancellationToken.ThrowIfCancellationRequested();
            if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
            {
                keys.Add(advisory.AdvisoryKey.Trim());
            }
        }

        var advisories = new List<Advisory>(keys.Count);
        foreach (var key in keys)
        {
            cancellationToken.ThrowIfCancellationRequested();
            var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false);
            if (!replay.Statements.IsDefaultOrEmpty)
            {
                advisories.Add(replay.Statements[0].Advisory);
            }
        }

        return advisories;
    }

    private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory)
    {
        if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName))
        {
            return;
        }

        var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName);

        try
        {
            if (Directory.Exists(latestPath) || File.Exists(latestPath))
            {
                TryRemoveExistingPointer(latestPath);
            }

            Directory.CreateSymbolicLink(latestPath, exportDirectory);
            _logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory);
        }
        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException)
        {
            _logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath);
        }
    }

    private void TryRemoveExistingPointer(string latestPath)
    {
        try
        {
            var attributes = File.GetAttributes(latestPath);
            if (attributes.HasFlag(FileAttributes.Directory))
            {
                Directory.Delete(latestPath, recursive: false);
            }
            else
            {
                File.Delete(latestPath);
            }
        }
        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
        {
            _logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath);
        }
    }

    private void TryDeleteDirectory(string path)
    {
        try
        {
            if (Directory.Exists(path))
            {
                Directory.Delete(path, recursive: true);
            }
        }
        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
        {
            _logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path);
        }
    }
}