git.stella-ops.org/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Exporter.Json;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Exporting;

namespace StellaOps.Feedser.Exporter.Json.Tests;

public sealed class JsonFeedExporterTests : IDisposable
{
    private readonly string _root;

    public JsonFeedExporterTests()
    {
        _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName;
    }

    [Fact]
    public async Task ExportAsync_SkipsWhenDigestUnchanged()
    {
        var advisory = new Advisory(
            advisoryKey: "CVE-2024-1234",
            title: "Test Advisory",
            summary: null,
            language: "en",
            published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture),
            modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture),
            severity: "high",
            exploitKnown: false,
            aliases: new[] { "CVE-2024-1234" },
            references: Array.Empty<AdvisoryReference>(),
            affectedPackages: Array.Empty<AffectedPackage>(),
            cvssMetrics: Array.Empty<CvssMetric>(),
            provenance: Array.Empty<AdvisoryProvenance>());

        var advisoryStore = new StubAdvisoryStore(advisory);
        var options = Options.Create(new JsonExportOptions
        {
            OutputRoot = _root,
            MaintainLatestSymlink = false,
        });

        var stateStore = new InMemoryExportStateStore();
        var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture));
        var stateManager = new ExportStateManager(stateStore, timeProvider);
        var exporter = new JsonFeedExporter(
            advisoryStore,
            options,
            new VulnListJsonExportPathResolver(),
            stateManager,
            NullLogger<JsonFeedExporter>.Instance,
            timeProvider);

        using var provider = new ServiceCollection().BuildServiceProvider();
        await exporter.ExportAsync(provider, CancellationToken.None);

        var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
        Assert.NotNull(record);
        var firstUpdated = record!.UpdatedAt;
        Assert.Equal("20240715T120000Z", record.BaseExportId);
        Assert.Equal(record.LastFullDigest, record.ExportCursor);

        var firstExportPath = Path.Combine(_root, "20240715T120000Z");
        Assert.True(Directory.Exists(firstExportPath));

        timeProvider.Advance(TimeSpan.FromMinutes(5));
        await exporter.ExportAsync(provider, CancellationToken.None);

        record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
        Assert.NotNull(record);
        Assert.Equal(firstUpdated, record!.UpdatedAt);

        var secondExportPath = Path.Combine(_root, "20240715T120500Z");
        Assert.False(Directory.Exists(secondExportPath));
    }

    [Fact]
    public async Task ExportAsync_WritesManifestMetadata()
    {
        var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture);
        var advisory = new Advisory(
            advisoryKey: "CVE-2024-4321",
            title: "Manifest Test",
            summary: null,
            language: "en",
            published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture),
            modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture),
            severity: "medium",
            exploitKnown: false,
            aliases: new[] { "CVE-2024-4321" },
            references: Array.Empty<AdvisoryReference>(),
            affectedPackages: Array.Empty<AffectedPackage>(),
            cvssMetrics: Array.Empty<CvssMetric>(),
            provenance: Array.Empty<AdvisoryProvenance>());

        var advisoryStore = new StubAdvisoryStore(advisory);
        var optionsValue = new JsonExportOptions
        {
            OutputRoot = _root,
            MaintainLatestSymlink = false,
        };

        var options = Options.Create(optionsValue);
        var stateStore = new InMemoryExportStateStore();
        var timeProvider = new TestTimeProvider(exportedAt);
        var stateManager = new ExportStateManager(stateStore, timeProvider);
        var exporter = new JsonFeedExporter(
            advisoryStore,
            options,
            new VulnListJsonExportPathResolver(),
            stateManager,
            NullLogger<JsonFeedExporter>.Instance,
            timeProvider);

        using var provider = new ServiceCollection().BuildServiceProvider();
        await exporter.ExportAsync(provider, CancellationToken.None);

        var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture);
        var exportDirectory = Path.Combine(_root, exportId);
        var manifestPath = Path.Combine(exportDirectory, "manifest.json");

        Assert.True(File.Exists(manifestPath));

        using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None));
        var root = document.RootElement;

        Assert.Equal(exportId, root.GetProperty("exportId").GetString());
        Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime());
        Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32());

        var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories)
            .Select(path => new
            {
                Absolute = path,
                Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal),
            })
            .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase))
            .OrderBy(file => file.Relative, StringComparer.Ordinal)
            .ToArray();

        var filesElement = root.GetProperty("files")
            .EnumerateArray()
            .Select(element => new
            {
                Path = element.GetProperty("path").GetString(),
                Bytes = element.GetProperty("bytes").GetInt64(),
                Digest = element.GetProperty("digest").GetString(),
            })
            .OrderBy(file => file.Path, StringComparer.Ordinal)
            .ToArray();

        Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray());

        long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum();
        Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64());
        Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32());

        var digest = root.GetProperty("digest").GetString();
        var digestResult = new JsonExportResult(
            exportDirectory,
            exportedAt,
            exportedFiles.Select(file =>
            {
                var manifestEntry = filesElement.First(f => f.Path == file.Relative);
                if (manifestEntry.Digest is null)
                {
                    throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest.");
                }

                return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest);
            }),
            exportedFiles.Length,
            totalBytes);
        var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult);
        Assert.Equal(expectedDigest, digest);

        var exporterVersion = root.GetProperty("exporterVersion").GetString();
        Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion);
    }

    public void Dispose()
    {
        try
        {
            if (Directory.Exists(_root))
            {
                Directory.Delete(_root, recursive: true);
            }
        }
        catch
        {
            // best effort cleanup
        }
    }

    private sealed class StubAdvisoryStore : IAdvisoryStore
    {
        private readonly IReadOnlyList<Advisory> _advisories;

        public StubAdvisoryStore(params Advisory[] advisories)
        {
            _advisories = advisories;
        }

        public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
            => Task.FromResult(_advisories);

        public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
            => Task.FromResult<Advisory?>(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey));

        public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken)
            => Task.CompletedTask;

        public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken)
        {
            return EnumerateAsync(cancellationToken);

            async IAsyncEnumerable<Advisory> EnumerateAsync([EnumeratorCancellation] CancellationToken ct)
            {
                foreach (var advisory in _advisories)
                {
                    ct.ThrowIfCancellationRequested();
                    yield return advisory;
                    await Task.Yield();
                }
            }
        }
    }

    private sealed class InMemoryExportStateStore : IExportStateStore
    {
        private ExportStateRecord? _record;

        public Task<ExportStateRecord?> FindAsync(string id, CancellationToken cancellationToken)
            => Task.FromResult(_record);

        public Task<ExportStateRecord> UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken)
        {
            _record = record;
            return Task.FromResult(record);
        }
    }

    private sealed class TestTimeProvider : TimeProvider
    {
        private DateTimeOffset _now;

        public TestTimeProvider(DateTimeOffset start) => _now = start;

        public override DateTimeOffset GetUtcNow() => _now;

        public void Advance(TimeSpan delta) => _now = _now.Add(delta);
    }
}