Initial commit (history squashed)

src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs

@@ -0,0 +1,265 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using StellaOps.Feedser.Exporter.Json;
+using StellaOps.Feedser.Models;
+using StellaOps.Feedser.Storage.Mongo.Advisories;
+using StellaOps.Feedser.Storage.Mongo.Exporting;
+
+namespace StellaOps.Feedser.Exporter.Json.Tests;
+
+public sealed class JsonFeedExporterTests : IDisposable
+{
+    private readonly string _root;
+
+    public JsonFeedExporterTests()
+    {
+        _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName;
+    }
+
+    [Fact]
+    public async Task ExportAsync_SkipsWhenDigestUnchanged()
+    {
+        var advisory = new Advisory(
+            advisoryKey: "CVE-2024-1234",
+            title: "Test Advisory",
+            summary: null,
+            language: "en",
+            published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture),
+            modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture),
+            severity: "high",
+            exploitKnown: false,
+            aliases: new[] { "CVE-2024-1234" },
+            references: Array.Empty<AdvisoryReference>(),
+            affectedPackages: Array.Empty<AffectedPackage>(),
+            cvssMetrics: Array.Empty<CvssMetric>(),
+            provenance: Array.Empty<AdvisoryProvenance>());
+
+        var advisoryStore = new StubAdvisoryStore(advisory);
+        var options = Options.Create(new JsonExportOptions
+        {
+            OutputRoot = _root,
+            MaintainLatestSymlink = false,
+        });
+
+        var stateStore = new InMemoryExportStateStore();
+        var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture));
+        var stateManager = new ExportStateManager(stateStore, timeProvider);
+        var exporter = new JsonFeedExporter(
+            advisoryStore,
+            options,
+            new VulnListJsonExportPathResolver(),
+            stateManager,
+            NullLogger<JsonFeedExporter>.Instance,
+            timeProvider);
+
+        using var provider = new ServiceCollection().BuildServiceProvider();
+        await exporter.ExportAsync(provider, CancellationToken.None);
+
+        var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
+        Assert.NotNull(record);
+        var firstUpdated = record!.UpdatedAt;
+        Assert.Equal("20240715T120000Z", record.BaseExportId);
+        Assert.Equal(record.LastFullDigest, record.ExportCursor);
+
+        var firstExportPath = Path.Combine(_root, "20240715T120000Z");
+        Assert.True(Directory.Exists(firstExportPath));
+
+        timeProvider.Advance(TimeSpan.FromMinutes(5));
+        await exporter.ExportAsync(provider, CancellationToken.None);
+
+        record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
+        Assert.NotNull(record);
+        Assert.Equal(firstUpdated, record!.UpdatedAt);
+
+        var secondExportPath = Path.Combine(_root, "20240715T120500Z");
+        Assert.False(Directory.Exists(secondExportPath));
+    }
+
+    [Fact]
+    public async Task ExportAsync_WritesManifestMetadata()
+    {
+        var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture);
+        var advisory = new Advisory(
+            advisoryKey: "CVE-2024-4321",
+            title: "Manifest Test",
+            summary: null,
+            language: "en",
+            published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture),
+            modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture),
+            severity: "medium",
+            exploitKnown: false,
+            aliases: new[] { "CVE-2024-4321" },
+            references: Array.Empty<AdvisoryReference>(),
+            affectedPackages: Array.Empty<AffectedPackage>(),
+            cvssMetrics: Array.Empty<CvssMetric>(),
+            provenance: Array.Empty<AdvisoryProvenance>());
+
+        var advisoryStore = new StubAdvisoryStore(advisory);
+        var optionsValue = new JsonExportOptions
+        {
+            OutputRoot = _root,
+            MaintainLatestSymlink = false,
+        };
+
+        var options = Options.Create(optionsValue);
+        var stateStore = new InMemoryExportStateStore();
+        var timeProvider = new TestTimeProvider(exportedAt);
+        var stateManager = new ExportStateManager(stateStore, timeProvider);
+        var exporter = new JsonFeedExporter(
+            advisoryStore,
+            options,
+            new VulnListJsonExportPathResolver(),
+            stateManager,
+            NullLogger<JsonFeedExporter>.Instance,
+            timeProvider);
+
+        using var provider = new ServiceCollection().BuildServiceProvider();
+        await exporter.ExportAsync(provider, CancellationToken.None);
+
+        var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture);
+        var exportDirectory = Path.Combine(_root, exportId);
+        var manifestPath = Path.Combine(exportDirectory, "manifest.json");
+
+        Assert.True(File.Exists(manifestPath));
+
+        using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None));
+        var root = document.RootElement;
+
+        Assert.Equal(exportId, root.GetProperty("exportId").GetString());
+        Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime());
+        Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32());
+
+        var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories)
+            .Select(path => new
+            {
+                Absolute = path,
+                Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal),
+            })
+            .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase))
+            .OrderBy(file => file.Relative, StringComparer.Ordinal)
+            .ToArray();
+
+        var filesElement = root.GetProperty("files")
+            .EnumerateArray()
+            .Select(element => new
+            {
+                Path = element.GetProperty("path").GetString(),
+                Bytes = element.GetProperty("bytes").GetInt64(),
+                Digest = element.GetProperty("digest").GetString(),
+            })
+            .OrderBy(file => file.Path, StringComparer.Ordinal)
+            .ToArray();
+
+        Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray());
+
+        long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum();
+        Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64());
+        Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32());
+
+        var digest = root.GetProperty("digest").GetString();
+        var digestResult = new JsonExportResult(
+            exportDirectory,
+            exportedAt,
+            exportedFiles.Select(file =>
+            {
+                var manifestEntry = filesElement.First(f => f.Path == file.Relative);
+                if (manifestEntry.Digest is null)
+                {
+                    throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest.");
+                }
+
+                return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest);
+            }),
+            exportedFiles.Length,
+            totalBytes);
+        var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult);
+        Assert.Equal(expectedDigest, digest);
+
+        var exporterVersion = root.GetProperty("exporterVersion").GetString();
+        Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion);
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            if (Directory.Exists(_root))
+            {
+                Directory.Delete(_root, recursive: true);
+            }
+        }
+        catch
+        {
+            // best effort cleanup
+        }
+    }
+
+    private sealed class StubAdvisoryStore : IAdvisoryStore
+    {
+        private readonly IReadOnlyList<Advisory> _advisories;
+
+        public StubAdvisoryStore(params Advisory[] advisories)
+        {
+            _advisories = advisories;
+        }
+
+        public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
+            => Task.FromResult(_advisories);
+
+        public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
+            => Task.FromResult<Advisory?>(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey));
+
+        public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken)
+            => Task.CompletedTask;
+
+        public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken)
+        {
+            return EnumerateAsync(cancellationToken);
+
+            async IAsyncEnumerable<Advisory> EnumerateAsync([EnumeratorCancellation] CancellationToken ct)
+            {
+                foreach (var advisory in _advisories)
+                {
+                    ct.ThrowIfCancellationRequested();
+                    yield return advisory;
+                    await Task.Yield();
+                }
+            }
+        }
+    }
+
+    private sealed class InMemoryExportStateStore : IExportStateStore
+    {
+        private ExportStateRecord? _record;
+
+        public Task<ExportStateRecord?> FindAsync(string id, CancellationToken cancellationToken)
+            => Task.FromResult(_record);
+
+        public Task<ExportStateRecord> UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken)
+        {
+            _record = record;
+            return Task.FromResult(record);
+        }
+    }
+
+    private sealed class TestTimeProvider : TimeProvider
+    {
+        private DateTimeOffset _now;
+
+        public TestTimeProvider(DateTimeOffset start) => _now = start;
+
+        public override DateTimeOffset GetUtcNow() => _now;
+
+        public void Advance(TimeSpan delta) => _now = _now.Add(delta);
+    }
+}