using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Feedser.Exporter.Json; using StellaOps.Feedser.Models; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Exporting; namespace StellaOps.Feedser.Exporter.Json.Tests; public sealed class JsonFeedExporterTests : IDisposable { private readonly string _root; public JsonFeedExporterTests() { _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName; } [Fact] public async Task ExportAsync_SkipsWhenDigestUnchanged() { var advisory = new Advisory( advisoryKey: "CVE-2024-1234", title: "Test Advisory", summary: null, language: "en", published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), severity: "high", exploitKnown: false, aliases: new[] { "CVE-2024-1234" }, references: Array.Empty(), affectedPackages: Array.Empty(), cvssMetrics: Array.Empty(), provenance: Array.Empty()); var advisoryStore = new StubAdvisoryStore(advisory); var options = Options.Create(new JsonExportOptions { OutputRoot = _root, MaintainLatestSymlink = false, }); var stateStore = new InMemoryExportStateStore(); var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture)); var stateManager = new ExportStateManager(stateStore, timeProvider); var exporter = new JsonFeedExporter( advisoryStore, options, new VulnListJsonExportPathResolver(), stateManager, NullLogger.Instance, timeProvider); using var provider = new ServiceCollection().BuildServiceProvider(); await exporter.ExportAsync(provider, CancellationToken.None); var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); Assert.NotNull(record); var firstUpdated = record!.UpdatedAt; Assert.Equal("20240715T120000Z", record.BaseExportId); Assert.Equal(record.LastFullDigest, record.ExportCursor); var firstExportPath = Path.Combine(_root, "20240715T120000Z"); Assert.True(Directory.Exists(firstExportPath)); timeProvider.Advance(TimeSpan.FromMinutes(5)); await exporter.ExportAsync(provider, CancellationToken.None); record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); Assert.NotNull(record); Assert.Equal(firstUpdated, record!.UpdatedAt); var secondExportPath = Path.Combine(_root, "20240715T120500Z"); Assert.False(Directory.Exists(secondExportPath)); } [Fact] public async Task ExportAsync_WritesManifestMetadata() { var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture); var advisory = new Advisory( advisoryKey: "CVE-2024-4321", title: "Manifest Test", summary: null, language: "en", published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture), modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture), severity: "medium", exploitKnown: false, aliases: new[] { "CVE-2024-4321" }, references: Array.Empty(), affectedPackages: Array.Empty(), cvssMetrics: Array.Empty(), provenance: Array.Empty()); var advisoryStore = new StubAdvisoryStore(advisory); var optionsValue = new JsonExportOptions { OutputRoot = _root, MaintainLatestSymlink = false, }; var options = Options.Create(optionsValue); var stateStore = new InMemoryExportStateStore(); var timeProvider = new TestTimeProvider(exportedAt); var stateManager = new ExportStateManager(stateStore, timeProvider); var exporter = new JsonFeedExporter( advisoryStore, options, new VulnListJsonExportPathResolver(), stateManager, NullLogger.Instance, timeProvider); using var provider = new ServiceCollection().BuildServiceProvider(); await exporter.ExportAsync(provider, CancellationToken.None); var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); var exportDirectory = Path.Combine(_root, exportId); var manifestPath = Path.Combine(exportDirectory, "manifest.json"); Assert.True(File.Exists(manifestPath)); using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None)); var root = document.RootElement; Assert.Equal(exportId, root.GetProperty("exportId").GetString()); Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime()); Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32()); var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories) .Select(path => new { Absolute = path, Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal), }) .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase)) .OrderBy(file => file.Relative, StringComparer.Ordinal) .ToArray(); var filesElement = root.GetProperty("files") .EnumerateArray() .Select(element => new { Path = element.GetProperty("path").GetString(), Bytes = element.GetProperty("bytes").GetInt64(), Digest = element.GetProperty("digest").GetString(), }) .OrderBy(file => file.Path, StringComparer.Ordinal) .ToArray(); Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray()); long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum(); Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64()); Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32()); var digest = root.GetProperty("digest").GetString(); var digestResult = new JsonExportResult( exportDirectory, exportedAt, exportedFiles.Select(file => { var manifestEntry = filesElement.First(f => f.Path == file.Relative); if (manifestEntry.Digest is null) { throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest."); } return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest); }), exportedFiles.Length, totalBytes); var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult); Assert.Equal(expectedDigest, digest); var exporterVersion = root.GetProperty("exporterVersion").GetString(); Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion); } public void Dispose() { try { if (Directory.Exists(_root)) { Directory.Delete(_root, recursive: true); } } catch { // best effort cleanup } } private sealed class StubAdvisoryStore : IAdvisoryStore { private readonly IReadOnlyList _advisories; public StubAdvisoryStore(params Advisory[] advisories) { _advisories = advisories; } public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) => Task.FromResult(_advisories); public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) => Task.CompletedTask; public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) { return EnumerateAsync(cancellationToken); async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) { foreach (var advisory in _advisories) { ct.ThrowIfCancellationRequested(); yield return advisory; await Task.Yield(); } } } } private sealed class InMemoryExportStateStore : IExportStateStore { private ExportStateRecord? _record; public Task FindAsync(string id, CancellationToken cancellationToken) => Task.FromResult(_record); public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) { _record = record; return Task.FromResult(record); } } private sealed class TestTimeProvider : TimeProvider { private DateTimeOffset _now; public TestTimeProvider(DateTimeOffset start) => _now = start; public override DateTimeOffset GetUtcNow() => _now; public void Advance(TimeSpan delta) => _now = _now.Add(delta); } }