Initial commit (history squashed)
This commit is contained in:
Vladimir Moushkov
@@ -0,0 +1,213 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Exporter.Json;
|
||||
using StellaOps.Feedser.Models;
|
||||
|
||||
namespace StellaOps.Feedser.Exporter.Json.Tests;
|
||||
|
||||
public sealed class JsonExportSnapshotBuilderTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public JsonExportSnapshotBuilderTests()
|
||||
{
|
||||
_root = Directory.CreateTempSubdirectory("feedser-json-export-tests").FullName;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WritesDeterministicTree()
|
||||
{
|
||||
var options = new JsonExportOptions { OutputRoot = _root };
|
||||
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
|
||||
var exportedAt = DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
var advisories = new[]
|
||||
{
|
||||
CreateAdvisory(
|
||||
advisoryKey: "CVE-2024-9999",
|
||||
aliases: new[] { "GHSA-zzzz-yyyy-xxxx", "CVE-2024-9999" },
|
||||
title: "Deterministic Snapshot",
|
||||
severity: "critical"),
|
||||
CreateAdvisory(
|
||||
advisoryKey: "VENDOR-2024-42",
|
||||
aliases: new[] { "ALIAS-1", "ALIAS-2" },
|
||||
title: "Vendor Advisory",
|
||||
severity: "medium"),
|
||||
};
|
||||
|
||||
var result = await builder.WriteAsync(advisories, exportedAt, cancellationToken: CancellationToken.None);
|
||||
|
||||
Assert.Equal(advisories.Length, result.AdvisoryCount);
|
||||
Assert.Equal(exportedAt, result.ExportedAt);
|
||||
|
||||
var expectedFiles = result.FilePaths.OrderBy(x => x, StringComparer.Ordinal).ToArray();
|
||||
Assert.Contains("nvd/2024/CVE-2024-9999.json", expectedFiles);
|
||||
Assert.Contains("misc/VENDOR-2024-42.json", expectedFiles);
|
||||
|
||||
var cvePath = ResolvePath(result.ExportDirectory, "nvd/2024/CVE-2024-9999.json");
|
||||
Assert.True(File.Exists(cvePath));
|
||||
var actualJson = await File.ReadAllTextAsync(cvePath, CancellationToken.None);
|
||||
Assert.Equal(SnapshotSerializer.ToSnapshot(advisories[0]), actualJson);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProducesIdenticalBytesAcrossRuns()
|
||||
{
|
||||
var options = new JsonExportOptions { OutputRoot = _root };
|
||||
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
|
||||
var exportedAt = DateTimeOffset.Parse("2024-05-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
var advisories = new[]
|
||||
{
|
||||
CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000", "GHSA-aaaa-bbbb-cccc" }, "Snapshot Stable", "high"),
|
||||
};
|
||||
|
||||
var first = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None);
|
||||
var firstDigest = ComputeDigest(first);
|
||||
|
||||
var second = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None);
|
||||
var secondDigest = ComputeDigest(second);
|
||||
|
||||
Assert.Equal(Convert.ToHexString(firstDigest), Convert.ToHexString(secondDigest));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteAsync_NormalizesInputOrdering()
|
||||
{
|
||||
var options = new JsonExportOptions { OutputRoot = _root };
|
||||
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
|
||||
var exportedAt = DateTimeOffset.Parse("2024-06-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
var advisoryA = CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000" }, "Alpha", "high");
|
||||
var advisoryB = CreateAdvisory("VENDOR-0001", new[] { "VENDOR-0001" }, "Vendor Advisory", "medium");
|
||||
|
||||
var result = await builder.WriteAsync(new[] { advisoryB, advisoryA }, exportedAt, cancellationToken: CancellationToken.None);
|
||||
|
||||
var expectedOrder = result.FilePaths.OrderBy(path => path, StringComparer.Ordinal).ToArray();
|
||||
Assert.Equal(expectedOrder, result.FilePaths.ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriteAsync_EnumeratesStreamOnlyOnce()
|
||||
{
|
||||
var options = new JsonExportOptions { OutputRoot = _root };
|
||||
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
|
||||
var exportedAt = DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
var advisories = new[]
|
||||
{
|
||||
CreateAdvisory("CVE-2024-2000", new[] { "CVE-2024-2000" }, "Streaming One", "medium"),
|
||||
CreateAdvisory("CVE-2024-2001", new[] { "CVE-2024-2001" }, "Streaming Two", "low"),
|
||||
};
|
||||
|
||||
var sequence = new SingleEnumerationAsyncSequence(advisories);
|
||||
var result = await builder.WriteAsync(sequence, exportedAt, cancellationToken: CancellationToken.None);
|
||||
|
||||
Assert.Equal(advisories.Length, result.AdvisoryCount);
|
||||
}
|
||||
|
||||
private static Advisory CreateAdvisory(string advisoryKey, string[] aliases, string title, string severity)
|
||||
{
|
||||
return new Advisory(
|
||||
advisoryKey: advisoryKey,
|
||||
title: title,
|
||||
summary: null,
|
||||
language: "EN",
|
||||
published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
severity: severity,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: new[]
|
||||
{
|
||||
new AdvisoryReference("https://example.com/advisory", "advisory", null, null, AdvisoryProvenance.Empty),
|
||||
},
|
||||
affectedPackages: new[]
|
||||
{
|
||||
new AffectedPackage(
|
||||
AffectedPackageTypes.SemVer,
|
||||
"sample/package",
|
||||
platform: null,
|
||||
versionRanges: Array.Empty<AffectedVersionRange>(),
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: Array.Empty<AdvisoryProvenance>()),
|
||||
},
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[]
|
||||
{
|
||||
new AdvisoryProvenance("feedser", "normalized", "canonical", DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture)),
|
||||
});
|
||||
}
|
||||
|
||||
private static byte[] ComputeDigest(JsonExportResult result)
|
||||
{
|
||||
using var sha256 = SHA256.Create();
|
||||
foreach (var relative in result.FilePaths.OrderBy(x => x, StringComparer.Ordinal))
|
||||
{
|
||||
var fullPath = ResolvePath(result.ExportDirectory, relative);
|
||||
var bytes = File.ReadAllBytes(fullPath);
|
||||
sha256.TransformBlock(bytes, 0, bytes.Length, null, 0);
|
||||
}
|
||||
|
||||
sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||
return sha256.Hash ?? Array.Empty<byte>();
|
||||
}
|
||||
|
||||
private static string ResolvePath(string root, string relative)
|
||||
{
|
||||
var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
return Path.Combine(new[] { root }.Concat(segments).ToArray());
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// best effort cleanup
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleEnumerationAsyncSequence : IAsyncEnumerable<Advisory>
|
||||
{
|
||||
private readonly IReadOnlyList<Advisory> _advisories;
|
||||
private int _enumerated;
|
||||
|
||||
public SingleEnumerationAsyncSequence(IReadOnlyList<Advisory> advisories)
|
||||
{
|
||||
_advisories = advisories ?? throw new ArgumentNullException(nameof(advisories));
|
||||
}
|
||||
|
||||
public IAsyncEnumerator<Advisory> GetAsyncEnumerator(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (Interlocked.Exchange(ref _enumerated, 1) == 1)
|
||||
{
|
||||
throw new InvalidOperationException("Sequence was enumerated more than once.");
|
||||
}
|
||||
|
||||
return Enumerate(cancellationToken);
|
||||
|
||||
async IAsyncEnumerator<Advisory> Enumerate([EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
foreach (var advisory in _advisories)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
yield return advisory;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Exporter.Json;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Exporting;
|
||||
using StellaOps.Feedser.Models;
|
||||
|
||||
namespace StellaOps.Feedser.Exporter.Json.Tests;
|
||||
|
||||
public sealed class JsonExporterDependencyInjectionRoutineTests
|
||||
{
|
||||
[Fact]
|
||||
public void Register_AddsJobDefinitionAndServices()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
services.AddSingleton<IAdvisoryStore, StubAdvisoryStore>();
|
||||
services.AddSingleton<IExportStateStore, StubExportStateStore>();
|
||||
services.AddOptions<JobSchedulerOptions>();
|
||||
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>())
|
||||
.Build();
|
||||
|
||||
var routine = new JsonExporterDependencyInjectionRoutine();
|
||||
routine.Register(services, configuration);
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var optionsAccessor = provider.GetRequiredService<IOptions<JobSchedulerOptions>>();
|
||||
var options = optionsAccessor.Value;
|
||||
|
||||
Assert.True(options.Definitions.TryGetValue(JsonExportJob.JobKind, out var definition));
|
||||
Assert.Equal(typeof(JsonExportJob), definition.JobType);
|
||||
Assert.True(definition.Enabled);
|
||||
|
||||
var exporter = provider.GetRequiredService<JsonFeedExporter>();
|
||||
Assert.NotNull(exporter);
|
||||
}
|
||||
|
||||
private sealed class StubAdvisoryStore : IAdvisoryStore
|
||||
{
|
||||
public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<Advisory>>(Array.Empty<Advisory>());
|
||||
|
||||
public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<Advisory?>(null);
|
||||
|
||||
public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return Enumerate(cancellationToken);
|
||||
|
||||
static async IAsyncEnumerable<Advisory> Enumerate([EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await Task.Yield();
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubExportStateStore : IExportStateStore
|
||||
{
|
||||
private ExportStateRecord? _record;
|
||||
|
||||
public Task<ExportStateRecord?> FindAsync(string id, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_record);
|
||||
|
||||
public Task<ExportStateRecord> UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken)
|
||||
{
|
||||
_record = record;
|
||||
return Task.FromResult(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Exporter.Json;
|
||||
using StellaOps.Feedser.Models;
|
||||
|
||||
namespace StellaOps.Feedser.Exporter.Json.Tests;
|
||||
|
||||
public sealed class JsonExporterParitySmokeTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public JsonExporterParitySmokeTests()
|
||||
{
|
||||
_root = Directory.CreateTempSubdirectory("feedser-json-parity-tests").FullName;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExportProducesVulnListCompatiblePaths()
|
||||
{
|
||||
var options = new JsonExportOptions { OutputRoot = _root };
|
||||
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
|
||||
var exportedAt = DateTimeOffset.Parse("2024-09-01T12:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
var advisories = CreateSampleAdvisories();
|
||||
var result = await builder.WriteAsync(advisories, exportedAt, exportName: "parity-test", CancellationToken.None);
|
||||
|
||||
var expected = new[]
|
||||
{
|
||||
"amazon/2/ALAS2-2024-1234.json",
|
||||
"debian/DLA-2024-1234.json",
|
||||
"ghsa/go/github.com%2Facme%2Fsample/GHSA-AAAA-BBBB-CCCC.json",
|
||||
"nvd/2023/CVE-2023-27524.json",
|
||||
"oracle/linux/ELSA-2024-12345.json",
|
||||
"redhat/oval/RHSA-2024_0252.json",
|
||||
"ubuntu/USN-6620-1.json",
|
||||
"wolfi/WOLFI-2024-0001.json",
|
||||
};
|
||||
|
||||
Assert.Equal(expected, result.FilePaths.ToArray());
|
||||
|
||||
foreach (var path in expected)
|
||||
{
|
||||
var fullPath = ResolvePath(result.ExportDirectory, path);
|
||||
Assert.True(File.Exists(fullPath), $"Expected export file '{path}' to be present");
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<Advisory> CreateSampleAdvisories()
|
||||
{
|
||||
var published = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
var modified = DateTimeOffset.Parse("2024-02-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
return new[]
|
||||
{
|
||||
CreateAdvisory(
|
||||
"CVE-2023-27524",
|
||||
"Apache Superset Improper Authentication",
|
||||
new[] { "CVE-2023-27524" },
|
||||
null,
|
||||
"nvd",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"GHSA-aaaa-bbbb-cccc",
|
||||
"Sample GHSA",
|
||||
new[] { "CVE-2024-2000" },
|
||||
new[]
|
||||
{
|
||||
new AffectedPackage(
|
||||
AffectedPackageTypes.SemVer,
|
||||
"pkg:go/github.com/acme/sample@1.0.0",
|
||||
provenance: new[] { new AdvisoryProvenance("ghsa", "map", "", published) })
|
||||
},
|
||||
"ghsa",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"USN-6620-1",
|
||||
"Ubuntu Security Notice",
|
||||
null,
|
||||
null,
|
||||
"ubuntu",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"DLA-2024-1234",
|
||||
"Debian LTS Advisory",
|
||||
null,
|
||||
null,
|
||||
"debian",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"RHSA-2024:0252",
|
||||
"Red Hat Security Advisory",
|
||||
null,
|
||||
null,
|
||||
"redhat",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"ALAS2-2024-1234",
|
||||
"Amazon Linux Advisory",
|
||||
null,
|
||||
null,
|
||||
"amazon",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"ELSA-2024-12345",
|
||||
"Oracle Linux Advisory",
|
||||
null,
|
||||
null,
|
||||
"oracle",
|
||||
published,
|
||||
modified),
|
||||
CreateAdvisory(
|
||||
"WOLFI-2024-0001",
|
||||
"Wolfi Advisory",
|
||||
null,
|
||||
null,
|
||||
"wolfi",
|
||||
published,
|
||||
modified),
|
||||
};
|
||||
}
|
||||
|
||||
private static Advisory CreateAdvisory(
|
||||
string advisoryKey,
|
||||
string title,
|
||||
IEnumerable<string>? aliases,
|
||||
IEnumerable<AffectedPackage>? packages,
|
||||
string? provenanceSource,
|
||||
DateTimeOffset? published,
|
||||
DateTimeOffset? modified)
|
||||
{
|
||||
var provenance = provenanceSource is null
|
||||
? Array.Empty<AdvisoryProvenance>()
|
||||
: new[] { new AdvisoryProvenance(provenanceSource, "normalize", "", modified ?? DateTimeOffset.UtcNow) };
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey,
|
||||
title,
|
||||
summary: null,
|
||||
language: "en",
|
||||
published,
|
||||
modified,
|
||||
severity: "medium",
|
||||
exploitKnown: false,
|
||||
aliases: aliases ?? Array.Empty<string>(),
|
||||
references: Array.Empty<AdvisoryReference>(),
|
||||
affectedPackages: packages ?? Array.Empty<AffectedPackage>(),
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
}
|
||||
|
||||
private static string ResolvePath(string root, string relative)
|
||||
{
|
||||
var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
return Path.Combine(new[] { root }.Concat(segments).ToArray());
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// best effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Exporter.Json;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Exporting;
|
||||
|
||||
namespace StellaOps.Feedser.Exporter.Json.Tests;
|
||||
|
||||
public sealed class JsonFeedExporterTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public JsonFeedExporterTests()
|
||||
{
|
||||
_root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExportAsync_SkipsWhenDigestUnchanged()
|
||||
{
|
||||
var advisory = new Advisory(
|
||||
advisoryKey: "CVE-2024-1234",
|
||||
title: "Test Advisory",
|
||||
summary: null,
|
||||
language: "en",
|
||||
published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
severity: "high",
|
||||
exploitKnown: false,
|
||||
aliases: new[] { "CVE-2024-1234" },
|
||||
references: Array.Empty<AdvisoryReference>(),
|
||||
affectedPackages: Array.Empty<AffectedPackage>(),
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: Array.Empty<AdvisoryProvenance>());
|
||||
|
||||
var advisoryStore = new StubAdvisoryStore(advisory);
|
||||
var options = Options.Create(new JsonExportOptions
|
||||
{
|
||||
OutputRoot = _root,
|
||||
MaintainLatestSymlink = false,
|
||||
});
|
||||
|
||||
var stateStore = new InMemoryExportStateStore();
|
||||
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture));
|
||||
var stateManager = new ExportStateManager(stateStore, timeProvider);
|
||||
var exporter = new JsonFeedExporter(
|
||||
advisoryStore,
|
||||
options,
|
||||
new VulnListJsonExportPathResolver(),
|
||||
stateManager,
|
||||
NullLogger<JsonFeedExporter>.Instance,
|
||||
timeProvider);
|
||||
|
||||
using var provider = new ServiceCollection().BuildServiceProvider();
|
||||
await exporter.ExportAsync(provider, CancellationToken.None);
|
||||
|
||||
var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
|
||||
Assert.NotNull(record);
|
||||
var firstUpdated = record!.UpdatedAt;
|
||||
Assert.Equal("20240715T120000Z", record.BaseExportId);
|
||||
Assert.Equal(record.LastFullDigest, record.ExportCursor);
|
||||
|
||||
var firstExportPath = Path.Combine(_root, "20240715T120000Z");
|
||||
Assert.True(Directory.Exists(firstExportPath));
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromMinutes(5));
|
||||
await exporter.ExportAsync(provider, CancellationToken.None);
|
||||
|
||||
record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None);
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(firstUpdated, record!.UpdatedAt);
|
||||
|
||||
var secondExportPath = Path.Combine(_root, "20240715T120500Z");
|
||||
Assert.False(Directory.Exists(secondExportPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExportAsync_WritesManifestMetadata()
|
||||
{
|
||||
var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
var advisory = new Advisory(
|
||||
advisoryKey: "CVE-2024-4321",
|
||||
title: "Manifest Test",
|
||||
summary: null,
|
||||
language: "en",
|
||||
published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture),
|
||||
severity: "medium",
|
||||
exploitKnown: false,
|
||||
aliases: new[] { "CVE-2024-4321" },
|
||||
references: Array.Empty<AdvisoryReference>(),
|
||||
affectedPackages: Array.Empty<AffectedPackage>(),
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: Array.Empty<AdvisoryProvenance>());
|
||||
|
||||
var advisoryStore = new StubAdvisoryStore(advisory);
|
||||
var optionsValue = new JsonExportOptions
|
||||
{
|
||||
OutputRoot = _root,
|
||||
MaintainLatestSymlink = false,
|
||||
};
|
||||
|
||||
var options = Options.Create(optionsValue);
|
||||
var stateStore = new InMemoryExportStateStore();
|
||||
var timeProvider = new TestTimeProvider(exportedAt);
|
||||
var stateManager = new ExportStateManager(stateStore, timeProvider);
|
||||
var exporter = new JsonFeedExporter(
|
||||
advisoryStore,
|
||||
options,
|
||||
new VulnListJsonExportPathResolver(),
|
||||
stateManager,
|
||||
NullLogger<JsonFeedExporter>.Instance,
|
||||
timeProvider);
|
||||
|
||||
using var provider = new ServiceCollection().BuildServiceProvider();
|
||||
await exporter.ExportAsync(provider, CancellationToken.None);
|
||||
|
||||
var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture);
|
||||
var exportDirectory = Path.Combine(_root, exportId);
|
||||
var manifestPath = Path.Combine(exportDirectory, "manifest.json");
|
||||
|
||||
Assert.True(File.Exists(manifestPath));
|
||||
|
||||
using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None));
|
||||
var root = document.RootElement;
|
||||
|
||||
Assert.Equal(exportId, root.GetProperty("exportId").GetString());
|
||||
Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime());
|
||||
Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32());
|
||||
|
||||
var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories)
|
||||
.Select(path => new
|
||||
{
|
||||
Absolute = path,
|
||||
Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal),
|
||||
})
|
||||
.Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase))
|
||||
.OrderBy(file => file.Relative, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
var filesElement = root.GetProperty("files")
|
||||
.EnumerateArray()
|
||||
.Select(element => new
|
||||
{
|
||||
Path = element.GetProperty("path").GetString(),
|
||||
Bytes = element.GetProperty("bytes").GetInt64(),
|
||||
Digest = element.GetProperty("digest").GetString(),
|
||||
})
|
||||
.OrderBy(file => file.Path, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray());
|
||||
|
||||
long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum();
|
||||
Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64());
|
||||
Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32());
|
||||
|
||||
var digest = root.GetProperty("digest").GetString();
|
||||
var digestResult = new JsonExportResult(
|
||||
exportDirectory,
|
||||
exportedAt,
|
||||
exportedFiles.Select(file =>
|
||||
{
|
||||
var manifestEntry = filesElement.First(f => f.Path == file.Relative);
|
||||
if (manifestEntry.Digest is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest.");
|
||||
}
|
||||
|
||||
return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest);
|
||||
}),
|
||||
exportedFiles.Length,
|
||||
totalBytes);
|
||||
var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult);
|
||||
Assert.Equal(expectedDigest, digest);
|
||||
|
||||
var exporterVersion = root.GetProperty("exporterVersion").GetString();
|
||||
Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// best effort cleanup
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubAdvisoryStore : IAdvisoryStore
|
||||
{
|
||||
private readonly IReadOnlyList<Advisory> _advisories;
|
||||
|
||||
public StubAdvisoryStore(params Advisory[] advisories)
|
||||
{
|
||||
_advisories = advisories;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_advisories);
|
||||
|
||||
public Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<Advisory?>(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey));
|
||||
|
||||
public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return EnumerateAsync(cancellationToken);
|
||||
|
||||
async IAsyncEnumerable<Advisory> EnumerateAsync([EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
foreach (var advisory in _advisories)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
yield return advisory;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryExportStateStore : IExportStateStore
|
||||
{
|
||||
private ExportStateRecord? _record;
|
||||
|
||||
public Task<ExportStateRecord?> FindAsync(string id, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_record);
|
||||
|
||||
public Task<ExportStateRecord> UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken)
|
||||
{
|
||||
_record = record;
|
||||
return Task.FromResult(record);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TestTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _now;
|
||||
|
||||
public TestTimeProvider(DateTimeOffset start) => _now = start;
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _now;
|
||||
|
||||
public void Advance(TimeSpan delta) => _now = _now.Add(delta);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using StellaOps.Feedser.Exporter.Json;
|
||||
using StellaOps.Feedser.Models;
|
||||
|
||||
namespace StellaOps.Feedser.Exporter.Json.Tests;
|
||||
|
||||
public sealed class VulnListJsonExportPathResolverTests
|
||||
{
|
||||
private static readonly DateTimeOffset DefaultPublished = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
[Fact]
|
||||
public void ResolvesCvePath()
|
||||
{
|
||||
var advisory = CreateAdvisory("CVE-2024-1234");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("nvd", "2024", "CVE-2024-1234.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesGhsaWithPackage()
|
||||
{
|
||||
var package = new AffectedPackage(
|
||||
AffectedPackageTypes.SemVer,
|
||||
"pkg:go/github.com/acme/widget@1.0.0",
|
||||
platform: null,
|
||||
versionRanges: Array.Empty<AffectedVersionRange>(),
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: Array.Empty<AdvisoryProvenance>());
|
||||
|
||||
var advisory = CreateAdvisory(
|
||||
"GHSA-aaaa-bbbb-cccc",
|
||||
aliases: new[] { "CVE-2024-2000" },
|
||||
packages: new[] { package });
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("ghsa", "go", "github.com%2Facme%2Fwidget", "GHSA-AAAA-BBBB-CCCC.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesUbuntuUsn()
|
||||
{
|
||||
var advisory = CreateAdvisory("USN-6620-1");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("ubuntu", "USN-6620-1.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesDebianDla()
|
||||
{
|
||||
var advisory = CreateAdvisory("DLA-1234-1");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("debian", "DLA-1234-1.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesRedHatRhsa()
|
||||
{
|
||||
var advisory = CreateAdvisory("RHSA-2024:0252");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("redhat", "oval", "RHSA-2024_0252.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesAmazonAlas()
|
||||
{
|
||||
var advisory = CreateAdvisory("ALAS2-2024-1234");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("amazon", "2", "ALAS2-2024-1234.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesOracleElsa()
|
||||
{
|
||||
var advisory = CreateAdvisory("ELSA-2024-12345");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("oracle", "linux", "ELSA-2024-12345.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesRockyRlsa()
|
||||
{
|
||||
var advisory = CreateAdvisory("RLSA-2024:0417");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("rocky", "RLSA-2024_0417.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesByProvenanceFallback()
|
||||
{
|
||||
var provenance = new[] { new AdvisoryProvenance("wolfi", "map", "", DefaultPublished) };
|
||||
var advisory = CreateAdvisory("WOLFI-2024-0001", provenance: provenance);
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("wolfi", "WOLFI-2024-0001.json"), path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DefaultsToMiscWhenUnmapped()
|
||||
{
|
||||
var advisory = CreateAdvisory("CUSTOM-2024-99");
|
||||
var resolver = new VulnListJsonExportPathResolver();
|
||||
var path = resolver.GetRelativePath(advisory);
|
||||
|
||||
Assert.Equal(Path.Combine("misc", "CUSTOM-2024-99.json"), path);
|
||||
}
|
||||
|
||||
private static Advisory CreateAdvisory(
|
||||
string advisoryKey,
|
||||
IEnumerable<string>? aliases = null,
|
||||
IEnumerable<AffectedPackage>? packages = null,
|
||||
IEnumerable<AdvisoryProvenance>? provenance = null)
|
||||
{
|
||||
return new Advisory(
|
||||
advisoryKey: advisoryKey,
|
||||
title: $"Advisory {advisoryKey}",
|
||||
summary: null,
|
||||
language: "en",
|
||||
published: DefaultPublished,
|
||||
modified: DefaultPublished,
|
||||
severity: "medium",
|
||||
exploitKnown: false,
|
||||
aliases: aliases ?? Array.Empty<string>(),
|
||||
references: Array.Empty<AdvisoryReference>(),
|
||||
affectedPackages: packages ?? Array.Empty<AffectedPackage>(),
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance ?? Array.Empty<AdvisoryProvenance>());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user