stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
src/Concelier/__Tests/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs
View File

@@ -1,136 +1,136 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Connector.Nvd;
using StellaOps.Concelier.Connector.Nvd.Configuration;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Testing;
using StellaOps.Concelier.Testing;
using System.Net;
namespace StellaOps.Concelier.Connector.Nvd.Tests;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Connector.Nvd;
using StellaOps.Concelier.Connector.Nvd.Configuration;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Testing;
using StellaOps.Concelier.Testing;
using System.Net;
namespace StellaOps.Concelier.Connector.Nvd.Tests;
[Collection(ConcelierFixtureCollection.Name)]
public sealed class NvdConnectorHarnessTests : IAsyncLifetime
{
private readonly ConnectorTestHarness _harness;
public NvdConnectorHarnessTests(ConcelierPostgresFixture fixture)
{
_harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero), NvdOptions.HttpClientName);
}
[Fact]
public async Task FetchAsync_MultiPagePersistsStartIndexMetadata()
{
await _harness.ResetAsync();
var options = new NvdOptions
{
BaseEndpoint = new Uri("https://nvd.example.test/api"),
WindowSize = TimeSpan.FromHours(1),
WindowOverlap = TimeSpan.FromMinutes(5),
InitialBackfill = TimeSpan.FromHours(2),
};
var timeProvider = _harness.TimeProvider;
var handler = _harness.Handler;
var windowStart = timeProvider.GetUtcNow() - options.InitialBackfill;
var windowEnd = windowStart + options.WindowSize;
var firstUri = BuildRequestUri(options, windowStart, windowEnd);
var secondUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 2);
var thirdUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 4);
handler.AddJsonResponse(firstUri, ReadFixture("nvd-multipage-1.json"));
handler.AddJsonResponse(secondUri, ReadFixture("nvd-multipage-2.json"));
handler.AddJsonResponse(thirdUri, ReadFixture("nvd-multipage-3.json"));
await _harness.EnsureServiceProviderAsync(services =>
{
services.AddNvdConnector(opts =>
{
opts.BaseEndpoint = options.BaseEndpoint;
opts.WindowSize = options.WindowSize;
opts.WindowOverlap = options.WindowOverlap;
opts.InitialBackfill = options.InitialBackfill;
});
});
var provider = _harness.ServiceProvider;
var connector = new NvdConnectorPlugin().Create(provider);
await connector.FetchAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var firstDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, firstUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal("0", firstDocument!.Metadata["startIndex"]);
var secondDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, secondUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal("2", secondDocument!.Metadata["startIndex"]);
var thirdDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, thirdUri.ToString(), CancellationToken.None);
Assert.NotNull(thirdDocument);
Assert.Equal("4", thirdDocument!.Metadata["startIndex"]);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending)
? pending.AsBsonArray
: new BsonArray();
Assert.Equal(3, pendingDocuments.Count);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => _harness.ResetAsync();
private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0)
{
var builder = new UriBuilder(options.BaseEndpoint);
var parameters = new Dictionary<string, string>
{
["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["resultsPerPage"] = "2000",
};
if (startIndex > 0)
{
parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture);
}
builder.Query = string.Join("&", parameters.Select(kvp => $"{WebUtility.UrlEncode(kvp.Key)}={WebUtility.UrlEncode(kvp.Value)}"));
return builder.Uri;
}
private static string ReadFixture(string filename)
{
var baseDirectory = AppContext.BaseDirectory;
var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename);
if (File.Exists(primary))
{
return File.ReadAllText(primary);
}
var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename);
if (File.Exists(secondary))
{
return File.ReadAllText(secondary);
}
throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory.");
}
}
public sealed class NvdConnectorHarnessTests : IAsyncLifetime
{
private readonly ConnectorTestHarness _harness;
public NvdConnectorHarnessTests(ConcelierPostgresFixture fixture)
{
_harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero), NvdOptions.HttpClientName);
}
[Fact]
public async Task FetchAsync_MultiPagePersistsStartIndexMetadata()
{
await _harness.ResetAsync();
var options = new NvdOptions
{
BaseEndpoint = new Uri("https://nvd.example.test/api"),
WindowSize = TimeSpan.FromHours(1),
WindowOverlap = TimeSpan.FromMinutes(5),
InitialBackfill = TimeSpan.FromHours(2),
};
var timeProvider = _harness.TimeProvider;
var handler = _harness.Handler;
var windowStart = timeProvider.GetUtcNow() - options.InitialBackfill;
var windowEnd = windowStart + options.WindowSize;
var firstUri = BuildRequestUri(options, windowStart, windowEnd);
var secondUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 2);
var thirdUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 4);
handler.AddJsonResponse(firstUri, ReadFixture("nvd-multipage-1.json"));
handler.AddJsonResponse(secondUri, ReadFixture("nvd-multipage-2.json"));
handler.AddJsonResponse(thirdUri, ReadFixture("nvd-multipage-3.json"));
await _harness.EnsureServiceProviderAsync(services =>
{
services.AddNvdConnector(opts =>
{
opts.BaseEndpoint = options.BaseEndpoint;
opts.WindowSize = options.WindowSize;
opts.WindowOverlap = options.WindowOverlap;
opts.InitialBackfill = options.InitialBackfill;
});
});
var provider = _harness.ServiceProvider;
var connector = new NvdConnectorPlugin().Create(provider);
await connector.FetchAsync(provider, CancellationToken.None);
var documentStore = provider.GetRequiredService<IDocumentStore>();
var firstDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, firstUri.ToString(), CancellationToken.None);
Assert.NotNull(firstDocument);
Assert.Equal("0", firstDocument!.Metadata["startIndex"]);
var secondDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, secondUri.ToString(), CancellationToken.None);
Assert.NotNull(secondDocument);
Assert.Equal("2", secondDocument!.Metadata["startIndex"]);
var thirdDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, thirdUri.ToString(), CancellationToken.None);
Assert.NotNull(thirdDocument);
Assert.Equal("4", thirdDocument!.Metadata["startIndex"]);
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending)
? pending.AsDocumentArray
: new DocumentArray();
Assert.Equal(3, pendingDocuments.Count);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync() => _harness.ResetAsync();
private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0)
{
var builder = new UriBuilder(options.BaseEndpoint);
var parameters = new Dictionary<string, string>
{
["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["resultsPerPage"] = "2000",
};
if (startIndex > 0)
{
parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture);
}
builder.Query = string.Join("&", parameters.Select(kvp => $"{WebUtility.UrlEncode(kvp.Key)}={WebUtility.UrlEncode(kvp.Value)}"));
return builder.Uri;
}
private static string ReadFixture(string filename)
{
var baseDirectory = AppContext.BaseDirectory;
var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename);
if (File.Exists(primary))
{
return File.ReadAllText(primary);
}
var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename);
if (File.Exists(secondary))
{
return File.ReadAllText(secondary);
}
throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory.");
}
}

1264
src/Concelier/__Tests/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorTests.cs
View File

File diff suppressed because it is too large Load Diff

196
src/Concelier/__Tests/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs
View File

@@ -1,98 +1,98 @@
using System;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using StellaOps.Concelier.Core;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using Xunit;
namespace StellaOps.Concelier.Connector.Nvd.Tests.Nvd;
public sealed class NvdMergeExportParityTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
[Fact]
public async Task CanonicalMerge_PreservesCreditsAndReferences_ExporterMaintainsParity()
{
var ghsa = LoadFixture("credit-parity.ghsa.json");
var osv = LoadFixture("credit-parity.osv.json");
var nvd = LoadFixture("credit-parity.nvd.json");
var merger = new CanonicalMerger();
var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv);
var merged = result.Advisory;
Assert.NotNull(merged);
var creditKeys = merged!.Credits
.Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}")
.ToHashSet(StringComparer.Ordinal);
Assert.Equal(2, creditKeys.Count);
Assert.Contains("reporter|Alice Researcher|mailto:alice.researcher@example.com", creditKeys);
Assert.Contains("remediation_developer|Bob Maintainer|https://github.com/acme/bob-maintainer", creditKeys);
var referenceUrls = merged.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase);
Assert.Equal(5, referenceUrls.Count);
Assert.Contains($"https://github.com/advisories/GHSA-credit-parity", referenceUrls);
Assert.Contains("https://example.com/ghsa/patch", referenceUrls);
Assert.Contains($"https://osv.dev/vulnerability/GHSA-credit-parity", referenceUrls);
Assert.Contains($"https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", referenceUrls);
Assert.Contains("https://example.com/nvd/reference", referenceUrls);
using var tempDirectory = new TempDirectory();
var options = new JsonExportOptions { OutputRoot = tempDirectory.Path };
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
var exportResult = await builder.WriteAsync(new[] { merged }, new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero));
Assert.Single(exportResult.Files);
var exportFile = exportResult.Files[0];
var exportPath = Path.Combine(exportResult.ExportDirectory, exportFile.RelativePath.Replace('/', Path.DirectorySeparatorChar));
Assert.True(File.Exists(exportPath));
var exported = JsonSerializer.Deserialize<Advisory>(await File.ReadAllTextAsync(exportPath), SerializerOptions);
Assert.NotNull(exported);
var exportedCredits = exported!.Credits
.Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}")
.ToHashSet(StringComparer.Ordinal);
Assert.Equal(creditKeys, exportedCredits);
var exportedReferences = exported.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase);
Assert.Equal(referenceUrls, exportedReferences);
}
private static Advisory LoadFixture(string fileName)
{
var path = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", fileName);
return JsonSerializer.Deserialize<Advisory>(File.ReadAllText(path), SerializerOptions)
?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'.");
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
Path = Directory.CreateTempSubdirectory("nvd-merge-export").FullName;
}
public string Path { get; }
public void Dispose()
{
try
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
catch
{
// best effort cleanup
}
}
}
}
using System;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using StellaOps.Concelier.Core;
using StellaOps.Concelier.Exporter.Json;
using StellaOps.Concelier.Models;
using Xunit;
namespace StellaOps.Concelier.Connector.Nvd.Tests.Nvd;
public sealed class NvdMergeExportParityTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
[Fact]
public async Task CanonicalMerge_PreservesCreditsAndReferences_ExporterMaintainsParity()
{
var ghsa = LoadFixture("credit-parity.ghsa.json");
var osv = LoadFixture("credit-parity.osv.json");
var nvd = LoadFixture("credit-parity.nvd.json");
var merger = new CanonicalMerger();
var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv);
var merged = result.Advisory;
Assert.NotNull(merged);
var creditKeys = merged!.Credits
.Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}")
.ToHashSet(StringComparer.Ordinal);
Assert.Equal(2, creditKeys.Count);
Assert.Contains("reporter|Alice Researcher|mailto:alice.researcher@example.com", creditKeys);
Assert.Contains("remediation_developer|Bob Maintainer|https://github.com/acme/bob-maintainer", creditKeys);
var referenceUrls = merged.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase);
Assert.Equal(5, referenceUrls.Count);
Assert.Contains($"https://github.com/advisories/GHSA-credit-parity", referenceUrls);
Assert.Contains("https://example.com/ghsa/patch", referenceUrls);
Assert.Contains($"https://osv.dev/vulnerability/GHSA-credit-parity", referenceUrls);
Assert.Contains($"https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", referenceUrls);
Assert.Contains("https://example.com/nvd/reference", referenceUrls);
using var tempDirectory = new TempDirectory();
var options = new JsonExportOptions { OutputRoot = tempDirectory.Path };
var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver());
var exportResult = await builder.WriteAsync(new[] { merged }, new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero));
Assert.Single(exportResult.Files);
var exportFile = exportResult.Files[0];
var exportPath = Path.Combine(exportResult.ExportDirectory, exportFile.RelativePath.Replace('/', Path.DirectorySeparatorChar));
Assert.True(File.Exists(exportPath));
var exported = JsonSerializer.Deserialize<Advisory>(await File.ReadAllTextAsync(exportPath), SerializerOptions);
Assert.NotNull(exported);
var exportedCredits = exported!.Credits
.Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}")
.ToHashSet(StringComparer.Ordinal);
Assert.Equal(creditKeys, exportedCredits);
var exportedReferences = exported.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase);
Assert.Equal(referenceUrls, exportedReferences);
}
private static Advisory LoadFixture(string fileName)
{
var path = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", fileName);
return JsonSerializer.Deserialize<Advisory>(File.ReadAllText(path), SerializerOptions)
?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'.");
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
Path = Directory.CreateTempSubdirectory("nvd-merge-export").FullName;
}
public string Path { get; }
public void Dispose()
{
try
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
catch
{
// best effort cleanup
}
}
}
}