Restore vendor connector internals and configure offline packages
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
root
2025-10-20 14:50:58 +03:00
parent 09b6a28172
commit 44ad31591c
59 changed files with 7512 additions and 3797 deletions

View File

@@ -1,309 +1,310 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers;
using Xunit;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
public sealed class UbuntuCsafConnectorTests
{
[Fact]
public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag()
{
var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var indexUri = new Uri(baseUri, "index.json");
var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z");
var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}");
var documentSha = ComputeSha256(documentPayload);
var indexJson = manifest.IndexJson;
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient);
var cache = new MemoryCache(new MemoryCacheOptions());
var fileSystem = new MockFileSystem();
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository();
var connector = new UbuntuCsafConnector(
loader,
httpFactory,
stateRepository,
new[] { optionsValidator },
NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
await connector.ValidateAsync(settings, CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().HaveCount(1);
sink.Documents.Should().HaveCount(1);
var stored = sink.Documents.Single();
stored.Digest.Should().Be($"sha256:{documentSha}");
stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue();
storedEtag.Should().Be("etag-123");
stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}");
stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123");
stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z"));
handler.DocumentRequestCount.Should().Be(1);
// Second run: Expect connector to send If-None-Match and skip download via 304.
sink.Documents.Clear();
documents.Clear();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty();
handler.DocumentRequestCount.Should().Be(2);
handler.SeenIfNoneMatch.Should().Contain("\"etag-123\"");
}
[Fact]
public async Task FetchAsync_SkipsWhenChecksumMismatch()
{
var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var indexUri = new Uri(baseUri, "index.json");
var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z");
var indexJson = manifest.IndexJson;
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient);
var cache = new MemoryCache(new MemoryCacheOptions());
var fileSystem = new MockFileSystem();
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository();
var connector = new UbuntuCsafConnector(
loader,
httpFactory,
stateRepository,
new[] { optionsValidator },
NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary<string, string>.Empty), CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty();
stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty();
handler.DocumentRequestCount.Should().Be(1);
}
private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp)
{
var indexJson = $$"""
{
"generated": "2025-10-18T00:00:00Z",
"channels": [
{
"name": "stable",
"catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json",
"sha256": "ignore"
}
]
}
""";
var catalogJson = $$"""
{
"resources": [
{
"id": "{{advisoryId}}",
"type": "csaf",
"url": "{{advisoryUri}}",
"last_modified": "{{timestamp}}",
"hashes": {
"sha256": "{{SHA256}}"
},
"etag": "\"etag-123\"",
"title": "{{advisoryId}}"
}
]
}
""";
return (indexJson, catalogJson);
}
private static string ComputeSha256(ReadOnlySpan<byte> payload)
{
Span<byte> buffer = stackalloc byte[32];
SHA256.HashData(payload, buffer);
return Convert.ToHexString(buffer).ToLowerInvariant();
}
private sealed class SingleClientFactory : IHttpClientFactory
{
private readonly HttpClient _client;
public SingleClientFactory(HttpClient client)
{
_client = client;
}
public HttpClient CreateClient(string name) => _client;
}
private sealed class UbuntuTestHttpHandler : HttpMessageHandler
{
private readonly Uri _indexUri;
private readonly string _indexPayload;
private readonly Uri _catalogUri;
private readonly string _catalogPayload;
private readonly Uri _documentUri;
private readonly byte[] _documentPayload;
private readonly string _expectedEtag;
public int DocumentRequestCount { get; private set; }
public List<string> SeenIfNoneMatch { get; } = new();
public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag)
{
_indexUri = indexUri;
_indexPayload = indexPayload;
_catalogUri = catalogUri;
_catalogPayload = catalogPayload;
_documentUri = documentUri;
_documentPayload = documentPayload;
_expectedEtag = expectedEtag;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri == _indexUri)
{
return Task.FromResult(CreateJsonResponse(_indexPayload));
}
if (request.RequestUri == _catalogUri)
{
return Task.FromResult(CreateJsonResponse(_catalogPayload));
}
if (request.RequestUri == _documentUri)
{
DocumentRequestCount++;
if (request.Headers.IfNoneMatch is { Count: > 0 })
{
var header = request.Headers.IfNoneMatch.First().ToString();
SeenIfNoneMatch.Add(header);
if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"")
{
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified));
}
}
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(_documentPayload),
};
response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\"");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return Task.FromResult(response);
}
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
{
Content = new StringContent($"No response configured for {request.RequestUri}"),
});
}
private static HttpResponseMessage CreateJsonResponse(string payload)
=> new(HttpStatusCode.OK)
{
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
};
}
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
{
public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
=> ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
{
CurrentState = state;
return ValueTask.CompletedTask;
}
}
private sealed class InMemoryRawSink : IVexRawDocumentSink
{
public List<VexRawDocument> Documents { get; } = new();
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
{
Documents.Add(document);
return ValueTask.CompletedTask;
}
}
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
{
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
}
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
{
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
}
}
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using FluentAssertions;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers;
using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
public sealed class UbuntuCsafConnectorTests
{
[Fact]
public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag()
{
var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var indexUri = new Uri(baseUri, "index.json");
var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z");
var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}");
var documentSha = ComputeSha256(documentPayload);
var indexJson = manifest.IndexJson;
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient);
var cache = new MemoryCache(new MemoryCacheOptions());
var fileSystem = new MockFileSystem();
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository();
var connector = new UbuntuCsafConnector(
loader,
httpFactory,
stateRepository,
new[] { optionsValidator },
NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
await connector.ValidateAsync(settings, CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().HaveCount(1);
sink.Documents.Should().HaveCount(1);
var stored = sink.Documents.Single();
stored.Digest.Should().Be($"sha256:{documentSha}");
stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue();
storedEtag.Should().Be("etag-123");
stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}");
stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123");
stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z"));
handler.DocumentRequestCount.Should().Be(1);
// Second run: Expect connector to send If-None-Match and skip download via 304.
sink.Documents.Clear();
documents.Clear();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty();
handler.DocumentRequestCount.Should().Be(2);
handler.SeenIfNoneMatch.Should().Contain("\"etag-123\"");
}
[Fact]
public async Task FetchAsync_SkipsWhenChecksumMismatch()
{
var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var indexUri = new Uri(baseUri, "index.json");
var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z");
var indexJson = manifest.IndexJson;
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient);
var cache = new MemoryCache(new MemoryCacheOptions());
var fileSystem = new MockFileSystem();
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository();
var connector = new UbuntuCsafConnector(
loader,
httpFactory,
stateRepository,
new[] { optionsValidator },
NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary<string, string>.Empty), CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{
documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty();
stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty();
handler.DocumentRequestCount.Should().Be(1);
}
private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp)
{
var indexJson = """
{
"generated": "2025-10-18T00:00:00Z",
"channels": [
{
"name": "stable",
"catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json",
"sha256": "ignore"
}
]
}
""";
var catalogJson = """
{
"resources": [
{
"id": "{{advisoryId}}",
"type": "csaf",
"url": "{{advisoryUri}}",
"last_modified": "{{timestamp}}",
"hashes": {
"sha256": "{{SHA256}}"
},
"etag": "\"etag-123\"",
"title": "{{advisoryId}}"
}
]
}
""";
return (indexJson, catalogJson);
}
private static string ComputeSha256(ReadOnlySpan<byte> payload)
{
Span<byte> buffer = stackalloc byte[32];
SHA256.HashData(payload, buffer);
return Convert.ToHexString(buffer).ToLowerInvariant();
}
private sealed class SingleClientFactory : IHttpClientFactory
{
private readonly HttpClient _client;
public SingleClientFactory(HttpClient client)
{
_client = client;
}
public HttpClient CreateClient(string name) => _client;
}
private sealed class UbuntuTestHttpHandler : HttpMessageHandler
{
private readonly Uri _indexUri;
private readonly string _indexPayload;
private readonly Uri _catalogUri;
private readonly string _catalogPayload;
private readonly Uri _documentUri;
private readonly byte[] _documentPayload;
private readonly string _expectedEtag;
public int DocumentRequestCount { get; private set; }
public List<string> SeenIfNoneMatch { get; } = new();
public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag)
{
_indexUri = indexUri;
_indexPayload = indexPayload;
_catalogUri = catalogUri;
_catalogPayload = catalogPayload;
_documentUri = documentUri;
_documentPayload = documentPayload;
_expectedEtag = expectedEtag;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (request.RequestUri == _indexUri)
{
return Task.FromResult(CreateJsonResponse(_indexPayload));
}
if (request.RequestUri == _catalogUri)
{
return Task.FromResult(CreateJsonResponse(_catalogPayload));
}
if (request.RequestUri == _documentUri)
{
DocumentRequestCount++;
if (request.Headers.IfNoneMatch is { Count: > 0 })
{
var header = request.Headers.IfNoneMatch.First().ToString();
SeenIfNoneMatch.Add(header);
if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"")
{
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified));
}
}
var response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new ByteArrayContent(_documentPayload),
};
response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\"");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return Task.FromResult(response);
}
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
{
Content = new StringContent($"No response configured for {request.RequestUri}"),
});
}
private static HttpResponseMessage CreateJsonResponse(string payload)
=> new(HttpStatusCode.OK)
{
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
};
}
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
{
public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
CurrentState = state;
return ValueTask.CompletedTask;
}
}
private sealed class InMemoryRawSink : IVexRawDocumentSink
{
public List<VexRawDocument> Documents { get; } = new();
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
{
Documents.Add(document);
return ValueTask.CompletedTask;
}
}
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
{
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
}
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
{
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
}
}