Rename Concelier Source modules to Connector
This commit is contained in:
master
@@ -0,0 +1,266 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Testing;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Testing;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Tests.Vmware;
|
||||
|
||||
[Collection("mongo-fixture")]
|
||||
public sealed class VmwareConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoIntegrationFixture _fixture;
|
||||
private readonly FakeTimeProvider _timeProvider;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
private static readonly Uri IndexUri = new("https://vmware.example/api/vmsa/index.json");
|
||||
private static readonly Uri DetailOne = new("https://vmware.example/api/vmsa/VMSA-2024-0001.json");
|
||||
private static readonly Uri DetailTwo = new("https://vmware.example/api/vmsa/VMSA-2024-0002.json");
|
||||
private static readonly Uri DetailThree = new("https://vmware.example/api/vmsa/VMSA-2024-0003.json");
|
||||
|
||||
public VmwareConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 5, 0, 0, 0, TimeSpan.Zero));
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
_output = output;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_ProducesSnapshotAndCoversResume()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedInitialResponses();
|
||||
|
||||
using var metrics = new VmwareMetricCollector();
|
||||
|
||||
var connector = provider.GetRequiredService<VmwareConnector>();
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
|
||||
var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray();
|
||||
|
||||
var snapshot = Normalize(SnapshotSerializer.ToSnapshot(ordered));
|
||||
var expected = Normalize(ReadFixture("vmware-advisories.snapshot.json"));
|
||||
if (!string.Equals(expected, snapshot, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", "vmware-advisories.actual.json");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
|
||||
File.WriteAllText(actualPath, snapshot);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, snapshot);
|
||||
|
||||
var psirtCollection = _fixture.Database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.PsirtFlags);
|
||||
var psirtFlags = await psirtCollection.Find(Builders<BsonDocument>.Filter.Empty).ToListAsync();
|
||||
_output.WriteLine("PSIRT flags after initial map: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("<missing>")).ToString())));
|
||||
Assert.Equal(2, psirtFlags.Count);
|
||||
Assert.All(psirtFlags, doc => Assert.Equal("VMware", doc["vendor"].AsString));
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(VmwareConnectorPlugin.SourceName, CancellationToken.None);
|
||||
Assert.NotNull(state);
|
||||
Assert.Empty(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray());
|
||||
Assert.Empty(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) ? pendingMaps.AsBsonArray : new BsonArray());
|
||||
var cursorSnapshot = VmwareCursor.FromBson(state.Cursor);
|
||||
_output.WriteLine($"Initial fetch cache entries: {cursorSnapshot.FetchCache.Count}");
|
||||
foreach (var entry in cursorSnapshot.FetchCache)
|
||||
{
|
||||
_output.WriteLine($"Cache seed: {entry.Key} -> {entry.Value.Sha256}");
|
||||
}
|
||||
|
||||
// Second run with unchanged advisories and one new advisory.
|
||||
SeedUpdateResponses();
|
||||
_timeProvider.Advance(TimeSpan.FromHours(1));
|
||||
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var resumeDocOne = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailOne.ToString(), CancellationToken.None);
|
||||
var resumeDocTwo = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailTwo.ToString(), CancellationToken.None);
|
||||
_output.WriteLine($"After resume fetch status: {resumeDocOne?.Status} ({resumeDocOne?.Sha256}), {resumeDocTwo?.Status} ({resumeDocTwo?.Sha256})");
|
||||
Assert.Equal(DocumentStatuses.Mapped, resumeDocOne?.Status);
|
||||
Assert.Equal(DocumentStatuses.Mapped, resumeDocTwo?.Status);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
|
||||
Assert.Equal(3, advisories.Count);
|
||||
Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "VMSA-2024-0003");
|
||||
|
||||
psirtFlags = await psirtCollection.Find(Builders<BsonDocument>.Filter.Empty).ToListAsync();
|
||||
_output.WriteLine("PSIRT flags after resume: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("<missing>")).ToString())));
|
||||
Assert.Equal(3, psirtFlags.Count);
|
||||
Assert.Contains(psirtFlags, doc => doc["_id"] == "VMSA-2024-0003");
|
||||
|
||||
var measurements = metrics.Measurements;
|
||||
_output.WriteLine("Captured metrics:");
|
||||
foreach (var measurement in measurements)
|
||||
{
|
||||
_output.WriteLine($"{measurement.Name} -> {measurement.Value}");
|
||||
}
|
||||
|
||||
Assert.Equal(0, Sum(measurements, "vmware.fetch.failures"));
|
||||
Assert.Equal(0, Sum(measurements, "vmware.parse.fail"));
|
||||
Assert.Equal(3, Sum(measurements, "vmware.fetch.items")); // two initial, one new
|
||||
|
||||
var affectedCounts = measurements
|
||||
.Where(m => m.Name == "vmware.map.affected_count")
|
||||
.Select(m => (int)m.Value)
|
||||
.OrderBy(v => v)
|
||||
.ToArray();
|
||||
Assert.Equal(new[] { 1, 1, 2 }, affectedCounts);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_handler.Clear();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task<ServiceProvider> BuildServiceProviderAsync()
|
||||
{
|
||||
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
|
||||
_handler.Clear();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
|
||||
services.AddSingleton<TimeProvider>(_timeProvider);
|
||||
services.AddSingleton(_handler);
|
||||
|
||||
services.AddMongoStorage(options =>
|
||||
{
|
||||
options.ConnectionString = _fixture.Runner.ConnectionString;
|
||||
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
});
|
||||
|
||||
services.AddSourceCommon();
|
||||
services.AddVmwareConnector(opts =>
|
||||
{
|
||||
opts.IndexUri = IndexUri;
|
||||
opts.InitialBackfill = TimeSpan.FromDays(30);
|
||||
opts.ModifiedTolerance = TimeSpan.FromMinutes(5);
|
||||
opts.MaxAdvisoriesPerFetch = 10;
|
||||
opts.RequestDelay = TimeSpan.Zero;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(VmwareOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler);
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
|
||||
await bootstrapper.InitializeAsync(CancellationToken.None);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private void SeedInitialResponses()
|
||||
{
|
||||
_handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-initial.json"));
|
||||
_handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json"));
|
||||
_handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json"));
|
||||
}
|
||||
|
||||
private void SeedUpdateResponses()
|
||||
{
|
||||
_handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-second.json"));
|
||||
_handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json"));
|
||||
_handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json"));
|
||||
_handler.AddJsonResponse(DetailThree, ReadFixture("vmware-detail-vmsa-2024-0003.json"));
|
||||
}
|
||||
|
||||
private static string ReadFixture(string name)
|
||||
{
|
||||
var primary = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", name);
|
||||
if (File.Exists(primary))
|
||||
{
|
||||
return File.ReadAllText(primary);
|
||||
}
|
||||
|
||||
var fallback = Path.Combine(AppContext.BaseDirectory, "Fixtures", name);
|
||||
if (File.Exists(fallback))
|
||||
{
|
||||
return File.ReadAllText(fallback);
|
||||
}
|
||||
|
||||
throw new FileNotFoundException($"Fixture '{name}' not found.", name);
|
||||
}
|
||||
|
||||
private static string Normalize(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
|
||||
|
||||
private static long Sum(IEnumerable<VmwareMetricCollector.MetricMeasurement> measurements, string name)
|
||||
=> measurements.Where(m => m.Name == name).Sum(m => m.Value);
|
||||
|
||||
private sealed class VmwareMetricCollector : IDisposable
|
||||
{
|
||||
private readonly MeterListener _listener;
|
||||
private readonly ConcurrentBag<MetricMeasurement> _measurements = new();
|
||||
|
||||
public VmwareMetricCollector()
|
||||
{
|
||||
_listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == VmwareDiagnostics.MeterName)
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) =>
|
||||
{
|
||||
var tagList = new List<KeyValuePair<string, object?>>(tags.Length);
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
tagList.Add(tag);
|
||||
}
|
||||
|
||||
_measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList));
|
||||
});
|
||||
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<MetricMeasurement> Measurements => _measurements;
|
||||
|
||||
public void Dispose() => _listener.Dispose();
|
||||
|
||||
public sealed record MetricMeasurement(string Name, long Value, IReadOnlyList<KeyValuePair<string, object?>> Tags);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user