using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Metrics; using System.IO; using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; using StellaOps.Concelier.Models; using StellaOps.Concelier.Connector.Common; using StellaOps.Concelier.Connector.Common.Http; using StellaOps.Concelier.Connector.Common.Testing; using StellaOps.Concelier.Connector.Vndr.Vmware; using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; using StellaOps.Concelier.Connector.Vndr.Vmware.Internal; using StellaOps.Concelier.Storage.Mongo; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Concelier.Testing; using Xunit.Abstractions; namespace StellaOps.Concelier.Connector.Vndr.Vmware.Tests.Vmware; [Collection("mongo-fixture")] public sealed class VmwareConnectorTests : IAsyncLifetime { private readonly MongoIntegrationFixture _fixture; private readonly FakeTimeProvider _timeProvider; private readonly CannedHttpMessageHandler _handler; private readonly ITestOutputHelper _output; private static readonly Uri IndexUri = new("https://vmware.example/api/vmsa/index.json"); private static readonly Uri DetailOne = new("https://vmware.example/api/vmsa/VMSA-2024-0001.json"); private static readonly Uri DetailTwo = new("https://vmware.example/api/vmsa/VMSA-2024-0002.json"); private static readonly Uri DetailThree = new("https://vmware.example/api/vmsa/VMSA-2024-0003.json"); public VmwareConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) { _fixture = fixture; _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 5, 0, 0, 0, TimeSpan.Zero)); _handler = new CannedHttpMessageHandler(); _output = output; } [Fact] public async Task FetchParseMap_ProducesSnapshotAndCoversResume() { await using var provider = await BuildServiceProviderAsync(); SeedInitialResponses(); using var metrics = new VmwareMetricCollector(); var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); await connector.ParseAsync(provider, CancellationToken.None); await connector.MapAsync(provider, CancellationToken.None); var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); var snapshot = Normalize(SnapshotSerializer.ToSnapshot(ordered)); var expected = Normalize(ReadFixture("vmware-advisories.snapshot.json")); if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) { var actualPath = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", "vmware-advisories.actual.json"); Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); File.WriteAllText(actualPath, snapshot); } Assert.Equal(expected, snapshot); var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); var psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); _output.WriteLine("PSIRT flags after initial map: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); Assert.Equal(2, psirtFlags.Count); Assert.All(psirtFlags, doc => Assert.Equal("VMware", doc["vendor"].AsString)); var stateRepository = provider.GetRequiredService(); var state = await stateRepository.TryGetAsync(VmwareConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); Assert.Empty(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray()); Assert.Empty(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) ? pendingMaps.AsBsonArray : new BsonArray()); var cursorSnapshot = VmwareCursor.FromBson(state.Cursor); _output.WriteLine($"Initial fetch cache entries: {cursorSnapshot.FetchCache.Count}"); foreach (var entry in cursorSnapshot.FetchCache) { _output.WriteLine($"Cache seed: {entry.Key} -> {entry.Value.Sha256}"); } // Second run with unchanged advisories and one new advisory. SeedUpdateResponses(); _timeProvider.Advance(TimeSpan.FromHours(1)); await connector.FetchAsync(provider, CancellationToken.None); var documentStore = provider.GetRequiredService(); var resumeDocOne = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailOne.ToString(), CancellationToken.None); var resumeDocTwo = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailTwo.ToString(), CancellationToken.None); _output.WriteLine($"After resume fetch status: {resumeDocOne?.Status} ({resumeDocOne?.Sha256}), {resumeDocTwo?.Status} ({resumeDocTwo?.Sha256})"); Assert.Equal(DocumentStatuses.Mapped, resumeDocOne?.Status); Assert.Equal(DocumentStatuses.Mapped, resumeDocTwo?.Status); await connector.ParseAsync(provider, CancellationToken.None); await connector.MapAsync(provider, CancellationToken.None); advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.Equal(3, advisories.Count); Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "VMSA-2024-0003"); psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); _output.WriteLine("PSIRT flags after resume: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); Assert.Equal(3, psirtFlags.Count); Assert.Contains(psirtFlags, doc => doc["_id"] == "VMSA-2024-0003"); var measurements = metrics.Measurements; _output.WriteLine("Captured metrics:"); foreach (var measurement in measurements) { _output.WriteLine($"{measurement.Name} -> {measurement.Value}"); } Assert.Equal(0, Sum(measurements, "vmware.fetch.failures")); Assert.Equal(0, Sum(measurements, "vmware.parse.fail")); Assert.Equal(3, Sum(measurements, "vmware.fetch.items")); // two initial, one new var affectedCounts = measurements .Where(m => m.Name == "vmware.map.affected_count") .Select(m => (int)m.Value) .OrderBy(v => v) .ToArray(); Assert.Equal(new[] { 1, 1, 2 }, affectedCounts); } public Task InitializeAsync() => Task.CompletedTask; public Task DisposeAsync() { _handler.Clear(); return Task.CompletedTask; } private async Task BuildServiceProviderAsync() { await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); _handler.Clear(); var services = new ServiceCollection(); services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); services.AddSingleton(_timeProvider); services.AddSingleton(_handler); services.AddMongoStorage(options => { options.ConnectionString = _fixture.Runner.ConnectionString; options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; options.CommandTimeout = TimeSpan.FromSeconds(5); }); services.AddSourceCommon(); services.AddVmwareConnector(opts => { opts.IndexUri = IndexUri; opts.InitialBackfill = TimeSpan.FromDays(30); opts.ModifiedTolerance = TimeSpan.FromMinutes(5); opts.MaxAdvisoriesPerFetch = 10; opts.RequestDelay = TimeSpan.Zero; }); services.Configure(VmwareOptions.HttpClientName, builderOptions => { builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); }); var provider = services.BuildServiceProvider(); var bootstrapper = provider.GetRequiredService(); await bootstrapper.InitializeAsync(CancellationToken.None); return provider; } private void SeedInitialResponses() { _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-initial.json")); _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); } private void SeedUpdateResponses() { _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-second.json")); _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); _handler.AddJsonResponse(DetailThree, ReadFixture("vmware-detail-vmsa-2024-0003.json")); } private static string ReadFixture(string name) { var primary = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", name); if (File.Exists(primary)) { return File.ReadAllText(primary); } var fallback = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); if (File.Exists(fallback)) { return File.ReadAllText(fallback); } throw new FileNotFoundException($"Fixture '{name}' not found.", name); } private static string Normalize(string value) => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); private static long Sum(IEnumerable measurements, string name) => measurements.Where(m => m.Name == name).Sum(m => m.Value); private sealed class VmwareMetricCollector : IDisposable { private readonly MeterListener _listener; private readonly ConcurrentBag _measurements = new(); public VmwareMetricCollector() { _listener = new MeterListener { InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name == VmwareDiagnostics.MeterName) { listener.EnableMeasurementEvents(instrument); } } }; _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => { var tagList = new List>(tags.Length); foreach (var tag in tags) { tagList.Add(tag); } _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList)); }); _listener.Start(); } public IReadOnlyCollection Measurements => _measurements; public void Dispose() => _listener.Dispose(); public sealed record MetricMeasurement(string Name, long Value, IReadOnlyList> Tags); } }