Initial commit (history squashed)
	
		
			
	
		
	
	
		
	
		
			Some checks failed
		
		
	
	
		
			
				
	
				Build Test Deploy / authority-container (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / docs (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / deploy (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / build-test (push) Has been cancelled
				
			
		
			
				
	
				Docs CI / lint-and-preview (push) Has been cancelled
				
			
		
		
	
	
				
					
				
			
		
			Some checks failed
		
		
	
	Build Test Deploy / authority-container (push) Has been cancelled
				
			Build Test Deploy / docs (push) Has been cancelled
				
			Build Test Deploy / deploy (push) Has been cancelled
				
			Build Test Deploy / build-test (push) Has been cancelled
				
			Docs CI / lint-and-preview (push) Has been cancelled
				
			This commit is contained in:
		
							
								
								
									
										28
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										28
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,28 @@ | ||||
| # AGENTS | ||||
| ## Role | ||||
| VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMware products; maps affected versions/builds and emits psirt_flags. | ||||
| ## Scope | ||||
| - Discover/fetch VMSA index and detail pages via Broadcom portal; window by advisory ID/date; follow updates/revisions. | ||||
| - Validate HTML or JSON; extract CVEs, affected product versions/builds, workarounds, fixed versions; normalize product naming. | ||||
| - Persist raw docs with sha256; manage source_state; idempotent mapping. | ||||
| ## Participants | ||||
| - Source.Common (HTTP, cookies/session handling if needed, validators). | ||||
| - Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). | ||||
| - Models (canonical). | ||||
| - Core/WebService (jobs: source:vmware:fetch|parse|map). | ||||
| - Merge engine (later) to prefer PSIRT ranges for VMware products. | ||||
| ## Interfaces & contracts | ||||
| - Aliases: VMSA-YYYY-NNNN plus CVEs. | ||||
| - Affected entries include Vendor=VMware, Product plus component; Versions carry fixed/fixedBy; tags may include build numbers or ESXi/VC levels. | ||||
| - References: advisory URL, KBs, workaround pages; typed; deduped. | ||||
| - Provenance: method=parser; value=VMSA id. | ||||
| ## In/Out of scope | ||||
| In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references. | ||||
| Out: customer portal authentication flows beyond public advisories; downloading patches. | ||||
| ## Observability & security expectations | ||||
| - Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names. | ||||
| - Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely. | ||||
| ## Tests | ||||
| - Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`. | ||||
| - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. | ||||
| - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. | ||||
| @@ -0,0 +1,54 @@ | ||||
| using System.Diagnostics.CodeAnalysis; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Configuration; | ||||
|  | ||||
| public sealed class VmwareOptions | ||||
| { | ||||
|     public const string HttpClientName = "source.vmware"; | ||||
|  | ||||
|     public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute); | ||||
|  | ||||
|     public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); | ||||
|  | ||||
|     public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(2); | ||||
|  | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 50; | ||||
|  | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); | ||||
|  | ||||
|     public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(2); | ||||
|  | ||||
|     [MemberNotNull(nameof(IndexUri))] | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (IndexUri is null || !IndexUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("VMware index URI must be absolute."); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfill <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Initial backfill must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (ModifiedTolerance < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Modified tolerance cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("Max advisories per fetch must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Request delay cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (HttpTimeout <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("HTTP timeout must be positive."); | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,172 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
|  | ||||
| internal sealed record VmwareCursor( | ||||
|     DateTimeOffset? LastModified, | ||||
|     IReadOnlyCollection<string> ProcessedIds, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings, | ||||
|     IReadOnlyDictionary<string, VmwareFetchCacheEntry> FetchCache) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); | ||||
|     private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>(); | ||||
|     private static readonly IReadOnlyDictionary<string, VmwareFetchCacheEntry> EmptyFetchCache = | ||||
|         new Dictionary<string, VmwareFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|     public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyFetchCache); | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastModified.HasValue) | ||||
|         { | ||||
|             document["lastModified"] = LastModified.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (ProcessedIds.Count > 0) | ||||
|         { | ||||
|             document["processedIds"] = new BsonArray(ProcessedIds); | ||||
|         } | ||||
|  | ||||
|         if (FetchCache.Count > 0) | ||||
|         { | ||||
|             var cacheDocument = new BsonDocument(); | ||||
|             foreach (var (key, entry) in FetchCache) | ||||
|             { | ||||
|                 cacheDocument[key] = entry.ToBsonDocument(); | ||||
|             } | ||||
|  | ||||
|             document["fetchCache"] = cacheDocument; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static VmwareCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var lastModified = document.TryGetValue("lastModified", out var value) | ||||
|             ? ParseDate(value) | ||||
|             : null; | ||||
|  | ||||
|         var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray | ||||
|             ? idsArray.OfType<BsonValue>() | ||||
|                 .Where(static x => x.BsonType == BsonType.String) | ||||
|                 .Select(static x => x.AsString) | ||||
|                 .ToArray() | ||||
|             : EmptyStringList; | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|         var fetchCache = ReadFetchCache(document); | ||||
|  | ||||
|         return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings, fetchCache); | ||||
|     } | ||||
|  | ||||
|     public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable<string> processedIds) | ||||
|         => this with | ||||
|         { | ||||
|             LastModified = timestamp.ToUniversalTime(), | ||||
|             ProcessedIds = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) | ||||
|                 .Select(static id => id.Trim()) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray() ?? EmptyStringList, | ||||
|         }; | ||||
|  | ||||
|     public VmwareCursor WithPendingDocuments(IEnumerable<Guid> ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public VmwareCursor WithPendingMappings(IEnumerable<Guid> ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public VmwareCursor WithFetchCache(IDictionary<string, VmwareFetchCacheEntry>? cache) | ||||
|     { | ||||
|         if (cache is null || cache.Count == 0) | ||||
|         { | ||||
|             return this with { FetchCache = EmptyFetchCache }; | ||||
|         } | ||||
|  | ||||
|         return this with { FetchCache = new Dictionary<string, VmwareFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; | ||||
|     } | ||||
|  | ||||
|     public bool TryGetFetchCache(string key, out VmwareFetchCacheEntry entry) | ||||
|     { | ||||
|         if (FetchCache.Count == 0) | ||||
|         { | ||||
|             entry = VmwareFetchCacheEntry.Empty; | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         return FetchCache.TryGetValue(key, out entry!); | ||||
|     } | ||||
|  | ||||
|     public VmwareCursor AddProcessedId(string id) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(id)) | ||||
|         { | ||||
|             return this; | ||||
|         } | ||||
|  | ||||
|         var set = new HashSet<string>(ProcessedIds, StringComparer.OrdinalIgnoreCase) { id.Trim() }; | ||||
|         return this with { ProcessedIds = set.ToArray() }; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidList; | ||||
|         } | ||||
|  | ||||
|         var results = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 results.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, VmwareFetchCacheEntry> ReadFetchCache(BsonDocument document) | ||||
|     { | ||||
|         if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) | ||||
|         { | ||||
|             return EmptyFetchCache; | ||||
|         } | ||||
|  | ||||
|         var cache = new Dictionary<string, VmwareFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var element in cacheDocument.Elements) | ||||
|         { | ||||
|             if (element.Value is BsonDocument entryDocument) | ||||
|             { | ||||
|                 cache[element.Name] = VmwareFetchCacheEntry.FromBson(entryDocument); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return cache; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(BsonValue value) | ||||
|         => value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
|  | ||||
| internal sealed record VmwareDetailDto | ||||
| { | ||||
|     [JsonPropertyName("id")] | ||||
|     public string AdvisoryId { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("title")] | ||||
|     public string Title { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("summary")] | ||||
|     public string? Summary { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("published")] | ||||
|     public DateTimeOffset? Published { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("modified")] | ||||
|     public DateTimeOffset? Modified { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cves")] | ||||
|     public IReadOnlyList<string>? CveIds { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("affected")] | ||||
|     public IReadOnlyList<VmwareAffectedProductDto>? Affected { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("references")] | ||||
|     public IReadOnlyList<VmwareReferenceDto>? References { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record VmwareAffectedProductDto | ||||
| { | ||||
|     [JsonPropertyName("product")] | ||||
|     public string Product { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("version")] | ||||
|     public string? Version { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("fixedVersion")] | ||||
|     public string? FixedVersion { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record VmwareReferenceDto | ||||
| { | ||||
|     [JsonPropertyName("type")] | ||||
|     public string? Type { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("url")] | ||||
|     public string Url { get; init; } = string.Empty; | ||||
| } | ||||
| @@ -0,0 +1,88 @@ | ||||
| using System; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
|  | ||||
| internal sealed record VmwareFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) | ||||
| { | ||||
|     public static VmwareFetchCacheEntry Empty { get; } = new(string.Empty, null, null); | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["sha256"] = Sha256 ?? string.Empty, | ||||
|         }; | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(ETag)) | ||||
|         { | ||||
|             document["etag"] = ETag; | ||||
|         } | ||||
|  | ||||
|         if (LastModified.HasValue) | ||||
|         { | ||||
|             document["lastModified"] = LastModified.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static VmwareFetchCacheEntry FromBson(BsonDocument document) | ||||
|     { | ||||
|         var sha256 = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty; | ||||
|         string? etag = null; | ||||
|         if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull) | ||||
|         { | ||||
|             etag = etagValue.ToString(); | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset? lastModified = null; | ||||
|         if (document.TryGetValue("lastModified", out var lastModifiedValue)) | ||||
|         { | ||||
|             lastModified = lastModifiedValue.BsonType switch | ||||
|             { | ||||
|                 BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), | ||||
|                 BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|                 _ => null, | ||||
|             }; | ||||
|         } | ||||
|  | ||||
|         return new VmwareFetchCacheEntry(sha256, etag, lastModified); | ||||
|     } | ||||
|  | ||||
|     public static VmwareFetchCacheEntry FromDocument(DocumentRecord document) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         return new VmwareFetchCacheEntry( | ||||
|             document.Sha256, | ||||
|             document.Etag, | ||||
|             document.LastModified?.ToUniversalTime()); | ||||
|     } | ||||
|  | ||||
|     public bool Matches(DocumentRecord document) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256) | ||||
|             && string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag) | ||||
|             && string.Equals(ETag, document.Etag, StringComparison.Ordinal)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (LastModified.HasValue && document.LastModified.HasValue | ||||
|             && LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime()) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,16 @@ | ||||
| using System; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
|  | ||||
| internal sealed record VmwareIndexItem | ||||
| { | ||||
|     [JsonPropertyName("id")] | ||||
|     public string Id { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("url")] | ||||
|     public string DetailUrl { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("modified")] | ||||
|     public DateTimeOffset? Modified { get; init; } | ||||
| } | ||||
| @@ -0,0 +1,235 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Packages; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Feedser.Storage.Mongo.PsirtFlags; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
|  | ||||
| internal static class VmwareMapper | ||||
| { | ||||
|     public static (Advisory Advisory, PsirtFlagRecord Flag) Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|         ArgumentNullException.ThrowIfNull(dtoRecord); | ||||
|  | ||||
|         var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); | ||||
|         var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime()); | ||||
|         var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, recordedAt); | ||||
|  | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var affectedPackages = BuildAffectedPackages(dto, recordedAt); | ||||
|  | ||||
|         var advisory = new Advisory( | ||||
|             dto.AdvisoryId, | ||||
|             dto.Title, | ||||
|             dto.Summary, | ||||
|             language: "en", | ||||
|             dto.Published?.ToUniversalTime(), | ||||
|             dto.Modified?.ToUniversalTime(), | ||||
|             severity: null, | ||||
|             exploitKnown: false, | ||||
|             aliases, | ||||
|             references, | ||||
|             affectedPackages, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { fetchProvenance, mappingProvenance }); | ||||
|  | ||||
|         var flag = new PsirtFlagRecord( | ||||
|             dto.AdvisoryId, | ||||
|             "VMware", | ||||
|             VmwareConnectorPlugin.SourceName, | ||||
|             dto.AdvisoryId, | ||||
|             recordedAt); | ||||
|  | ||||
|         return (advisory, flag); | ||||
|     } | ||||
|  | ||||
|     private static IEnumerable<string> BuildAliases(VmwareDetailDto dto) | ||||
|     { | ||||
|         var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId }; | ||||
|         if (dto.CveIds is not null) | ||||
|         { | ||||
|             foreach (var cve in dto.CveIds) | ||||
|             { | ||||
|                 if (!string.IsNullOrWhiteSpace(cve)) | ||||
|                 { | ||||
|                     set.Add(cve.Trim()); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return set; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> BuildReferences(VmwareDetailDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.References is null || dto.References.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryReference>(); | ||||
|         } | ||||
|  | ||||
|         var references = new List<AdvisoryReference>(dto.References.Count); | ||||
|         foreach (var reference in dto.References) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(reference.Url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var kind = NormalizeReferenceKind(reference.Type); | ||||
|             var provenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "reference", reference.Url, recordedAt); | ||||
|             try | ||||
|             { | ||||
|                 references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance)); | ||||
|             } | ||||
|             catch (ArgumentException) | ||||
|             { | ||||
|                 // ignore invalid urls | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); | ||||
|         return references.Count == 0 ? Array.Empty<AdvisoryReference>() : references; | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizeReferenceKind(string? type) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(type)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return type.Trim().ToLowerInvariant() switch | ||||
|         { | ||||
|             "advisory" => "advisory", | ||||
|             "kb" or "kb_article" => "kb", | ||||
|             "patch" => "patch", | ||||
|             "workaround" => "workaround", | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(VmwareDetailDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Affected is null || dto.Affected.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Affected.Count); | ||||
|         foreach (var product in dto.Affected) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(product.Product)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var provenance = new[] | ||||
|             { | ||||
|                 new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "affected", product.Product, recordedAt), | ||||
|             }; | ||||
|  | ||||
|             var ranges = new List<AffectedVersionRange>(); | ||||
|             if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion)) | ||||
|             { | ||||
|                 var rangeProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt); | ||||
|                 ranges.Add(new AffectedVersionRange( | ||||
|                     rangeKind: "vendor", | ||||
|                     introducedVersion: product.Version, | ||||
|                     fixedVersion: product.FixedVersion, | ||||
|                     lastAffectedVersion: null, | ||||
|                     rangeExpression: product.Version, | ||||
|                     provenance: rangeProvenance, | ||||
|                     primitives: BuildRangePrimitives(product))); | ||||
|             } | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 AffectedPackageTypes.Vendor, | ||||
|                 product.Product, | ||||
|                 platform: null, | ||||
|                 versionRanges: ranges, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: provenance)); | ||||
|         } | ||||
|  | ||||
|         return packages; | ||||
|     } | ||||
|  | ||||
|     private static RangePrimitives? BuildRangePrimitives(VmwareAffectedProductDto product) | ||||
|     { | ||||
|         var extensions = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         AddExtension(extensions, "vmware.product", product.Product); | ||||
|         AddExtension(extensions, "vmware.version.raw", product.Version); | ||||
|         AddExtension(extensions, "vmware.fixedVersion.raw", product.FixedVersion); | ||||
|  | ||||
|         var semVer = BuildSemVerPrimitive(product.Version, product.FixedVersion); | ||||
|         if (semVer is null && extensions.Count == 0) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); | ||||
|     } | ||||
|  | ||||
|     private static SemVerPrimitive? BuildSemVerPrimitive(string? introduced, string? fixedVersion) | ||||
|     { | ||||
|         var introducedNormalized = NormalizeSemVer(introduced); | ||||
|         var fixedNormalized = NormalizeSemVer(fixedVersion); | ||||
|  | ||||
|         if (introducedNormalized is null && fixedNormalized is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new SemVerPrimitive( | ||||
|             introducedNormalized, | ||||
|             IntroducedInclusive: true, | ||||
|             fixedNormalized, | ||||
|             FixedInclusive: false, | ||||
|             LastAffected: null, | ||||
|             LastAffectedInclusive: false, | ||||
|             ConstraintExpression: null); | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizeSemVer(string? value) | ||||
|     { | ||||
|         if (PackageCoordinateHelper.TryParseSemVer(value, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) | ||||
|         { | ||||
|             return normalized; | ||||
|         } | ||||
|  | ||||
|         if (Version.TryParse(value, out var parsed)) | ||||
|         { | ||||
|             if (parsed.Build >= 0 && parsed.Revision >= 0) | ||||
|             { | ||||
|                 return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; | ||||
|             } | ||||
|  | ||||
|             if (parsed.Build >= 0) | ||||
|             { | ||||
|                 return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; | ||||
|             } | ||||
|  | ||||
|             return $"{parsed.Major}.{parsed.Minor}"; | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static void AddExtension(Dictionary<string, string> extensions, string key, string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         extensions[key] = value.Trim(); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										46
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										46
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,46 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| internal static class VmwareJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:vmware:fetch"; | ||||
|     public const string Parse = "source:vmware:parse"; | ||||
|     public const string Map = "source:vmware:map"; | ||||
| } | ||||
|  | ||||
| internal sealed class VmwareFetchJob : IJob | ||||
| { | ||||
|     private readonly VmwareConnector _connector; | ||||
|  | ||||
|     public VmwareFetchJob(VmwareConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class VmwareParseJob : IJob | ||||
| { | ||||
|     private readonly VmwareConnector _connector; | ||||
|  | ||||
|     public VmwareParseJob(VmwareConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.ParseAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class VmwareMapJob : IJob | ||||
| { | ||||
|     private readonly VmwareConnector _connector; | ||||
|  | ||||
|     public VmwareMapJob(VmwareConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.MapAsync(context.Services, cancellationToken); | ||||
| } | ||||
| @@ -0,0 +1,3 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Vmware.Tests")] | ||||
| @@ -0,0 +1,23 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj" /> | ||||
|   </ItemGroup> | ||||
|   <ItemGroup> | ||||
|     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> | ||||
|       <_Parameter1>StellaOps.Feedser.Tests</_Parameter1> | ||||
|     </AssemblyAttribute> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|  | ||||
							
								
								
									
										17
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										17
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,17 @@ | ||||
| # Source.Vndr.Vmware — Task Board | ||||
|  | ||||
| | ID   | Task                                          | Owner | Status | Depends On | Notes | | ||||
| |------|-----------------------------------------------|-------|--------|------------|-------| | ||||
| | VM1  | Advisory listing discovery + cursor           | Conn  | DONE   | Common     | **DONE** – fetch pipeline uses index JSON with sliding cursor + processed id tracking. | | ||||
| | VM2  | VMSA parser → DTO                             | QA    | DONE   |            | **DONE** – JSON DTO deserialization wired with sanitization. | | ||||
| | VM3  | Canonical mapping (aliases/affected/refs)     | Conn  | DONE   | Models     | **DONE** – `VmwareMapper` emits aliases/affected/reference ordering and persists PSIRT flags via `PsirtFlagStore`. | | ||||
| | VM4  | Snapshot tests + resume                       | QA    | DONE   | Storage    | **DONE** – integration test validates snapshot output and resume flow with cached state. | | ||||
| | VM5  | Observability                                 | QA    | DONE   |            | **DONE** – diagnostics meter exposes fetch/parse/map metrics and structured logs. | | ||||
| | VM6  | SourceState + hash dedupe                     | Conn  | DONE   | Storage    | **DONE** – fetch cache stores sha/etag to skip unchanged advisories during resume. | | ||||
| | VM6a | Options & HttpClient configuration | Conn  | DONE   | Source.Common | **DONE** – `AddVmwareConnector` configures allowlisted HttpClient + options. | | ||||
| | VM7  | Dependency injection routine & scheduler registration | Conn  | DONE   | Core        | **DONE** – `VmwareDependencyInjectionRoutine` registers fetch/parse/map jobs. | | ||||
| | VM8  | Replace stub plugin with connector pipeline skeleton | Conn  | DONE   | Source.Common | **DONE** – connector implements fetch/parse/map persisting docs, DTOs, advisories. | | ||||
| | VM9  | Range primitives + provenance diagnostics refresh | Conn  | DONE   | Models, Storage.Mongo | Vendor primitives emitted (SemVer + vendor extensions), provenance tags/logging updated, snapshots refreshed. | | ||||
|  | ||||
| ## Changelog | ||||
| - YYYY-MM-DD: Created. | ||||
							
								
								
									
										454
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										454
									
								
								src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,454 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Net.Http; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using MongoDB.Bson.IO; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; | ||||
| using StellaOps.Feedser.Source.Vndr.Vmware.Internal; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Feedser.Storage.Mongo.PsirtFlags; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| public sealed class VmwareConnector : IFeedConnector | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, | ||||
|     }; | ||||
|  | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly IPsirtFlagStore _psirtFlagStore; | ||||
|     private readonly VmwareOptions _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly VmwareDiagnostics _diagnostics; | ||||
|     private readonly ILogger<VmwareConnector> _logger; | ||||
|  | ||||
|     public VmwareConnector( | ||||
|         IHttpClientFactory httpClientFactory, | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IPsirtFlagStore psirtFlagStore, | ||||
|         IOptions<VmwareOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         VmwareDiagnostics diagnostics, | ||||
|         ILogger<VmwareConnector> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => VmwareConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var fetchCache = new Dictionary<string, VmwareFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); | ||||
|         var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         var remainingCapacity = _options.MaxAdvisoriesPerFetch; | ||||
|  | ||||
|         IReadOnlyList<VmwareIndexItem> indexItems; | ||||
|         try | ||||
|         { | ||||
|             indexItems = await FetchIndexAsync(cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _diagnostics.FetchFailure(); | ||||
|             _logger.LogError(ex, "Failed to retrieve VMware advisory index"); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         if (indexItems.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var orderedItems = indexItems | ||||
|             .Where(static item => !string.IsNullOrWhiteSpace(item.Id) && !string.IsNullOrWhiteSpace(item.DetailUrl)) | ||||
|             .OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue) | ||||
|             .ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|  | ||||
|         var baseline = cursor.LastModified ?? now - _options.InitialBackfill; | ||||
|         var resumeStart = baseline - _options.ModifiedTolerance; | ||||
|         ProvenanceDiagnostics.ReportResumeWindow(SourceName, resumeStart, _logger); | ||||
|         var processedIds = new HashSet<string>(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); | ||||
|         var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; | ||||
|         var processedUpdated = false; | ||||
|  | ||||
|         foreach (var item in orderedItems) | ||||
|         { | ||||
|             if (remainingCapacity <= 0) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var modified = (item.Modified ?? DateTimeOffset.MinValue).ToUniversalTime(); | ||||
|             if (modified < baseline - _options.ModifiedTolerance) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (cursor.LastModified.HasValue && modified < cursor.LastModified.Value - _options.ModifiedTolerance) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (modified == cursor.LastModified && cursor.ProcessedIds.Contains(item.Id, StringComparer.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!Uri.TryCreate(item.DetailUrl, UriKind.Absolute, out var detailUri)) | ||||
|             { | ||||
|                 _logger.LogWarning("VMware advisory {AdvisoryId} has invalid detail URL {Url}", item.Id, item.DetailUrl); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var cacheKey = detailUri.AbsoluteUri; | ||||
|             touchedResources.Add(cacheKey); | ||||
|  | ||||
|             var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); | ||||
|             var metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|             { | ||||
|                 ["vmware.id"] = item.Id, | ||||
|                 ["vmware.modified"] = modified.ToString("O"), | ||||
|             }; | ||||
|  | ||||
|             SourceFetchResult result; | ||||
|             try | ||||
|             { | ||||
|                 result = await _fetchService.FetchAsync( | ||||
|                     new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, detailUri) | ||||
|                     { | ||||
|                         Metadata = metadata, | ||||
|                         ETag = existing?.Etag, | ||||
|                         LastModified = existing?.LastModified, | ||||
|                         AcceptHeaders = new[] { "application/json" }, | ||||
|                     }, | ||||
|                     cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 _logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id); | ||||
|                 await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             if (result.IsNotModified) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|                 if (existing is not null) | ||||
|                 { | ||||
|                     fetchCache[cacheKey] = VmwareFetchCacheEntry.FromDocument(existing); | ||||
|                     pendingDocuments.Remove(existing.Id); | ||||
|                     pendingMappings.Remove(existing.Id); | ||||
|                     _logger.LogInformation("VMware advisory {AdvisoryId} returned 304 Not Modified", item.Id); | ||||
|                 } | ||||
|  | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!result.IsSuccess || result.Document is null) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             remainingCapacity--; | ||||
|  | ||||
|             if (modified > maxModified) | ||||
|             { | ||||
|                 maxModified = modified; | ||||
|                 processedIds.Clear(); | ||||
|                 processedUpdated = true; | ||||
|             } | ||||
|  | ||||
|             if (modified == maxModified) | ||||
|             { | ||||
|                 processedIds.Add(item.Id); | ||||
|                 processedUpdated = true; | ||||
|             } | ||||
|  | ||||
|             var cacheEntry = VmwareFetchCacheEntry.FromDocument(result.Document); | ||||
|  | ||||
|             if (existing is not null | ||||
|                 && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal) | ||||
|                 && cursor.TryGetFetchCache(cacheKey, out var cachedEntry) | ||||
|                 && cachedEntry.Matches(result.Document)) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|                 fetchCache[cacheKey] = cacheEntry; | ||||
|                 pendingDocuments.Remove(result.Document.Id); | ||||
|                 pendingMappings.Remove(result.Document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); | ||||
|                 _logger.LogInformation("VMware advisory {AdvisoryId} unchanged; skipping reprocessing", item.Id); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             _diagnostics.FetchItem(); | ||||
|             fetchCache[cacheKey] = cacheEntry; | ||||
|             pendingDocuments.Add(result.Document.Id); | ||||
|             _logger.LogInformation( | ||||
|                 "VMware advisory {AdvisoryId} fetched (documentId={DocumentId}, sha256={Sha})", | ||||
|                 item.Id, | ||||
|                 result.Document.Id, | ||||
|                 result.Document.Sha256); | ||||
|  | ||||
|             if (_options.RequestDelay > TimeSpan.Zero) | ||||
|             { | ||||
|                 try | ||||
|                 { | ||||
|                     await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 catch (TaskCanceledException) | ||||
|                 { | ||||
|                     break; | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (fetchCache.Count > 0 && touchedResources.Count > 0) | ||||
|         { | ||||
|             var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); | ||||
|             foreach (var key in stale) | ||||
|             { | ||||
|                 fetchCache.Remove(key); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings) | ||||
|             .WithFetchCache(fetchCache); | ||||
|  | ||||
|         if (processedUpdated) | ||||
|         { | ||||
|             updatedCursor = updatedCursor.WithLastModified(maxModified, processedIds); | ||||
|         } | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remaining = cursor.PendingDocuments.ToList(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 remaining.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             byte[] bytes; | ||||
|             try | ||||
|             { | ||||
|                 bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed downloading VMware document {DocumentId}", document.Id); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             VmwareDetailDto? detail; | ||||
|             try | ||||
|             { | ||||
|                 detail = JsonSerializer.Deserialize<VmwareDetailDto>(bytes, SerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) | ||||
|             { | ||||
|                 _logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var sanitized = JsonSerializer.Serialize(detail, SerializerOptions); | ||||
|             var payload = MongoDB.Bson.BsonDocument.Parse(sanitized); | ||||
|             var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "vmware.v1", payload, _timeProvider.GetUtcNow()); | ||||
|  | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             remaining.Remove(documentId); | ||||
|             if (!pendingMappings.Contains(documentId)) | ||||
|             { | ||||
|                 pendingMappings.Add(documentId); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remaining) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (dto is null || document is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var json = dto.Payload.ToJson(new JsonWriterSettings | ||||
|             { | ||||
|                 OutputMode = JsonOutputMode.RelaxedExtendedJson, | ||||
|             }); | ||||
|  | ||||
|             VmwareDetailDto? detail; | ||||
|             try | ||||
|             { | ||||
|                 detail = JsonSerializer.Deserialize<VmwareDetailDto>(json, SerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed to deserialize VMware DTO for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) | ||||
|             { | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var (advisory, flag) = VmwareMapper.Map(detail, document, dto); | ||||
|             await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|             await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|             _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); | ||||
|             _logger.LogInformation( | ||||
|                 "VMware advisory {AdvisoryId} mapped with {AffectedCount} affected packages", | ||||
|                 detail.AdvisoryId, | ||||
|                 advisory.AffectedPackages.Length); | ||||
|  | ||||
|             pendingMappings.Remove(documentId); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private async Task<IReadOnlyList<VmwareIndexItem>> FetchIndexAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName); | ||||
|         using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false); | ||||
|         response.EnsureSuccessStatusCode(); | ||||
|  | ||||
|         await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var items = await JsonSerializer.DeserializeAsync<IReadOnlyList<VmwareIndexItem>>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|         return items ?? Array.Empty<VmwareIndexItem>(); | ||||
|     } | ||||
|  | ||||
|     private async Task<VmwareCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? VmwareCursor.Empty : VmwareCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(VmwareCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBsonDocument(); | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,20 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| public sealed class VmwareConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public static string SourceName => "vmware"; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => services is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return ActivatorUtilities.CreateInstance<VmwareConnector>(services); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
| using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "feedser:sources:vmware"; | ||||
|     private const string FetchCron = "10,40 * * * *"; | ||||
|     private const string ParseCron = "15,45 * * * *"; | ||||
|     private const string MapCron = "20,50 * * * *"; | ||||
|  | ||||
|     private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(10); | ||||
|     private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); | ||||
|     private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(15); | ||||
|     private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddVmwareConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         var scheduler = new JobSchedulerBuilder(services); | ||||
|         scheduler | ||||
|             .AddJob<VmwareFetchJob>( | ||||
|                 VmwareJobKinds.Fetch, | ||||
|                 cronExpression: FetchCron, | ||||
|                 timeout: FetchTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<VmwareParseJob>( | ||||
|                 VmwareJobKinds.Parse, | ||||
|                 cronExpression: ParseCron, | ||||
|                 timeout: ParseTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<VmwareMapJob>( | ||||
|                 VmwareJobKinds.Map, | ||||
|                 cronExpression: MapCron, | ||||
|                 timeout: MapTimeout, | ||||
|                 leaseDuration: LeaseDuration); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,67 @@ | ||||
| using System; | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| /// <summary> | ||||
| /// VMware connector metrics (fetch, parse, map). | ||||
| /// </summary> | ||||
| public sealed class VmwareDiagnostics : IDisposable | ||||
| { | ||||
|     public const string MeterName = "StellaOps.Feedser.Source.Vndr.Vmware"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _fetchItems; | ||||
|     private readonly Counter<long> _fetchFailures; | ||||
|     private readonly Counter<long> _fetchUnchanged; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Histogram<long> _mapAffectedCount; | ||||
|  | ||||
|     public VmwareDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _fetchItems = _meter.CreateCounter<long>( | ||||
|             name: "vmware.fetch.items", | ||||
|             unit: "documents", | ||||
|             description: "Number of VMware advisory documents fetched."); | ||||
|         _fetchFailures = _meter.CreateCounter<long>( | ||||
|             name: "vmware.fetch.failures", | ||||
|             unit: "operations", | ||||
|             description: "Number of VMware fetch failures."); | ||||
|         _fetchUnchanged = _meter.CreateCounter<long>( | ||||
|             name: "vmware.fetch.unchanged", | ||||
|             unit: "documents", | ||||
|             description: "Number of VMware advisories skipped due to unchanged content."); | ||||
|         _parseFailures = _meter.CreateCounter<long>( | ||||
|             name: "vmware.parse.fail", | ||||
|             unit: "documents", | ||||
|             description: "Number of VMware advisory documents that failed to parse."); | ||||
|         _mapAffectedCount = _meter.CreateHistogram<long>( | ||||
|             name: "vmware.map.affected_count", | ||||
|             unit: "packages", | ||||
|             description: "Distribution of affected-package counts emitted per VMware advisory."); | ||||
|     } | ||||
|  | ||||
|     public void FetchItem() => _fetchItems.Add(1); | ||||
|  | ||||
|     public void FetchFailure() => _fetchFailures.Add(1); | ||||
|  | ||||
|     public void FetchUnchanged() => _fetchUnchanged.Add(1); | ||||
|  | ||||
|     public void ParseFailure() => _parseFailures.Add(1); | ||||
|  | ||||
|     public void MapAffectedCount(int count) | ||||
|     { | ||||
|         if (count < 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         _mapAffectedCount.Record(count); | ||||
|     } | ||||
|  | ||||
|     public Meter Meter => _meter; | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
| } | ||||
| @@ -0,0 +1,39 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Common.Http; | ||||
| using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Vmware; | ||||
|  | ||||
| public static class VmwareServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddVmwareConnector(this IServiceCollection services, Action<VmwareOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<VmwareOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static opts => opts.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(VmwareOptions.HttpClientName, (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<VmwareOptions>>().Value; | ||||
|             clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority)); | ||||
|             clientOptions.Timeout = options.HttpTimeout; | ||||
|             clientOptions.UserAgent = "StellaOps.Feedser.VMware/1.0"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.IndexUri.Host); | ||||
|             clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<VmwareDiagnostics>(); | ||||
|         services.AddTransient<VmwareConnector>(); | ||||
|         services.AddTransient<VmwareFetchJob>(); | ||||
|         services.AddTransient<VmwareParseJob>(); | ||||
|         services.AddTransient<VmwareMapJob>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
		Reference in New Issue
	
	Block a user