Restructure solution layout by module
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMware products; maps affected versions/builds and emits psirt_flags.
|
||||
## Scope
|
||||
- Discover/fetch VMSA index and detail pages via Broadcom portal; window by advisory ID/date; follow updates/revisions.
|
||||
- Validate HTML or JSON; extract CVEs, affected product versions/builds, workarounds, fixed versions; normalize product naming.
|
||||
- Persist raw docs with sha256; manage source_state; idempotent mapping.
|
||||
## Participants
|
||||
- Source.Common (HTTP, cookies/session handling if needed, validators).
|
||||
- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state).
|
||||
- Models (canonical).
|
||||
- Core/WebService (jobs: source:vmware:fetch|parse|map).
|
||||
- Merge engine (later) to prefer PSIRT ranges for VMware products.
|
||||
## Interfaces & contracts
|
||||
- Aliases: VMSA-YYYY-NNNN plus CVEs.
|
||||
- Affected entries include Vendor=VMware, Product plus component; Versions carry fixed/fixedBy; tags may include build numbers or ESXi/VC levels.
|
||||
- References: advisory URL, KBs, workaround pages; typed; deduped.
|
||||
- Provenance: method=parser; value=VMSA id.
|
||||
## In/Out of scope
|
||||
In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references.
|
||||
Out: customer portal authentication flows beyond public advisories; downloading patches.
|
||||
## Observability & security expectations
|
||||
- Metrics: SourceDiagnostics emits shared `concelier.source.http.*` counters/histograms tagged `concelier.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names.
|
||||
- Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Vmware.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Configuration;
|
||||
|
||||
public sealed class VmwareOptions
|
||||
{
|
||||
public const string HttpClientName = "source.vmware";
|
||||
|
||||
public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute);
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(2);
|
||||
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 50;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
[MemberNotNull(nameof(IndexUri))]
|
||||
public void Validate()
|
||||
{
|
||||
if (IndexUri is null || !IndexUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("VMware index URI must be absolute.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Initial backfill must be positive.");
|
||||
}
|
||||
|
||||
if (ModifiedTolerance < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Modified tolerance cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("Max advisories per fetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Request delay cannot be negative.");
|
||||
}
|
||||
|
||||
if (HttpTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("HTTP timeout must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareCursor(
|
||||
DateTimeOffset? LastModified,
|
||||
IReadOnlyCollection<string> ProcessedIds,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, VmwareFetchCacheEntry> FetchCache)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
|
||||
private static readonly IReadOnlyDictionary<string, VmwareFetchCacheEntry> EmptyFetchCache =
|
||||
new Dictionary<string, VmwareFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyFetchCache);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastModified.HasValue)
|
||||
{
|
||||
document["lastModified"] = LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (ProcessedIds.Count > 0)
|
||||
{
|
||||
document["processedIds"] = new BsonArray(ProcessedIds);
|
||||
}
|
||||
|
||||
if (FetchCache.Count > 0)
|
||||
{
|
||||
var cacheDocument = new BsonDocument();
|
||||
foreach (var (key, entry) in FetchCache)
|
||||
{
|
||||
cacheDocument[key] = entry.ToBsonDocument();
|
||||
}
|
||||
|
||||
document["fetchCache"] = cacheDocument;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static VmwareCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastModified = document.TryGetValue("lastModified", out var value)
|
||||
? ParseDate(value)
|
||||
: null;
|
||||
|
||||
var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray
|
||||
? idsArray.OfType<BsonValue>()
|
||||
.Where(static x => x.BsonType == BsonType.String)
|
||||
.Select(static x => x.AsString)
|
||||
.ToArray()
|
||||
: EmptyStringList;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var fetchCache = ReadFetchCache(document);
|
||||
|
||||
return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings, fetchCache);
|
||||
}
|
||||
|
||||
public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable<string> processedIds)
|
||||
=> this with
|
||||
{
|
||||
LastModified = timestamp.ToUniversalTime(),
|
||||
ProcessedIds = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? EmptyStringList,
|
||||
};
|
||||
|
||||
public VmwareCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public VmwareCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public VmwareCursor WithFetchCache(IDictionary<string, VmwareFetchCacheEntry>? cache)
|
||||
{
|
||||
if (cache is null || cache.Count == 0)
|
||||
{
|
||||
return this with { FetchCache = EmptyFetchCache };
|
||||
}
|
||||
|
||||
return this with { FetchCache = new Dictionary<string, VmwareFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) };
|
||||
}
|
||||
|
||||
public bool TryGetFetchCache(string key, out VmwareFetchCacheEntry entry)
|
||||
{
|
||||
if (FetchCache.Count == 0)
|
||||
{
|
||||
entry = VmwareFetchCacheEntry.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
return FetchCache.TryGetValue(key, out entry!);
|
||||
}
|
||||
|
||||
public VmwareCursor AddProcessedId(string id)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(ProcessedIds, StringComparer.OrdinalIgnoreCase) { id.Trim() };
|
||||
return this with { ProcessedIds = set.ToArray() };
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, VmwareFetchCacheEntry> ReadFetchCache(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0)
|
||||
{
|
||||
return EmptyFetchCache;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, VmwareFetchCacheEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var element in cacheDocument.Elements)
|
||||
{
|
||||
if (element.Value is BsonDocument entryDocument)
|
||||
{
|
||||
cache[element.Name] = VmwareFetchCacheEntry.FromBson(entryDocument);
|
||||
}
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareDetailDto
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("cves")]
|
||||
public IReadOnlyList<string>? CveIds { get; init; }
|
||||
|
||||
[JsonPropertyName("affected")]
|
||||
public IReadOnlyList<VmwareAffectedProductDto>? Affected { get; init; }
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<VmwareReferenceDto>? References { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record VmwareAffectedProductDto
|
||||
{
|
||||
[JsonPropertyName("product")]
|
||||
public string Product { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; init; }
|
||||
|
||||
[JsonPropertyName("fixedVersion")]
|
||||
public string? FixedVersion { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record VmwareReferenceDto
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string? Type { get; init; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string Url { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified)
|
||||
{
|
||||
public static VmwareFetchCacheEntry Empty { get; } = new(string.Empty, null, null);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["sha256"] = Sha256 ?? string.Empty,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ETag))
|
||||
{
|
||||
document["etag"] = ETag;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue)
|
||||
{
|
||||
document["lastModified"] = LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static VmwareFetchCacheEntry FromBson(BsonDocument document)
|
||||
{
|
||||
var sha256 = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty;
|
||||
string? etag = null;
|
||||
if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull)
|
||||
{
|
||||
etag = etagValue.ToString();
|
||||
}
|
||||
|
||||
DateTimeOffset? lastModified = null;
|
||||
if (document.TryGetValue("lastModified", out var lastModifiedValue))
|
||||
{
|
||||
lastModified = lastModifiedValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
return new VmwareFetchCacheEntry(sha256, etag, lastModified);
|
||||
}
|
||||
|
||||
public static VmwareFetchCacheEntry FromDocument(DocumentRecord document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
return new VmwareFetchCacheEntry(
|
||||
document.Sha256,
|
||||
document.Etag,
|
||||
document.LastModified?.ToUniversalTime());
|
||||
}
|
||||
|
||||
public bool Matches(DocumentRecord document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256)
|
||||
&& string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag)
|
||||
&& string.Equals(ETag, document.Etag, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (LastModified.HasValue && document.LastModified.HasValue
|
||||
&& LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareIndexItem
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string DetailUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Packages;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
|
||||
internal static class VmwareMapper
|
||||
{
|
||||
public static (Advisory Advisory, PsirtFlagRecord Flag) Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
|
||||
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
|
||||
var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime());
|
||||
var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, recordedAt);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var affectedPackages = BuildAffectedPackages(dto, recordedAt);
|
||||
|
||||
var advisory = new Advisory(
|
||||
dto.AdvisoryId,
|
||||
dto.Title,
|
||||
dto.Summary,
|
||||
language: "en",
|
||||
dto.Published?.ToUniversalTime(),
|
||||
dto.Modified?.ToUniversalTime(),
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
references,
|
||||
affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mappingProvenance });
|
||||
|
||||
var flag = new PsirtFlagRecord(
|
||||
dto.AdvisoryId,
|
||||
"VMware",
|
||||
VmwareConnectorPlugin.SourceName,
|
||||
dto.AdvisoryId,
|
||||
recordedAt);
|
||||
|
||||
return (advisory, flag);
|
||||
}
|
||||
|
||||
private static IEnumerable<string> BuildAliases(VmwareDetailDto dto)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId };
|
||||
if (dto.CveIds is not null)
|
||||
{
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
set.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(VmwareDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.References is null || dto.References.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>(dto.References.Count);
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var kind = NormalizeReferenceKind(reference.Type);
|
||||
var provenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "reference", reference.Url, recordedAt);
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore invalid urls
|
||||
}
|
||||
}
|
||||
|
||||
references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
|
||||
return references.Count == 0 ? Array.Empty<AdvisoryReference>() : references;
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceKind(string? type)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return type.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"advisory" => "advisory",
|
||||
"kb" or "kb_article" => "kb",
|
||||
"patch" => "patch",
|
||||
"workaround" => "workaround",
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(VmwareDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Affected is null || dto.Affected.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Affected.Count);
|
||||
foreach (var product in dto.Affected)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.Product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "affected", product.Product, recordedAt),
|
||||
};
|
||||
|
||||
var ranges = new List<AffectedVersionRange>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion))
|
||||
{
|
||||
var rangeProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt);
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: product.Version,
|
||||
fixedVersion: product.FixedVersion,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.Version,
|
||||
provenance: rangeProvenance,
|
||||
primitives: BuildRangePrimitives(product)));
|
||||
}
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
product.Product,
|
||||
platform: null,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static RangePrimitives? BuildRangePrimitives(VmwareAffectedProductDto product)
|
||||
{
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
AddExtension(extensions, "vmware.product", product.Product);
|
||||
AddExtension(extensions, "vmware.version.raw", product.Version);
|
||||
AddExtension(extensions, "vmware.fixedVersion.raw", product.FixedVersion);
|
||||
|
||||
var semVer = BuildSemVerPrimitive(product.Version, product.FixedVersion);
|
||||
if (semVer is null && extensions.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions);
|
||||
}
|
||||
|
||||
private static SemVerPrimitive? BuildSemVerPrimitive(string? introduced, string? fixedVersion)
|
||||
{
|
||||
var introducedNormalized = NormalizeSemVer(introduced);
|
||||
var fixedNormalized = NormalizeSemVer(fixedVersion);
|
||||
|
||||
if (introducedNormalized is null && fixedNormalized is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new SemVerPrimitive(
|
||||
introducedNormalized,
|
||||
IntroducedInclusive: true,
|
||||
fixedNormalized,
|
||||
FixedInclusive: false,
|
||||
LastAffected: null,
|
||||
LastAffectedInclusive: false,
|
||||
ConstraintExpression: null);
|
||||
}
|
||||
|
||||
private static string? NormalizeSemVer(string? value)
|
||||
{
|
||||
if (PackageCoordinateHelper.TryParseSemVer(value, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (Version.TryParse(value, out var parsed))
|
||||
{
|
||||
if (parsed.Build >= 0 && parsed.Revision >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}";
|
||||
}
|
||||
|
||||
if (parsed.Build >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}";
|
||||
}
|
||||
|
||||
return $"{parsed.Major}.{parsed.Minor}";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static void AddExtension(Dictionary<string, string> extensions, string key, string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
extensions[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
internal static class VmwareJobKinds
|
||||
{
|
||||
public const string Fetch = "source:vmware:fetch";
|
||||
public const string Parse = "source:vmware:parse";
|
||||
public const string Map = "source:vmware:map";
|
||||
}
|
||||
|
||||
internal sealed class VmwareFetchJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareFetchJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VmwareParseJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareParseJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VmwareMapJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareMapJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Vmware.Tests")]
|
||||
@@ -0,0 +1,23 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
||||
<_Parameter1>StellaOps.Concelier.Tests</_Parameter1>
|
||||
</AssemblyAttribute>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,17 @@
|
||||
# Source.Vndr.Vmware — Task Board
|
||||
|
||||
| ID | Task | Owner | Status | Depends On | Notes |
|
||||
|------|-----------------------------------------------|-------|--------|------------|-------|
|
||||
| VM1 | Advisory listing discovery + cursor | Conn | DONE | Common | **DONE** – fetch pipeline uses index JSON with sliding cursor + processed id tracking. |
|
||||
| VM2 | VMSA parser → DTO | QA | DONE | | **DONE** – JSON DTO deserialization wired with sanitization. |
|
||||
| VM3 | Canonical mapping (aliases/affected/refs) | Conn | DONE | Models | **DONE** – `VmwareMapper` emits aliases/affected/reference ordering and persists PSIRT flags via `PsirtFlagStore`. |
|
||||
| VM4 | Snapshot tests + resume | QA | DONE | Storage | **DONE** – integration test validates snapshot output and resume flow with cached state. |
|
||||
| VM5 | Observability | QA | DONE | | **DONE** – diagnostics meter exposes fetch/parse/map metrics and structured logs. |
|
||||
| VM6 | SourceState + hash dedupe | Conn | DONE | Storage | **DONE** – fetch cache stores sha/etag to skip unchanged advisories during resume. |
|
||||
| VM6a | Options & HttpClient configuration | Conn | DONE | Source.Common | **DONE** – `AddVmwareConnector` configures allowlisted HttpClient + options. |
|
||||
| VM7 | Dependency injection routine & scheduler registration | Conn | DONE | Core | **DONE** – `VmwareDependencyInjectionRoutine` registers fetch/parse/map jobs. |
|
||||
| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | DONE | Source.Common | **DONE** – connector implements fetch/parse/map persisting docs, DTOs, advisories. |
|
||||
| VM9 | Range primitives + provenance diagnostics refresh | Conn | DONE | Models, Storage.Mongo | Vendor primitives emitted (SemVer + vendor extensions), provenance tags/logging updated, snapshots refreshed. |
|
||||
|
||||
## Changelog
|
||||
- YYYY-MM-DD: Created.
|
||||
@@ -0,0 +1,454 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly IPsirtFlagStore _psirtFlagStore;
|
||||
private readonly VmwareOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly VmwareDiagnostics _diagnostics;
|
||||
private readonly ILogger<VmwareConnector> _logger;
|
||||
|
||||
public VmwareConnector(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IPsirtFlagStore psirtFlagStore,
|
||||
IOptions<VmwareOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
VmwareDiagnostics diagnostics,
|
||||
ILogger<VmwareConnector> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => VmwareConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var fetchCache = new Dictionary<string, VmwareFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase);
|
||||
var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var remainingCapacity = _options.MaxAdvisoriesPerFetch;
|
||||
|
||||
IReadOnlyList<VmwareIndexItem> indexItems;
|
||||
try
|
||||
{
|
||||
indexItems = await FetchIndexAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to retrieve VMware advisory index");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (indexItems.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var orderedItems = indexItems
|
||||
.Where(static item => !string.IsNullOrWhiteSpace(item.Id) && !string.IsNullOrWhiteSpace(item.DetailUrl))
|
||||
.OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue)
|
||||
.ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var baseline = cursor.LastModified ?? now - _options.InitialBackfill;
|
||||
var resumeStart = baseline - _options.ModifiedTolerance;
|
||||
ProvenanceDiagnostics.ReportResumeWindow(SourceName, resumeStart, _logger);
|
||||
var processedIds = new HashSet<string>(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase);
|
||||
var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue;
|
||||
var processedUpdated = false;
|
||||
|
||||
foreach (var item in orderedItems)
|
||||
{
|
||||
if (remainingCapacity <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var modified = (item.Modified ?? DateTimeOffset.MinValue).ToUniversalTime();
|
||||
if (modified < baseline - _options.ModifiedTolerance)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cursor.LastModified.HasValue && modified < cursor.LastModified.Value - _options.ModifiedTolerance)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (modified == cursor.LastModified && cursor.ProcessedIds.Contains(item.Id, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(item.DetailUrl, UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
_logger.LogWarning("VMware advisory {AdvisoryId} has invalid detail URL {Url}", item.Id, item.DetailUrl);
|
||||
continue;
|
||||
}
|
||||
|
||||
var cacheKey = detailUri.AbsoluteUri;
|
||||
touchedResources.Add(cacheKey);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["vmware.id"] = item.Id,
|
||||
["vmware.modified"] = modified.ToString("O"),
|
||||
};
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, detailUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
ETag = existing?.Etag,
|
||||
LastModified = existing?.LastModified,
|
||||
AcceptHeaders = new[] { "application/json" },
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
if (existing is not null)
|
||||
{
|
||||
fetchCache[cacheKey] = VmwareFetchCacheEntry.FromDocument(existing);
|
||||
pendingDocuments.Remove(existing.Id);
|
||||
pendingMappings.Remove(existing.Id);
|
||||
_logger.LogInformation("VMware advisory {AdvisoryId} returned 304 Not Modified", item.Id);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
remainingCapacity--;
|
||||
|
||||
if (modified > maxModified)
|
||||
{
|
||||
maxModified = modified;
|
||||
processedIds.Clear();
|
||||
processedUpdated = true;
|
||||
}
|
||||
|
||||
if (modified == maxModified)
|
||||
{
|
||||
processedIds.Add(item.Id);
|
||||
processedUpdated = true;
|
||||
}
|
||||
|
||||
var cacheEntry = VmwareFetchCacheEntry.FromDocument(result.Document);
|
||||
|
||||
if (existing is not null
|
||||
&& string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)
|
||||
&& cursor.TryGetFetchCache(cacheKey, out var cachedEntry)
|
||||
&& cachedEntry.Matches(result.Document))
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
fetchCache[cacheKey] = cacheEntry;
|
||||
pendingDocuments.Remove(result.Document.Id);
|
||||
pendingMappings.Remove(result.Document.Id);
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("VMware advisory {AdvisoryId} unchanged; skipping reprocessing", item.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.FetchItem();
|
||||
fetchCache[cacheKey] = cacheEntry;
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
_logger.LogInformation(
|
||||
"VMware advisory {AdvisoryId} fetched (documentId={DocumentId}, sha256={Sha})",
|
||||
item.Id,
|
||||
result.Document.Id,
|
||||
result.Document.Sha256);
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fetchCache.Count > 0 && touchedResources.Count > 0)
|
||||
{
|
||||
var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray();
|
||||
foreach (var key in stale)
|
||||
{
|
||||
fetchCache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithFetchCache(fetchCache);
|
||||
|
||||
if (processedUpdated)
|
||||
{
|
||||
updatedCursor = updatedCursor.WithLastModified(maxModified, processedIds);
|
||||
}
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remaining = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] bytes;
|
||||
try
|
||||
{
|
||||
bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed downloading VMware document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
VmwareDetailDto? detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<VmwareDetailDto>(bytes, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId))
|
||||
{
|
||||
_logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
var sanitized = JsonSerializer.Serialize(detail, SerializerOptions);
|
||||
var payload = MongoDB.Bson.BsonDocument.Parse(sanitized);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "vmware.v1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remaining.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remaining)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var json = dto.Payload.ToJson(new JsonWriterSettings
|
||||
{
|
||||
OutputMode = JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
VmwareDetailDto? detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<VmwareDetailDto>(json, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize VMware DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId))
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (advisory, flag) = VmwareMapper.Map(detail, document, dto);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
_diagnostics.MapAffectedCount(advisory.AffectedPackages.Length);
|
||||
_logger.LogInformation(
|
||||
"VMware advisory {AdvisoryId} mapped with {AffectedCount} affected packages",
|
||||
detail.AdvisoryId,
|
||||
advisory.AffectedPackages.Length);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<VmwareIndexItem>> FetchIndexAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var items = await JsonSerializer.DeserializeAsync<IReadOnlyList<VmwareIndexItem>>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
return items ?? Array.Empty<VmwareIndexItem>();
|
||||
}
|
||||
|
||||
private async Task<VmwareCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? VmwareCursor.Empty : VmwareCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(VmwareCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public string Name => SourceName;
|
||||
|
||||
public static string SourceName => "vmware";
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<VmwareConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:vmware";
|
||||
private const string FetchCron = "10,40 * * * *";
|
||||
private const string ParseCron = "15,45 * * * *";
|
||||
private const string MapCron = "20,50 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(15);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddVmwareConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<VmwareFetchJob>(
|
||||
VmwareJobKinds.Fetch,
|
||||
cronExpression: FetchCron,
|
||||
timeout: FetchTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<VmwareParseJob>(
|
||||
VmwareJobKinds.Parse,
|
||||
cronExpression: ParseCron,
|
||||
timeout: ParseTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<VmwareMapJob>(
|
||||
VmwareJobKinds.Map,
|
||||
cronExpression: MapCron,
|
||||
timeout: MapTimeout,
|
||||
leaseDuration: LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
/// <summary>
|
||||
/// VMware connector metrics (fetch, parse, map).
|
||||
/// </summary>
|
||||
public sealed class VmwareDiagnostics : IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Vmware";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchItems;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _mapAffectedCount;
|
||||
|
||||
public VmwareDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchItems = _meter.CreateCounter<long>(
|
||||
name: "vmware.fetch.items",
|
||||
unit: "documents",
|
||||
description: "Number of VMware advisory documents fetched.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "vmware.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VMware fetch failures.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "vmware.fetch.unchanged",
|
||||
unit: "documents",
|
||||
description: "Number of VMware advisories skipped due to unchanged content.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "vmware.parse.fail",
|
||||
unit: "documents",
|
||||
description: "Number of VMware advisory documents that failed to parse.");
|
||||
_mapAffectedCount = _meter.CreateHistogram<long>(
|
||||
name: "vmware.map.affected_count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected-package counts emitted per VMware advisory.");
|
||||
}
|
||||
|
||||
public void FetchItem() => _fetchItems.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void MapAffectedCount(int count)
|
||||
{
|
||||
if (count < 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_mapAffectedCount.Record(count);
|
||||
}
|
||||
|
||||
public Meter Meter => _meter;
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Vmware;
|
||||
|
||||
public static class VmwareServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddVmwareConnector(this IServiceCollection services, Action<VmwareOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<VmwareOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(VmwareOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<VmwareOptions>>().Value;
|
||||
clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority));
|
||||
clientOptions.Timeout = options.HttpTimeout;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.VMware/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.IndexUri.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.TryAddSingleton<VmwareDiagnostics>();
|
||||
services.AddTransient<VmwareConnector>();
|
||||
services.AddTransient<VmwareFetchJob>();
|
||||
services.AddTransient<VmwareParseJob>();
|
||||
services.AddTransient<VmwareMapJob>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user