Initial commit (history squashed)
This commit is contained in:
Vladimir Moushkov
29
src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md
Normal file
29
src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMware products; maps affected versions/builds and emits psirt_flags.
|
||||
## Scope
|
||||
- Discover/fetch VMSA index and detail pages via Broadcom portal; window by advisory ID/date; follow updates/revisions.
|
||||
- Validate HTML or JSON; extract CVEs, affected product versions/builds, workarounds, fixed versions; normalize product naming.
|
||||
- Persist raw docs with sha256; manage source_state; idempotent mapping.
|
||||
## Participants
|
||||
- Source.Common (HTTP, cookies/session handling if needed, validators).
|
||||
- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state).
|
||||
- Models (canonical).
|
||||
- Core/WebService (jobs: source:vmware:fetch|parse|map).
|
||||
- Merge engine (later) to prefer PSIRT ranges for VMware products.
|
||||
## Interfaces & contracts
|
||||
- Aliases: VMSA-YYYY-NNNN plus CVEs.
|
||||
- Affected entries include Vendor=VMware, Product plus component; Versions carry fixed/fixedBy; tags may include build numbers or ESXi/VC levels.
|
||||
- References: advisory URL, KBs, workaround pages; typed; deduped.
|
||||
- Provenance: method=parser; value=VMSA id.
|
||||
## In/Out of scope
|
||||
In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references.
|
||||
Out: customer portal authentication flows beyond public advisories; downloading patches.
|
||||
## Observability & security expectations
|
||||
- Metrics: vmware.fetch.items, vmware.parse.fail, vmware.map.affected_count.
|
||||
- Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware.Configuration;
|
||||
|
||||
public sealed class VmwareOptions
|
||||
{
|
||||
public const string HttpClientName = "source.vmware";
|
||||
|
||||
public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute);
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(2);
|
||||
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 50;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
[MemberNotNull(nameof(IndexUri))]
|
||||
public void Validate()
|
||||
{
|
||||
if (IndexUri is null || !IndexUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("VMware index URI must be absolute.");
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Initial backfill must be positive.");
|
||||
}
|
||||
|
||||
if (ModifiedTolerance < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Modified tolerance cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("Max advisories per fetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("Request delay cannot be negative.");
|
||||
}
|
||||
|
||||
if (HttpTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("HTTP timeout must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareCursor(
|
||||
DateTimeOffset? LastModified,
|
||||
IReadOnlyCollection<string> ProcessedIds,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
|
||||
|
||||
public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastModified.HasValue)
|
||||
{
|
||||
document["lastModified"] = LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (ProcessedIds.Count > 0)
|
||||
{
|
||||
document["processedIds"] = new BsonArray(ProcessedIds);
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static VmwareCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastModified = document.TryGetValue("lastModified", out var value)
|
||||
? ParseDate(value)
|
||||
: null;
|
||||
|
||||
var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray
|
||||
? idsArray.OfType<BsonValue>().Where(static x => x.BsonType == BsonType.String).Select(static x => x.AsString).ToArray()
|
||||
: EmptyStringList;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings);
|
||||
}
|
||||
|
||||
public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable<string> processedIds)
|
||||
=> this with
|
||||
{
|
||||
LastModified = timestamp.ToUniversalTime(),
|
||||
ProcessedIds = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? EmptyStringList,
|
||||
};
|
||||
|
||||
public VmwareCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public VmwareCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public VmwareCursor AddProcessedId(string id)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(ProcessedIds, StringComparer.OrdinalIgnoreCase) { id.Trim() };
|
||||
return this with { ProcessedIds = set.ToArray() };
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareDetailDto
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("cves")]
|
||||
public IReadOnlyList<string>? CveIds { get; init; }
|
||||
|
||||
[JsonPropertyName("affected")]
|
||||
public IReadOnlyList<VmwareAffectedProductDto>? Affected { get; init; }
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<VmwareReferenceDto>? References { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record VmwareAffectedProductDto
|
||||
{
|
||||
[JsonPropertyName("product")]
|
||||
public string Product { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; init; }
|
||||
|
||||
[JsonPropertyName("fixedVersion")]
|
||||
public string? FixedVersion { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record VmwareReferenceDto
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string? Type { get; init; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string Url { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal;
|
||||
|
||||
internal sealed record VmwareIndexItem
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string DetailUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal;
|
||||
|
||||
internal static class VmwareMapper
|
||||
{
|
||||
public static Advisory Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt);
|
||||
var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, dtoRecord.ValidatedAt);
|
||||
var affectedPackages = BuildAffectedPackages(dto, dtoRecord.ValidatedAt);
|
||||
|
||||
return new Advisory(
|
||||
dto.AdvisoryId,
|
||||
dto.Title,
|
||||
dto.Summary,
|
||||
language: "en",
|
||||
dto.Published?.ToUniversalTime(),
|
||||
dto.Modified?.ToUniversalTime(),
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
references,
|
||||
affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mappingProvenance });
|
||||
}
|
||||
|
||||
private static IEnumerable<string> BuildAliases(VmwareDetailDto dto)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId };
|
||||
if (dto.CveIds is not null)
|
||||
{
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
set.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return set;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(VmwareDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.References is null || dto.References.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>(dto.References.Count);
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var kind = NormalizeReferenceKind(reference.Type);
|
||||
var provenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "reference", reference.Url, recordedAt);
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore invalid urls
|
||||
}
|
||||
}
|
||||
|
||||
references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
|
||||
return references.Count == 0 ? Array.Empty<AdvisoryReference>() : references;
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceKind(string? type)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return type.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"advisory" => "advisory",
|
||||
"kb" or "kb_article" => "kb",
|
||||
"patch" => "patch",
|
||||
"workaround" => "workaround",
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(VmwareDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Affected is null || dto.Affected.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Affected.Count);
|
||||
foreach (var product in dto.Affected)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.Product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "affected", product.Product, recordedAt),
|
||||
};
|
||||
|
||||
var ranges = new List<AffectedVersionRange>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion))
|
||||
{
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: product.Version,
|
||||
fixedVersion: product.FixedVersion,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.Version,
|
||||
provenance: new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt)));
|
||||
}
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
product.Product,
|
||||
platform: null,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
}
|
||||
46
src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs
Normal file
46
src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware;
|
||||
|
||||
internal static class VmwareJobKinds
|
||||
{
|
||||
public const string Fetch = "source:vmware:fetch";
|
||||
public const string Parse = "source:vmware:parse";
|
||||
public const string Map = "source:vmware:map";
|
||||
}
|
||||
|
||||
internal sealed class VmwareFetchJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareFetchJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VmwareParseJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareParseJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class VmwareMapJob : IJob
|
||||
{
|
||||
private readonly VmwareConnector _connector;
|
||||
|
||||
public VmwareMapJob(VmwareConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
||||
<_Parameter1>StellaOps.Feedser.Tests</_Parameter1>
|
||||
</AssemblyAttribute>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
16
src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md
Normal file
16
src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md
Normal file
@@ -0,0 +1,16 @@
|
||||
# Source.Vndr.Vmware — Task Board
|
||||
|
||||
| ID | Task | Owner | Status | Depends On | Notes |
|
||||
|------|-----------------------------------------------|-------|--------|------------|-------|
|
||||
| VM1 | Advisory listing discovery + cursor | Conn | TODO | Common | Track revisions; respect VMware PSIRT cadence. |
|
||||
| VM2 | VMSA parser → DTO | QA | TODO | | Extract product/version/CVE/severity; capture fixed build numbers. |
|
||||
| VM3 | Canonical mapping (aliases/affected/refs) | Conn | TODO | Models | Deterministic ordering; set vendor="VMware" and advisory_id_text=VMSA. |
|
||||
| VM4 | Snapshot tests + resume | QA | TODO | Storage | |
|
||||
| VM5 | Observability | QA | TODO | | Metrics counters. |
|
||||
| VM6 | SourceState + hash dedupe | Conn | TODO | Storage | Skip unchanged advisories; ensure idempotent reruns. |
|
||||
| VM6a | Options & HttpClient configuration | Conn | TODO | Source.Common | Introduce `VmwareOptions` with base portal URLs and allowlisted HttpClient setup. |
|
||||
| VM7 | Dependency injection routine & scheduler registration | Conn | TODO | Core | Wire HttpClient/options and register fetch/parse/map jobs consistent with other connectors. |
|
||||
| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | TODO | Source.Common | Implement fetch/parse/map scaffolding persisting source_state, documents, and canonical advisories. |
|
||||
|
||||
## Changelog
|
||||
- YYYY-MM-DD: Created.
|
||||
374
src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs
Normal file
374
src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs
Normal file
@@ -0,0 +1,374 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Source.Vndr.Vmware.Configuration;
|
||||
using StellaOps.Feedser.Source.Vndr.Vmware.Internal;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly VmwareOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<VmwareConnector> _logger;
|
||||
|
||||
public VmwareConnector(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<VmwareOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<VmwareConnector> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => VmwareConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var remainingCapacity = _options.MaxAdvisoriesPerFetch;
|
||||
|
||||
IReadOnlyList<VmwareIndexItem> indexItems;
|
||||
try
|
||||
{
|
||||
indexItems = await FetchIndexAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to retrieve VMware advisory index");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (indexItems.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var orderedItems = indexItems
|
||||
.Where(static item => !string.IsNullOrWhiteSpace(item.Id) && !string.IsNullOrWhiteSpace(item.DetailUrl))
|
||||
.OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue)
|
||||
.ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var baseline = cursor.LastModified ?? now - _options.InitialBackfill;
|
||||
var processedIds = new HashSet<string>(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase);
|
||||
var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue;
|
||||
var processedUpdated = false;
|
||||
|
||||
foreach (var item in orderedItems)
|
||||
{
|
||||
if (remainingCapacity <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var modified = (item.Modified ?? DateTimeOffset.MinValue).ToUniversalTime();
|
||||
if (modified < baseline - _options.ModifiedTolerance)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cursor.LastModified.HasValue && modified < cursor.LastModified.Value - _options.ModifiedTolerance)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (modified == cursor.LastModified && cursor.ProcessedIds.Contains(item.Id, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["vmware.id"] = item.Id,
|
||||
["vmware.modified"] = modified.ToString("O"),
|
||||
};
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, new Uri(item.DetailUrl))
|
||||
{
|
||||
Metadata = metadata,
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
remainingCapacity--;
|
||||
|
||||
if (modified > maxModified)
|
||||
{
|
||||
maxModified = modified;
|
||||
processedIds.Clear();
|
||||
processedUpdated = true;
|
||||
}
|
||||
|
||||
if (modified == maxModified)
|
||||
{
|
||||
processedIds.Add(item.Id);
|
||||
processedUpdated = true;
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(cursor.PendingMappings);
|
||||
|
||||
if (processedUpdated)
|
||||
{
|
||||
updatedCursor = updatedCursor.WithLastModified(maxModified, processedIds);
|
||||
}
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remaining = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] bytes;
|
||||
try
|
||||
{
|
||||
bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed downloading VMware document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
VmwareDetailDto? detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<VmwareDetailDto>(bytes, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId))
|
||||
{
|
||||
_logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remaining.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var sanitized = JsonSerializer.Serialize(detail, SerializerOptions);
|
||||
var payload = MongoDB.Bson.BsonDocument.Parse(sanitized);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "vmware.v1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remaining.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remaining)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var json = dto.Payload.ToJson(new JsonWriterSettings
|
||||
{
|
||||
OutputMode = JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
VmwareDetailDto? detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<VmwareDetailDto>(json, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize VMware DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId))
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisory = VmwareMapper.Map(detail, document, dto);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<VmwareIndexItem>> FetchIndexAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var items = await JsonSerializer.DeserializeAsync<IReadOnlyList<VmwareIndexItem>>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
return items ?? Array.Empty<VmwareIndexItem>();
|
||||
}
|
||||
|
||||
private async Task<VmwareCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? VmwareCursor.Empty : VmwareCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(VmwareCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public string Name => SourceName;
|
||||
|
||||
public static string SourceName => "vmware";
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<VmwareConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Source.Vndr.Vmware.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware;
|
||||
|
||||
public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "feedser:sources:vmware";
|
||||
private const string FetchCron = "10,40 * * * *";
|
||||
private const string ParseCron = "15,45 * * * *";
|
||||
private const string MapCron = "20,50 * * * *";
|
||||
|
||||
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10);
|
||||
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(15);
|
||||
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddVmwareConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
var scheduler = new JobSchedulerBuilder(services);
|
||||
scheduler
|
||||
.AddJob<VmwareFetchJob>(
|
||||
VmwareJobKinds.Fetch,
|
||||
cronExpression: FetchCron,
|
||||
timeout: FetchTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<VmwareParseJob>(
|
||||
VmwareJobKinds.Parse,
|
||||
cronExpression: ParseCron,
|
||||
timeout: ParseTimeout,
|
||||
leaseDuration: LeaseDuration)
|
||||
.AddJob<VmwareMapJob>(
|
||||
VmwareJobKinds.Map,
|
||||
cronExpression: MapCron,
|
||||
timeout: MapTimeout,
|
||||
leaseDuration: LeaseDuration);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Vndr.Vmware.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Vndr.Vmware;
|
||||
|
||||
public static class VmwareServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddVmwareConnector(this IServiceCollection services, Action<VmwareOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<VmwareOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(VmwareOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<VmwareOptions>>().Value;
|
||||
clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority));
|
||||
clientOptions.Timeout = options.HttpTimeout;
|
||||
clientOptions.UserAgent = "StellaOps.Feedser.VMware/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.IndexUri.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.AddTransient<VmwareConnector>();
|
||||
services.AddTransient<VmwareFetchJob>();
|
||||
services.AddTransient<VmwareParseJob>();
|
||||
services.AddTransient<VmwareMapJob>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user