Rename Concelier Source modules to Connector
This commit is contained in:
master
757
src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs
Normal file
757
src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs
Normal file
@@ -0,0 +1,757 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Json.Schema;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Json;
|
||||
using StellaOps.Concelier.Connector.Common.Packages;
|
||||
using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe;
|
||||
|
||||
public sealed class AdobeConnector : IFeedConnector
|
||||
{
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly IPsirtFlagStore _psirtFlagStore;
|
||||
private readonly IJsonSchemaValidator _schemaValidator;
|
||||
private readonly AdobeOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly AdobeDiagnostics _diagnostics;
|
||||
private readonly ILogger<AdobeConnector> _logger;
|
||||
|
||||
private static readonly JsonSchema Schema = AdobeSchemaProvider.Schema;
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
public AdobeConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IPsirtFlagStore psirtFlagStore,
|
||||
IJsonSchemaValidator schemaValidator,
|
||||
IOptions<AdobeOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
AdobeDiagnostics diagnostics,
|
||||
ILogger<AdobeConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore));
|
||||
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(AdobeProductEntry product, AdvisoryProvenance provenance)
|
||||
{
|
||||
if (!TryResolveAvailabilityStatus(product.Availability, out var status))
|
||||
{
|
||||
return Array.Empty<AffectedPackageStatus>();
|
||||
}
|
||||
|
||||
return new[] { new AffectedPackageStatus(status, provenance) };
|
||||
}
|
||||
|
||||
private static bool TryResolveAvailabilityStatus(string? availability, out string status)
|
||||
{
|
||||
status = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(availability))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = availability.Trim();
|
||||
|
||||
if (AffectedPackageStatusCatalog.TryNormalize(trimmed, out var normalized))
|
||||
{
|
||||
status = normalized;
|
||||
return true;
|
||||
}
|
||||
|
||||
var token = SanitizeStatusToken(trimmed);
|
||||
if (token.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (AvailabilityStatusMap.TryGetValue(token, out var mapped))
|
||||
{
|
||||
status = mapped;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string SanitizeStatusToken(string value)
|
||||
{
|
||||
var buffer = new char[value.Length];
|
||||
var index = 0;
|
||||
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
buffer[index++] = char.ToLowerInvariant(ch);
|
||||
}
|
||||
}
|
||||
|
||||
return index == 0 ? string.Empty : new string(buffer, 0, index);
|
||||
}
|
||||
|
||||
private static readonly Dictionary<string, string> AvailabilityStatusMap = new(StringComparer.Ordinal)
|
||||
{
|
||||
["available"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["availabletoday"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["availablenow"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["updateavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["patchavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["fixavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["mitigationavailable"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["workaroundavailable"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["mitigationprovided"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["workaroundprovided"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["planned"] = AffectedPackageStatusCatalog.Pending,
|
||||
["updateplanned"] = AffectedPackageStatusCatalog.Pending,
|
||||
["plannedupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["scheduled"] = AffectedPackageStatusCatalog.Pending,
|
||||
["scheduledupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingavailability"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingfix"] = AffectedPackageStatusCatalog.Pending,
|
||||
["notavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["unavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["notcurrentlyavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["notapplicable"] = AffectedPackageStatusCatalog.NotApplicable,
|
||||
};
|
||||
|
||||
private AffectedVersionRange? BuildVersionRange(AdobeProductEntry product, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.AffectedVersion) && string.IsNullOrWhiteSpace(product.UpdatedVersion))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var key = string.IsNullOrWhiteSpace(product.Platform)
|
||||
? product.Product
|
||||
: $"{product.Product}:{product.Platform}";
|
||||
|
||||
var provenance = new AdvisoryProvenance(SourceName, "range", key, recordedAt);
|
||||
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
AddExtension(extensions, "adobe.track", product.Track);
|
||||
AddExtension(extensions, "adobe.platform", product.Platform);
|
||||
AddExtension(extensions, "adobe.affected.raw", product.AffectedVersion);
|
||||
AddExtension(extensions, "adobe.updated.raw", product.UpdatedVersion);
|
||||
AddExtension(extensions, "adobe.priority", product.Priority);
|
||||
AddExtension(extensions, "adobe.availability", product.Availability);
|
||||
|
||||
var lastAffected = ExtractVersionNumber(product.AffectedVersion);
|
||||
var fixedVersion = ExtractVersionNumber(product.UpdatedVersion);
|
||||
|
||||
var primitives = BuildRangePrimitives(lastAffected, fixedVersion, extensions);
|
||||
|
||||
return new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: null,
|
||||
fixedVersion: fixedVersion,
|
||||
lastAffectedVersion: lastAffected,
|
||||
rangeExpression: product.AffectedVersion ?? product.UpdatedVersion,
|
||||
provenance: provenance,
|
||||
primitives: primitives);
|
||||
}
|
||||
|
||||
private static RangePrimitives? BuildRangePrimitives(string? lastAffected, string? fixedVersion, Dictionary<string, string> extensions)
|
||||
{
|
||||
var semVer = BuildSemVerPrimitive(lastAffected, fixedVersion);
|
||||
|
||||
if (semVer is null && extensions.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions);
|
||||
}
|
||||
|
||||
private static SemVerPrimitive? BuildSemVerPrimitive(string? lastAffected, string? fixedVersion)
|
||||
{
|
||||
var fixedNormalized = NormalizeSemVer(fixedVersion);
|
||||
var lastNormalized = NormalizeSemVer(lastAffected);
|
||||
|
||||
if (fixedNormalized is null && lastNormalized is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new SemVerPrimitive(
|
||||
Introduced: null,
|
||||
IntroducedInclusive: true,
|
||||
Fixed: fixedNormalized,
|
||||
FixedInclusive: false,
|
||||
LastAffected: lastNormalized,
|
||||
LastAffectedInclusive: true,
|
||||
ConstraintExpression: null);
|
||||
}
|
||||
|
||||
private static string? NormalizeSemVer(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (PackageCoordinateHelper.TryParseSemVer(trimmed, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (Version.TryParse(trimmed, out var parsed))
|
||||
{
|
||||
if (parsed.Build >= 0 && parsed.Revision >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}";
|
||||
}
|
||||
|
||||
if (parsed.Build >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}";
|
||||
}
|
||||
|
||||
return $"{parsed.Major}.{parsed.Minor}";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ExtractVersionNumber(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = VersionPattern.Match(text);
|
||||
return match.Success ? match.Value : null;
|
||||
}
|
||||
|
||||
private static void AddExtension(IDictionary<string, string> extensions, string key, string? value)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
extensions[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly Regex VersionPattern = new("\\d+(?:\\.\\d+)+", RegexOptions.Compiled);
|
||||
|
||||
public string SourceName => VndrAdobeConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var backfillStart = now - _options.InitialBackfill;
|
||||
var windowStart = cursor.LastPublished.HasValue
|
||||
? cursor.LastPublished.Value - _options.WindowOverlap
|
||||
: backfillStart;
|
||||
if (windowStart < backfillStart)
|
||||
{
|
||||
windowStart = backfillStart;
|
||||
}
|
||||
|
||||
var maxPublished = cursor.LastPublished;
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var fetchCache = cursor.FetchCache is null
|
||||
? new Dictionary<string, AdobeFetchCacheEntry>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, AdobeFetchCacheEntry>(cursor.FetchCache, StringComparer.Ordinal);
|
||||
var touchedResources = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var collectedEntries = new Dictionary<string, AdobeIndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var indexUri in EnumerateIndexUris())
|
||||
{
|
||||
_diagnostics.FetchAttempt();
|
||||
string? html = null;
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(AdobeOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
html = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to download Adobe index page {Uri}", indexUri);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(html))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
IReadOnlyCollection<AdobeIndexEntry> entries;
|
||||
try
|
||||
{
|
||||
entries = AdobeIndexParser.Parse(html, indexUri);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to parse Adobe index page {Uri}", indexUri);
|
||||
_diagnostics.FetchFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (entry.PublishedUtc < windowStart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!collectedEntries.TryGetValue(entry.AdvisoryId, out var existing) || entry.PublishedUtc > existing.PublishedUtc)
|
||||
{
|
||||
collectedEntries[entry.AdvisoryId] = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var entry in collectedEntries.Values.OrderBy(static e => e.PublishedUtc))
|
||||
{
|
||||
if (!maxPublished.HasValue || entry.PublishedUtc > maxPublished)
|
||||
{
|
||||
maxPublished = entry.PublishedUtc;
|
||||
}
|
||||
|
||||
var cacheKey = entry.DetailUri.ToString();
|
||||
touchedResources.Add(cacheKey);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["advisoryId"] = entry.AdvisoryId,
|
||||
["published"] = entry.PublishedUtc.ToString("O"),
|
||||
["title"] = entry.Title ?? string.Empty,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(AdobeOptions.HttpClientName, SourceName, entry.DetailUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" },
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cursor.TryGetFetchCache(cacheKey, out var cached)
|
||||
&& string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256);
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.FetchDocument();
|
||||
fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256);
|
||||
|
||||
if (!pendingDocuments.Contains(result.Document.Id))
|
||||
{
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to fetch Adobe advisory {AdvisoryId} ({Uri})", entry.AdvisoryId, entry.DetailUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in fetchCache.Keys.ToList())
|
||||
{
|
||||
if (!touchedResources.Contains(key))
|
||||
{
|
||||
fetchCache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithLastPublished(maxPublished)
|
||||
.WithFetchCache(fetchCache);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Adobe document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeDocumentMetadata metadata;
|
||||
try
|
||||
{
|
||||
metadata = AdobeDocumentMetadata.FromDocument(document);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe metadata parse failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeBulletinDto dto;
|
||||
try
|
||||
{
|
||||
var bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
var html = Encoding.UTF8.GetString(bytes);
|
||||
dto = AdobeDetailParser.Parse(html, metadata);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(dto, SerializerOptions);
|
||||
using var jsonDocument = JsonDocument.Parse(json);
|
||||
_schemaValidator.Validate(jsonDocument, Schema, metadata.AdvisoryId);
|
||||
|
||||
var payload = MongoDB.Bson.BsonDocument.Parse(json);
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
"adobe.bulletin.v1",
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dtoRecord is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeBulletinDto? dto;
|
||||
try
|
||||
{
|
||||
var json = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
dto = JsonSerializer.Deserialize<AdobeBulletinDto>(json, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe DTO deserialization failed for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_logger.LogWarning("Adobe DTO payload deserialized as null for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisory = BuildAdvisory(dto, now);
|
||||
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
|
||||
{
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var flag = new PsirtFlagRecord(
|
||||
advisory.AdvisoryKey,
|
||||
"Adobe",
|
||||
SourceName,
|
||||
dto.AdvisoryId,
|
||||
now);
|
||||
|
||||
await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("Skipping PSIRT flag for advisory with missing key (document {DocumentId})", documentId);
|
||||
}
|
||||
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private IEnumerable<Uri> EnumerateIndexUris()
|
||||
{
|
||||
yield return _options.IndexUri;
|
||||
foreach (var uri in _options.AdditionalIndexUris)
|
||||
{
|
||||
yield return uri;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<AdobeCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return AdobeCursor.FromBsonDocument(record?.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(AdobeCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var updatedAt = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), updatedAt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Advisory BuildAdvisory(AdobeBulletinDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(SourceName, "parser", dto.AdvisoryId, recordedAt);
|
||||
|
||||
var aliasSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
dto.AdvisoryId,
|
||||
};
|
||||
foreach (var cve in dto.Cves)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliasSet.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
var comparer = StringComparer.OrdinalIgnoreCase;
|
||||
var references = new List<(AdvisoryReference Reference, int Priority)>
|
||||
{
|
||||
(new AdvisoryReference(dto.DetailUrl, "advisory", "adobe-psirt", dto.Summary, provenance), 0),
|
||||
};
|
||||
|
||||
foreach (var cve in dto.Cves)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var url = $"https://www.cve.org/CVERecord?id={cve}";
|
||||
references.Add((new AdvisoryReference(url, "advisory", cve, null, provenance), 1));
|
||||
}
|
||||
|
||||
var orderedReferences = references
|
||||
.GroupBy(tuple => tuple.Reference.Url, comparer)
|
||||
.Select(group => group
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.First())
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.Select(t => t.Reference)
|
||||
.ToArray();
|
||||
|
||||
var affected = dto.Products
|
||||
.Select(product => BuildPackage(product, recordedAt))
|
||||
.ToArray();
|
||||
|
||||
var aliases = aliasSet
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Select(static alias => alias.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new Advisory(
|
||||
dto.AdvisoryId,
|
||||
dto.Title,
|
||||
dto.Summary,
|
||||
language: "en",
|
||||
published: dto.Published,
|
||||
modified: null,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
orderedReferences,
|
||||
affected,
|
||||
Array.Empty<CvssMetric>(),
|
||||
new[] { provenance });
|
||||
}
|
||||
|
||||
private AffectedPackage BuildPackage(AdobeProductEntry product, DateTimeOffset recordedAt)
|
||||
{
|
||||
var identifier = string.IsNullOrWhiteSpace(product.Product)
|
||||
? "Adobe Product"
|
||||
: product.Product.Trim();
|
||||
|
||||
var platform = string.IsNullOrWhiteSpace(product.Platform) ? null : product.Platform;
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
SourceName,
|
||||
"affected",
|
||||
string.IsNullOrWhiteSpace(platform) ? identifier : $"{identifier}:{platform}",
|
||||
recordedAt);
|
||||
|
||||
var range = BuildVersionRange(product, recordedAt);
|
||||
var ranges = range is null ? Array.Empty<AffectedVersionRange>() : new[] { range };
|
||||
var normalizedVersions = BuildNormalizedVersions(product, ranges);
|
||||
var statuses = BuildStatuses(product, provenance);
|
||||
|
||||
return new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform,
|
||||
ranges,
|
||||
statuses,
|
||||
new[] { provenance },
|
||||
normalizedVersions);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
AdobeProductEntry product,
|
||||
IReadOnlyList<AffectedVersionRange> ranges)
|
||||
{
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var segments = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Product))
|
||||
{
|
||||
segments.Add(product.Product.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Platform))
|
||||
{
|
||||
segments.Add(product.Platform.Trim());
|
||||
}
|
||||
|
||||
var note = segments.Count == 0 ? null : $"adobe:{string.Join(':', segments)}";
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(ranges.Count);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var rule = range.ToNormalizedVersionRule(note);
|
||||
if (rule is not null)
|
||||
{
|
||||
rules.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user