using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Json.Schema; using MongoDB.Bson; using StellaOps.Concelier.Connector.Common; using StellaOps.Concelier.Connector.Common.Fetch; using StellaOps.Concelier.Connector.Common.Json; using StellaOps.Concelier.Connector.Common.Packages; using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; using StellaOps.Concelier.Connector.Vndr.Adobe.Internal; using StellaOps.Concelier.Storage.Mongo; using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Concelier.Storage.Mongo.Documents; using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Concelier.Storage.Mongo.PsirtFlags; using StellaOps.Concelier.Models; using StellaOps.Plugin; namespace StellaOps.Concelier.Connector.Vndr.Adobe; public sealed class AdobeConnector : IFeedConnector { private readonly SourceFetchService _fetchService; private readonly RawDocumentStorage _rawDocumentStorage; private readonly IDocumentStore _documentStore; private readonly IDtoStore _dtoStore; private readonly IAdvisoryStore _advisoryStore; private readonly ISourceStateRepository _stateRepository; private readonly IPsirtFlagStore _psirtFlagStore; private readonly IJsonSchemaValidator _schemaValidator; private readonly AdobeOptions _options; private readonly TimeProvider _timeProvider; private readonly IHttpClientFactory _httpClientFactory; private readonly AdobeDiagnostics _diagnostics; private readonly ILogger _logger; private static readonly JsonSchema Schema = AdobeSchemaProvider.Schema; private static readonly JsonSerializerOptions SerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; public AdobeConnector( SourceFetchService fetchService, RawDocumentStorage rawDocumentStorage, IDocumentStore documentStore, IDtoStore dtoStore, IAdvisoryStore advisoryStore, ISourceStateRepository stateRepository, IPsirtFlagStore psirtFlagStore, IJsonSchemaValidator schemaValidator, IOptions options, TimeProvider? timeProvider, IHttpClientFactory httpClientFactory, AdobeDiagnostics diagnostics, ILogger logger) { _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _options.Validate(); _timeProvider = timeProvider ?? TimeProvider.System; _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } private static IReadOnlyList BuildStatuses(AdobeProductEntry product, AdvisoryProvenance provenance) { if (!TryResolveAvailabilityStatus(product.Availability, out var status)) { return Array.Empty(); } return new[] { new AffectedPackageStatus(status, provenance) }; } private static bool TryResolveAvailabilityStatus(string? availability, out string status) { status = string.Empty; if (string.IsNullOrWhiteSpace(availability)) { return false; } var trimmed = availability.Trim(); if (AffectedPackageStatusCatalog.TryNormalize(trimmed, out var normalized)) { status = normalized; return true; } var token = SanitizeStatusToken(trimmed); if (token.Length == 0) { return false; } if (AvailabilityStatusMap.TryGetValue(token, out var mapped)) { status = mapped; return true; } return false; } private static string SanitizeStatusToken(string value) { var buffer = new char[value.Length]; var index = 0; foreach (var ch in value) { if (char.IsLetterOrDigit(ch)) { buffer[index++] = char.ToLowerInvariant(ch); } } return index == 0 ? string.Empty : new string(buffer, 0, index); } private static readonly Dictionary AvailabilityStatusMap = new(StringComparer.Ordinal) { ["available"] = AffectedPackageStatusCatalog.Fixed, ["availabletoday"] = AffectedPackageStatusCatalog.Fixed, ["availablenow"] = AffectedPackageStatusCatalog.Fixed, ["updateavailable"] = AffectedPackageStatusCatalog.Fixed, ["patchavailable"] = AffectedPackageStatusCatalog.Fixed, ["fixavailable"] = AffectedPackageStatusCatalog.Fixed, ["mitigationavailable"] = AffectedPackageStatusCatalog.Mitigated, ["workaroundavailable"] = AffectedPackageStatusCatalog.Mitigated, ["mitigationprovided"] = AffectedPackageStatusCatalog.Mitigated, ["workaroundprovided"] = AffectedPackageStatusCatalog.Mitigated, ["planned"] = AffectedPackageStatusCatalog.Pending, ["updateplanned"] = AffectedPackageStatusCatalog.Pending, ["plannedupdate"] = AffectedPackageStatusCatalog.Pending, ["scheduled"] = AffectedPackageStatusCatalog.Pending, ["scheduledupdate"] = AffectedPackageStatusCatalog.Pending, ["pendingavailability"] = AffectedPackageStatusCatalog.Pending, ["pendingupdate"] = AffectedPackageStatusCatalog.Pending, ["pendingfix"] = AffectedPackageStatusCatalog.Pending, ["notavailable"] = AffectedPackageStatusCatalog.Unknown, ["unavailable"] = AffectedPackageStatusCatalog.Unknown, ["notcurrentlyavailable"] = AffectedPackageStatusCatalog.Unknown, ["notapplicable"] = AffectedPackageStatusCatalog.NotApplicable, }; private AffectedVersionRange? BuildVersionRange(AdobeProductEntry product, DateTimeOffset recordedAt) { if (string.IsNullOrWhiteSpace(product.AffectedVersion) && string.IsNullOrWhiteSpace(product.UpdatedVersion)) { return null; } var key = string.IsNullOrWhiteSpace(product.Platform) ? product.Product : $"{product.Product}:{product.Platform}"; var provenance = new AdvisoryProvenance(SourceName, "range", key, recordedAt); var extensions = new Dictionary(StringComparer.Ordinal); AddExtension(extensions, "adobe.track", product.Track); AddExtension(extensions, "adobe.platform", product.Platform); AddExtension(extensions, "adobe.affected.raw", product.AffectedVersion); AddExtension(extensions, "adobe.updated.raw", product.UpdatedVersion); AddExtension(extensions, "adobe.priority", product.Priority); AddExtension(extensions, "adobe.availability", product.Availability); var lastAffected = ExtractVersionNumber(product.AffectedVersion); var fixedVersion = ExtractVersionNumber(product.UpdatedVersion); var primitives = BuildRangePrimitives(lastAffected, fixedVersion, extensions); return new AffectedVersionRange( rangeKind: "vendor", introducedVersion: null, fixedVersion: fixedVersion, lastAffectedVersion: lastAffected, rangeExpression: product.AffectedVersion ?? product.UpdatedVersion, provenance: provenance, primitives: primitives); } private static RangePrimitives? BuildRangePrimitives(string? lastAffected, string? fixedVersion, Dictionary extensions) { var semVer = BuildSemVerPrimitive(lastAffected, fixedVersion); if (semVer is null && extensions.Count == 0) { return null; } return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); } private static SemVerPrimitive? BuildSemVerPrimitive(string? lastAffected, string? fixedVersion) { var fixedNormalized = NormalizeSemVer(fixedVersion); var lastNormalized = NormalizeSemVer(lastAffected); if (fixedNormalized is null && lastNormalized is null) { return null; } return new SemVerPrimitive( Introduced: null, IntroducedInclusive: true, Fixed: fixedNormalized, FixedInclusive: false, LastAffected: lastNormalized, LastAffectedInclusive: true, ConstraintExpression: null); } private static string? NormalizeSemVer(string? value) { if (string.IsNullOrWhiteSpace(value)) { return null; } var trimmed = value.Trim(); if (PackageCoordinateHelper.TryParseSemVer(trimmed, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) { return normalized; } if (Version.TryParse(trimmed, out var parsed)) { if (parsed.Build >= 0 && parsed.Revision >= 0) { return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; } if (parsed.Build >= 0) { return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; } return $"{parsed.Major}.{parsed.Minor}"; } return null; } private static string? ExtractVersionNumber(string? text) { if (string.IsNullOrWhiteSpace(text)) { return null; } var match = VersionPattern.Match(text); return match.Success ? match.Value : null; } private static void AddExtension(IDictionary extensions, string key, string? value) { if (!string.IsNullOrWhiteSpace(value)) { extensions[key] = value.Trim(); } } private static readonly Regex VersionPattern = new("\\d+(?:\\.\\d+)+", RegexOptions.Compiled); public string SourceName => VndrAdobeConnectorPlugin.SourceName; public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) { var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var now = _timeProvider.GetUtcNow(); var backfillStart = now - _options.InitialBackfill; var windowStart = cursor.LastPublished.HasValue ? cursor.LastPublished.Value - _options.WindowOverlap : backfillStart; if (windowStart < backfillStart) { windowStart = backfillStart; } var maxPublished = cursor.LastPublished; var pendingDocuments = cursor.PendingDocuments.ToList(); var pendingMappings = cursor.PendingMappings.ToList(); var fetchCache = cursor.FetchCache is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(cursor.FetchCache, StringComparer.Ordinal); var touchedResources = new HashSet(StringComparer.Ordinal); var collectedEntries = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var indexUri in EnumerateIndexUris()) { _diagnostics.FetchAttempt(); string? html = null; try { var client = _httpClientFactory.CreateClient(AdobeOptions.HttpClientName); using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); html = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); } catch (Exception ex) { _diagnostics.FetchFailure(); _logger.LogError(ex, "Failed to download Adobe index page {Uri}", indexUri); continue; } if (string.IsNullOrEmpty(html)) { continue; } IReadOnlyCollection entries; try { entries = AdobeIndexParser.Parse(html, indexUri); } catch (Exception ex) { _logger.LogError(ex, "Failed to parse Adobe index page {Uri}", indexUri); _diagnostics.FetchFailure(); continue; } foreach (var entry in entries) { if (entry.PublishedUtc < windowStart) { continue; } if (!collectedEntries.TryGetValue(entry.AdvisoryId, out var existing) || entry.PublishedUtc > existing.PublishedUtc) { collectedEntries[entry.AdvisoryId] = entry; } } } foreach (var entry in collectedEntries.Values.OrderBy(static e => e.PublishedUtc)) { if (!maxPublished.HasValue || entry.PublishedUtc > maxPublished) { maxPublished = entry.PublishedUtc; } var cacheKey = entry.DetailUri.ToString(); touchedResources.Add(cacheKey); var metadata = new Dictionary(StringComparer.Ordinal) { ["advisoryId"] = entry.AdvisoryId, ["published"] = entry.PublishedUtc.ToString("O"), ["title"] = entry.Title ?? string.Empty, }; try { var result = await _fetchService.FetchAsync( new SourceFetchRequest(AdobeOptions.HttpClientName, SourceName, entry.DetailUri) { Metadata = metadata, AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, }, cancellationToken).ConfigureAwait(false); if (!result.IsSuccess || result.Document is null) { continue; } if (cursor.TryGetFetchCache(cacheKey, out var cached) && string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)) { _diagnostics.FetchUnchanged(); fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); continue; } _diagnostics.FetchDocument(); fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); if (!pendingDocuments.Contains(result.Document.Id)) { pendingDocuments.Add(result.Document.Id); } } catch (Exception ex) { _diagnostics.FetchFailure(); _logger.LogError(ex, "Failed to fetch Adobe advisory {AdvisoryId} ({Uri})", entry.AdvisoryId, entry.DetailUri); await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); throw; } } foreach (var key in fetchCache.Keys.ToList()) { if (!touchedResources.Contains(key)) { fetchCache.Remove(key); } } var updatedCursor = cursor .WithPendingDocuments(pendingDocuments) .WithPendingMappings(pendingMappings) .WithLastPublished(maxPublished) .WithFetchCache(fetchCache); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) { var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); if (cursor.PendingDocuments.Count == 0) { return; } var pendingDocuments = cursor.PendingDocuments.ToList(); var pendingMappings = cursor.PendingMappings.ToList(); foreach (var documentId in cursor.PendingDocuments) { var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); if (document is null) { pendingDocuments.Remove(documentId); pendingMappings.Remove(documentId); continue; } if (!document.GridFsId.HasValue) { _logger.LogWarning("Adobe document {DocumentId} missing GridFS payload", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); pendingDocuments.Remove(documentId); pendingMappings.Remove(documentId); continue; } AdobeDocumentMetadata metadata; try { metadata = AdobeDocumentMetadata.FromDocument(document); } catch (Exception ex) { _logger.LogError(ex, "Adobe metadata parse failed for document {DocumentId}", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); pendingDocuments.Remove(documentId); pendingMappings.Remove(documentId); continue; } AdobeBulletinDto dto; try { var bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); var html = Encoding.UTF8.GetString(bytes); dto = AdobeDetailParser.Parse(html, metadata); } catch (Exception ex) { _logger.LogError(ex, "Adobe parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); pendingDocuments.Remove(documentId); pendingMappings.Remove(documentId); continue; } var json = JsonSerializer.Serialize(dto, SerializerOptions); using var jsonDocument = JsonDocument.Parse(json); _schemaValidator.Validate(jsonDocument, Schema, metadata.AdvisoryId); var payload = MongoDB.Bson.BsonDocument.Parse(json); var dtoRecord = new DtoRecord( Guid.NewGuid(), document.Id, SourceName, "adobe.bulletin.v1", payload, _timeProvider.GetUtcNow()); await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); pendingDocuments.Remove(documentId); if (!pendingMappings.Contains(documentId)) { pendingMappings.Add(documentId); } } var updatedCursor = cursor .WithPendingDocuments(pendingDocuments) .WithPendingMappings(pendingMappings); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) { var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); if (cursor.PendingMappings.Count == 0) { return; } var pendingMappings = cursor.PendingMappings.ToList(); var now = _timeProvider.GetUtcNow(); foreach (var documentId in cursor.PendingMappings) { var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); if (dtoRecord is null || document is null) { pendingMappings.Remove(documentId); continue; } AdobeBulletinDto? dto; try { var json = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings { OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, }); dto = JsonSerializer.Deserialize(json, SerializerOptions); } catch (Exception ex) { _logger.LogError(ex, "Adobe DTO deserialization failed for document {DocumentId}", documentId); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); pendingMappings.Remove(documentId); continue; } if (dto is null) { _logger.LogWarning("Adobe DTO payload deserialized as null for document {DocumentId}", documentId); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); pendingMappings.Remove(documentId); continue; } var advisory = BuildAdvisory(dto, now); if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) { await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); var flag = new PsirtFlagRecord( advisory.AdvisoryKey, "Adobe", SourceName, dto.AdvisoryId, now); await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); } else { _logger.LogWarning("Skipping PSIRT flag for advisory with missing key (document {DocumentId})", documentId); } await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); pendingMappings.Remove(documentId); } var updatedCursor = cursor.WithPendingMappings(pendingMappings); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); } private IEnumerable EnumerateIndexUris() { yield return _options.IndexUri; foreach (var uri in _options.AdditionalIndexUris) { yield return uri; } } private async Task GetCursorAsync(CancellationToken cancellationToken) { var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); return AdobeCursor.FromBsonDocument(record?.Cursor); } private async Task UpdateCursorAsync(AdobeCursor cursor, CancellationToken cancellationToken) { var updatedAt = _timeProvider.GetUtcNow(); await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), updatedAt, cancellationToken).ConfigureAwait(false); } private Advisory BuildAdvisory(AdobeBulletinDto dto, DateTimeOffset recordedAt) { var provenance = new AdvisoryProvenance(SourceName, "parser", dto.AdvisoryId, recordedAt); var aliasSet = new HashSet(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId, }; foreach (var cve in dto.Cves) { if (!string.IsNullOrWhiteSpace(cve)) { aliasSet.Add(cve); } } var comparer = StringComparer.OrdinalIgnoreCase; var references = new List<(AdvisoryReference Reference, int Priority)> { (new AdvisoryReference(dto.DetailUrl, "advisory", "adobe-psirt", dto.Summary, provenance), 0), }; foreach (var cve in dto.Cves) { if (string.IsNullOrWhiteSpace(cve)) { continue; } var url = $"https://www.cve.org/CVERecord?id={cve}"; references.Add((new AdvisoryReference(url, "advisory", cve, null, provenance), 1)); } var orderedReferences = references .GroupBy(tuple => tuple.Reference.Url, comparer) .Select(group => group .OrderBy(t => t.Priority) .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) .ThenBy(t => t.Reference.Url, comparer) .First()) .OrderBy(t => t.Priority) .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) .ThenBy(t => t.Reference.Url, comparer) .Select(t => t.Reference) .ToArray(); var affected = dto.Products .Select(product => BuildPackage(product, recordedAt)) .ToArray(); var aliases = aliasSet .Where(static alias => !string.IsNullOrWhiteSpace(alias)) .Select(static alias => alias.Trim()) .Distinct(StringComparer.OrdinalIgnoreCase) .OrderBy(static alias => alias, StringComparer.Ordinal) .ToArray(); return new Advisory( dto.AdvisoryId, dto.Title, dto.Summary, language: "en", published: dto.Published, modified: null, severity: null, exploitKnown: false, aliases, orderedReferences, affected, Array.Empty(), new[] { provenance }); } private AffectedPackage BuildPackage(AdobeProductEntry product, DateTimeOffset recordedAt) { var identifier = string.IsNullOrWhiteSpace(product.Product) ? "Adobe Product" : product.Product.Trim(); var platform = string.IsNullOrWhiteSpace(product.Platform) ? null : product.Platform; var provenance = new AdvisoryProvenance( SourceName, "affected", string.IsNullOrWhiteSpace(platform) ? identifier : $"{identifier}:{platform}", recordedAt); var range = BuildVersionRange(product, recordedAt); var ranges = range is null ? Array.Empty() : new[] { range }; var normalizedVersions = BuildNormalizedVersions(product, ranges); var statuses = BuildStatuses(product, provenance); return new AffectedPackage( AffectedPackageTypes.Vendor, identifier, platform, ranges, statuses, new[] { provenance }, normalizedVersions); } private static IReadOnlyList BuildNormalizedVersions( AdobeProductEntry product, IReadOnlyList ranges) { if (ranges.Count == 0) { return Array.Empty(); } var segments = new List(); if (!string.IsNullOrWhiteSpace(product.Product)) { segments.Add(product.Product.Trim()); } if (!string.IsNullOrWhiteSpace(product.Platform)) { segments.Add(product.Platform.Trim()); } var note = segments.Count == 0 ? null : $"adobe:{string.Join(':', segments)}"; var rules = new List(ranges.Count); foreach (var range in ranges) { var rule = range.ToNormalizedVersionRule(note); if (rule is not null) { rules.Add(rule); } } return rules.Count == 0 ? Array.Empty() : rules.ToArray(); } }