Rename Concelier Source modules to Connector
This commit is contained in:
master
28
src/StellaOps.Concelier.Connector.Vndr.Adobe/AGENTS.md
Normal file
28
src/StellaOps.Concelier.Connector.Vndr.Adobe/AGENTS.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Adobe PSIRT connector ingesting APSB/APA advisories; authoritative for Adobe products; emits psirt_flags and affected ranges; establishes PSIRT precedence over registry or distro data for Adobe software.
|
||||
## Scope
|
||||
- Discover and fetch APSB/APA index and detail pages; follow product links as needed; window by advisory ID/date.
|
||||
- Validate HTML or JSON; normalize titles, CVE lists, product components, fixed versions/builds; capture mitigation notes and KBs.
|
||||
- Persist raw docs with sha256 and headers; maintain source_state cursors; ensure idempotent mapping.
|
||||
## Participants
|
||||
- Source.Common (HTTP, HTML parsing, retries/backoff, validators).
|
||||
- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state).
|
||||
- Models (canonical Advisory/Affected/Provenance).
|
||||
- Core/WebService (jobs: source:adobe:fetch|parse|map).
|
||||
- Merge engine (later) to apply PSIRT override policy for Adobe packages.
|
||||
## Interfaces & contracts
|
||||
- Aliases include APSB-YYYY-XX (and APA-* when present) plus CVE ids.
|
||||
- Affected entries capture Vendor=Adobe, Product/component names, Type=vendor, Identifier stable (for example product slug), Versions with fixed/fixedBy where available.
|
||||
- References typed: advisory, patch, mitigation, release notes; URLs normalized and deduped.
|
||||
- Provenance.method="parser"; value carries advisory id and URL; recordedAt=fetch time.
|
||||
## In/Out of scope
|
||||
In: PSIRT ingestion, aliases, affected plus fixedBy, psirt_flags, watermark/resume.
|
||||
Out: signing, package artifact downloads, non-Adobe product truth.
|
||||
## Observability & security expectations
|
||||
- Metrics: SourceDiagnostics produces `concelier.source.http.*` counters/histograms tagged `concelier.source=adobe`; operators filter on that tag to monitor fetch counts, parse failures, map affected counts, and cursor movement without bespoke metric names.
|
||||
- Logs: advisory ids, product counts, extraction timings; hosts allowlisted; no secret logging.
|
||||
## Tests
|
||||
- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Adobe.Tests`.
|
||||
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
|
||||
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.
|
||||
757
src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs
Normal file
757
src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs
Normal file
@@ -0,0 +1,757 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Json.Schema;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Json;
|
||||
using StellaOps.Concelier.Connector.Common.Packages;
|
||||
using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe;
|
||||
|
||||
public sealed class AdobeConnector : IFeedConnector
|
||||
{
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly IPsirtFlagStore _psirtFlagStore;
|
||||
private readonly IJsonSchemaValidator _schemaValidator;
|
||||
private readonly AdobeOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly AdobeDiagnostics _diagnostics;
|
||||
private readonly ILogger<AdobeConnector> _logger;
|
||||
|
||||
private static readonly JsonSchema Schema = AdobeSchemaProvider.Schema;
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
public AdobeConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IPsirtFlagStore psirtFlagStore,
|
||||
IJsonSchemaValidator schemaValidator,
|
||||
IOptions<AdobeOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
AdobeDiagnostics diagnostics,
|
||||
ILogger<AdobeConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore));
|
||||
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(AdobeProductEntry product, AdvisoryProvenance provenance)
|
||||
{
|
||||
if (!TryResolveAvailabilityStatus(product.Availability, out var status))
|
||||
{
|
||||
return Array.Empty<AffectedPackageStatus>();
|
||||
}
|
||||
|
||||
return new[] { new AffectedPackageStatus(status, provenance) };
|
||||
}
|
||||
|
||||
private static bool TryResolveAvailabilityStatus(string? availability, out string status)
|
||||
{
|
||||
status = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(availability))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = availability.Trim();
|
||||
|
||||
if (AffectedPackageStatusCatalog.TryNormalize(trimmed, out var normalized))
|
||||
{
|
||||
status = normalized;
|
||||
return true;
|
||||
}
|
||||
|
||||
var token = SanitizeStatusToken(trimmed);
|
||||
if (token.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (AvailabilityStatusMap.TryGetValue(token, out var mapped))
|
||||
{
|
||||
status = mapped;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string SanitizeStatusToken(string value)
|
||||
{
|
||||
var buffer = new char[value.Length];
|
||||
var index = 0;
|
||||
|
||||
foreach (var ch in value)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
buffer[index++] = char.ToLowerInvariant(ch);
|
||||
}
|
||||
}
|
||||
|
||||
return index == 0 ? string.Empty : new string(buffer, 0, index);
|
||||
}
|
||||
|
||||
private static readonly Dictionary<string, string> AvailabilityStatusMap = new(StringComparer.Ordinal)
|
||||
{
|
||||
["available"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["availabletoday"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["availablenow"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["updateavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["patchavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["fixavailable"] = AffectedPackageStatusCatalog.Fixed,
|
||||
["mitigationavailable"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["workaroundavailable"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["mitigationprovided"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["workaroundprovided"] = AffectedPackageStatusCatalog.Mitigated,
|
||||
["planned"] = AffectedPackageStatusCatalog.Pending,
|
||||
["updateplanned"] = AffectedPackageStatusCatalog.Pending,
|
||||
["plannedupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["scheduled"] = AffectedPackageStatusCatalog.Pending,
|
||||
["scheduledupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingavailability"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingupdate"] = AffectedPackageStatusCatalog.Pending,
|
||||
["pendingfix"] = AffectedPackageStatusCatalog.Pending,
|
||||
["notavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["unavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["notcurrentlyavailable"] = AffectedPackageStatusCatalog.Unknown,
|
||||
["notapplicable"] = AffectedPackageStatusCatalog.NotApplicable,
|
||||
};
|
||||
|
||||
private AffectedVersionRange? BuildVersionRange(AdobeProductEntry product, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.AffectedVersion) && string.IsNullOrWhiteSpace(product.UpdatedVersion))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var key = string.IsNullOrWhiteSpace(product.Platform)
|
||||
? product.Product
|
||||
: $"{product.Product}:{product.Platform}";
|
||||
|
||||
var provenance = new AdvisoryProvenance(SourceName, "range", key, recordedAt);
|
||||
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
AddExtension(extensions, "adobe.track", product.Track);
|
||||
AddExtension(extensions, "adobe.platform", product.Platform);
|
||||
AddExtension(extensions, "adobe.affected.raw", product.AffectedVersion);
|
||||
AddExtension(extensions, "adobe.updated.raw", product.UpdatedVersion);
|
||||
AddExtension(extensions, "adobe.priority", product.Priority);
|
||||
AddExtension(extensions, "adobe.availability", product.Availability);
|
||||
|
||||
var lastAffected = ExtractVersionNumber(product.AffectedVersion);
|
||||
var fixedVersion = ExtractVersionNumber(product.UpdatedVersion);
|
||||
|
||||
var primitives = BuildRangePrimitives(lastAffected, fixedVersion, extensions);
|
||||
|
||||
return new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: null,
|
||||
fixedVersion: fixedVersion,
|
||||
lastAffectedVersion: lastAffected,
|
||||
rangeExpression: product.AffectedVersion ?? product.UpdatedVersion,
|
||||
provenance: provenance,
|
||||
primitives: primitives);
|
||||
}
|
||||
|
||||
private static RangePrimitives? BuildRangePrimitives(string? lastAffected, string? fixedVersion, Dictionary<string, string> extensions)
|
||||
{
|
||||
var semVer = BuildSemVerPrimitive(lastAffected, fixedVersion);
|
||||
|
||||
if (semVer is null && extensions.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions);
|
||||
}
|
||||
|
||||
private static SemVerPrimitive? BuildSemVerPrimitive(string? lastAffected, string? fixedVersion)
|
||||
{
|
||||
var fixedNormalized = NormalizeSemVer(fixedVersion);
|
||||
var lastNormalized = NormalizeSemVer(lastAffected);
|
||||
|
||||
if (fixedNormalized is null && lastNormalized is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new SemVerPrimitive(
|
||||
Introduced: null,
|
||||
IntroducedInclusive: true,
|
||||
Fixed: fixedNormalized,
|
||||
FixedInclusive: false,
|
||||
LastAffected: lastNormalized,
|
||||
LastAffectedInclusive: true,
|
||||
ConstraintExpression: null);
|
||||
}
|
||||
|
||||
private static string? NormalizeSemVer(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (PackageCoordinateHelper.TryParseSemVer(trimmed, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (Version.TryParse(trimmed, out var parsed))
|
||||
{
|
||||
if (parsed.Build >= 0 && parsed.Revision >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}";
|
||||
}
|
||||
|
||||
if (parsed.Build >= 0)
|
||||
{
|
||||
return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}";
|
||||
}
|
||||
|
||||
return $"{parsed.Major}.{parsed.Minor}";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ExtractVersionNumber(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = VersionPattern.Match(text);
|
||||
return match.Success ? match.Value : null;
|
||||
}
|
||||
|
||||
private static void AddExtension(IDictionary<string, string> extensions, string key, string? value)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
extensions[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
private static readonly Regex VersionPattern = new("\\d+(?:\\.\\d+)+", RegexOptions.Compiled);
|
||||
|
||||
public string SourceName => VndrAdobeConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var backfillStart = now - _options.InitialBackfill;
|
||||
var windowStart = cursor.LastPublished.HasValue
|
||||
? cursor.LastPublished.Value - _options.WindowOverlap
|
||||
: backfillStart;
|
||||
if (windowStart < backfillStart)
|
||||
{
|
||||
windowStart = backfillStart;
|
||||
}
|
||||
|
||||
var maxPublished = cursor.LastPublished;
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var fetchCache = cursor.FetchCache is null
|
||||
? new Dictionary<string, AdobeFetchCacheEntry>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, AdobeFetchCacheEntry>(cursor.FetchCache, StringComparer.Ordinal);
|
||||
var touchedResources = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
var collectedEntries = new Dictionary<string, AdobeIndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var indexUri in EnumerateIndexUris())
|
||||
{
|
||||
_diagnostics.FetchAttempt();
|
||||
string? html = null;
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(AdobeOptions.HttpClientName);
|
||||
using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
html = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to download Adobe index page {Uri}", indexUri);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(html))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
IReadOnlyCollection<AdobeIndexEntry> entries;
|
||||
try
|
||||
{
|
||||
entries = AdobeIndexParser.Parse(html, indexUri);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to parse Adobe index page {Uri}", indexUri);
|
||||
_diagnostics.FetchFailure();
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (entry.PublishedUtc < windowStart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!collectedEntries.TryGetValue(entry.AdvisoryId, out var existing) || entry.PublishedUtc > existing.PublishedUtc)
|
||||
{
|
||||
collectedEntries[entry.AdvisoryId] = entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var entry in collectedEntries.Values.OrderBy(static e => e.PublishedUtc))
|
||||
{
|
||||
if (!maxPublished.HasValue || entry.PublishedUtc > maxPublished)
|
||||
{
|
||||
maxPublished = entry.PublishedUtc;
|
||||
}
|
||||
|
||||
var cacheKey = entry.DetailUri.ToString();
|
||||
touchedResources.Add(cacheKey);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["advisoryId"] = entry.AdvisoryId,
|
||||
["published"] = entry.PublishedUtc.ToString("O"),
|
||||
["title"] = entry.Title ?? string.Empty,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(AdobeOptions.HttpClientName, SourceName, entry.DetailUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" },
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cursor.TryGetFetchCache(cacheKey, out var cached)
|
||||
&& string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256);
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.FetchDocument();
|
||||
fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256);
|
||||
|
||||
if (!pendingDocuments.Contains(result.Document.Id))
|
||||
{
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Failed to fetch Adobe advisory {AdvisoryId} ({Uri})", entry.AdvisoryId, entry.DetailUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in fetchCache.Keys.ToList())
|
||||
{
|
||||
if (!touchedResources.Contains(key))
|
||||
{
|
||||
fetchCache.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithLastPublished(maxPublished)
|
||||
.WithFetchCache(fetchCache);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Adobe document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeDocumentMetadata metadata;
|
||||
try
|
||||
{
|
||||
metadata = AdobeDocumentMetadata.FromDocument(document);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe metadata parse failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeBulletinDto dto;
|
||||
try
|
||||
{
|
||||
var bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
var html = Encoding.UTF8.GetString(bytes);
|
||||
dto = AdobeDetailParser.Parse(html, metadata);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(dto, SerializerOptions);
|
||||
using var jsonDocument = JsonDocument.Parse(json);
|
||||
_schemaValidator.Validate(jsonDocument, Schema, metadata.AdvisoryId);
|
||||
|
||||
var payload = MongoDB.Bson.BsonDocument.Parse(json);
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
"adobe.bulletin.v1",
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dtoRecord is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AdobeBulletinDto? dto;
|
||||
try
|
||||
{
|
||||
var json = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
dto = JsonSerializer.Deserialize<AdobeBulletinDto>(json, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Adobe DTO deserialization failed for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_logger.LogWarning("Adobe DTO payload deserialized as null for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisory = BuildAdvisory(dto, now);
|
||||
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
|
||||
{
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var flag = new PsirtFlagRecord(
|
||||
advisory.AdvisoryKey,
|
||||
"Adobe",
|
||||
SourceName,
|
||||
dto.AdvisoryId,
|
||||
now);
|
||||
|
||||
await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("Skipping PSIRT flag for advisory with missing key (document {DocumentId})", documentId);
|
||||
}
|
||||
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private IEnumerable<Uri> EnumerateIndexUris()
|
||||
{
|
||||
yield return _options.IndexUri;
|
||||
foreach (var uri in _options.AdditionalIndexUris)
|
||||
{
|
||||
yield return uri;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<AdobeCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return AdobeCursor.FromBsonDocument(record?.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(AdobeCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var updatedAt = _timeProvider.GetUtcNow();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), updatedAt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Advisory BuildAdvisory(AdobeBulletinDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(SourceName, "parser", dto.AdvisoryId, recordedAt);
|
||||
|
||||
var aliasSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
dto.AdvisoryId,
|
||||
};
|
||||
foreach (var cve in dto.Cves)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliasSet.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
var comparer = StringComparer.OrdinalIgnoreCase;
|
||||
var references = new List<(AdvisoryReference Reference, int Priority)>
|
||||
{
|
||||
(new AdvisoryReference(dto.DetailUrl, "advisory", "adobe-psirt", dto.Summary, provenance), 0),
|
||||
};
|
||||
|
||||
foreach (var cve in dto.Cves)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var url = $"https://www.cve.org/CVERecord?id={cve}";
|
||||
references.Add((new AdvisoryReference(url, "advisory", cve, null, provenance), 1));
|
||||
}
|
||||
|
||||
var orderedReferences = references
|
||||
.GroupBy(tuple => tuple.Reference.Url, comparer)
|
||||
.Select(group => group
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.First())
|
||||
.OrderBy(t => t.Priority)
|
||||
.ThenBy(t => t.Reference.Kind ?? string.Empty, comparer)
|
||||
.ThenBy(t => t.Reference.Url, comparer)
|
||||
.Select(t => t.Reference)
|
||||
.ToArray();
|
||||
|
||||
var affected = dto.Products
|
||||
.Select(product => BuildPackage(product, recordedAt))
|
||||
.ToArray();
|
||||
|
||||
var aliases = aliasSet
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Select(static alias => alias.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new Advisory(
|
||||
dto.AdvisoryId,
|
||||
dto.Title,
|
||||
dto.Summary,
|
||||
language: "en",
|
||||
published: dto.Published,
|
||||
modified: null,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
orderedReferences,
|
||||
affected,
|
||||
Array.Empty<CvssMetric>(),
|
||||
new[] { provenance });
|
||||
}
|
||||
|
||||
private AffectedPackage BuildPackage(AdobeProductEntry product, DateTimeOffset recordedAt)
|
||||
{
|
||||
var identifier = string.IsNullOrWhiteSpace(product.Product)
|
||||
? "Adobe Product"
|
||||
: product.Product.Trim();
|
||||
|
||||
var platform = string.IsNullOrWhiteSpace(product.Platform) ? null : product.Platform;
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
SourceName,
|
||||
"affected",
|
||||
string.IsNullOrWhiteSpace(platform) ? identifier : $"{identifier}:{platform}",
|
||||
recordedAt);
|
||||
|
||||
var range = BuildVersionRange(product, recordedAt);
|
||||
var ranges = range is null ? Array.Empty<AffectedVersionRange>() : new[] { range };
|
||||
var normalizedVersions = BuildNormalizedVersions(product, ranges);
|
||||
var statuses = BuildStatuses(product, provenance);
|
||||
|
||||
return new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform,
|
||||
ranges,
|
||||
statuses,
|
||||
new[] { provenance },
|
||||
normalizedVersions);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
AdobeProductEntry product,
|
||||
IReadOnlyList<AffectedVersionRange> ranges)
|
||||
{
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var segments = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Product))
|
||||
{
|
||||
segments.Add(product.Product.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Platform))
|
||||
{
|
||||
segments.Add(product.Platform.Trim());
|
||||
}
|
||||
|
||||
var note = segments.Count == 0 ? null : $"adobe:{string.Join(':', segments)}";
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(ranges.Count);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var rule = range.ToNormalizedVersionRule(note);
|
||||
if (rule is not null)
|
||||
{
|
||||
rules.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe;
|
||||
|
||||
public sealed class VndrAdobeConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "vndr-adobe";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<AdobeConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<AdobeConnector>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe;
|
||||
|
||||
public sealed class AdobeDiagnostics : IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Adobe";
|
||||
private static readonly string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchDocuments;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
|
||||
public AdobeDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "adobe.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of Adobe index fetch operations.");
|
||||
_fetchDocuments = _meter.CreateCounter<long>(
|
||||
name: "adobe.fetch.documents",
|
||||
unit: "documents",
|
||||
description: "Number of Adobe advisory documents captured.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "adobe.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of Adobe fetch failures.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "adobe.fetch.unchanged",
|
||||
unit: "documents",
|
||||
description: "Number of Adobe advisories skipped due to unchanged content.");
|
||||
}
|
||||
|
||||
public Meter Meter => _meter;
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchDocument() => _fetchDocuments.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe;
|
||||
|
||||
public static class AdobeServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddAdobeConnector(this IServiceCollection services, Action<AdobeOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<AdobeOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(AdobeOptions.HttpClientName, static (sp, options) =>
|
||||
{
|
||||
var adobeOptions = sp.GetRequiredService<IOptions<AdobeOptions>>().Value;
|
||||
options.BaseAddress = adobeOptions.IndexUri;
|
||||
options.UserAgent = "StellaOps.Concelier.VndrAdobe/1.0";
|
||||
options.Timeout = TimeSpan.FromSeconds(20);
|
||||
options.AllowedHosts.Clear();
|
||||
options.AllowedHosts.Add(adobeOptions.IndexUri.Host);
|
||||
foreach (var additional in adobeOptions.AdditionalIndexUris)
|
||||
{
|
||||
options.AllowedHosts.Add(additional.Host);
|
||||
}
|
||||
});
|
||||
|
||||
services.TryAddSingleton<AdobeDiagnostics>();
|
||||
services.AddTransient<AdobeConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Configuration;
|
||||
|
||||
public sealed class AdobeOptions
|
||||
{
|
||||
public const string HttpClientName = "source-vndr-adobe";
|
||||
|
||||
public Uri IndexUri { get; set; } = new("https://helpx.adobe.com/security/security-bulletin.html");
|
||||
|
||||
public List<Uri> AdditionalIndexUris { get; } = new();
|
||||
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90);
|
||||
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(3);
|
||||
|
||||
public int MaxEntriesPerFetch { get; set; } = 100;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (IndexUri is null || !IndexUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new ArgumentException("IndexUri must be an absolute URI.", nameof(IndexUri));
|
||||
}
|
||||
|
||||
foreach (var uri in AdditionalIndexUris)
|
||||
{
|
||||
if (uri is null || !uri.IsAbsoluteUri)
|
||||
{
|
||||
throw new ArgumentException("Additional index URIs must be absolute.", nameof(AdditionalIndexUris));
|
||||
}
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentException("InitialBackfill must be positive.", nameof(InitialBackfill));
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentException("WindowOverlap cannot be negative.", nameof(WindowOverlap));
|
||||
}
|
||||
|
||||
if (MaxEntriesPerFetch <= 0)
|
||||
{
|
||||
throw new ArgumentException("MaxEntriesPerFetch must be positive.", nameof(MaxEntriesPerFetch));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal sealed record AdobeBulletinDto(
|
||||
string AdvisoryId,
|
||||
string Title,
|
||||
DateTimeOffset Published,
|
||||
IReadOnlyList<AdobeProductEntry> Products,
|
||||
IReadOnlyList<string> Cves,
|
||||
string DetailUrl,
|
||||
string? Summary)
|
||||
{
|
||||
public static AdobeBulletinDto Create(
|
||||
string advisoryId,
|
||||
string title,
|
||||
DateTimeOffset published,
|
||||
IEnumerable<AdobeProductEntry>? products,
|
||||
IEnumerable<string>? cves,
|
||||
Uri detailUri,
|
||||
string? summary)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(advisoryId);
|
||||
ArgumentException.ThrowIfNullOrEmpty(title);
|
||||
ArgumentNullException.ThrowIfNull(detailUri);
|
||||
|
||||
var productList = products?
|
||||
.Where(static p => !string.IsNullOrWhiteSpace(p.Product))
|
||||
.Select(static p => p with { Product = p.Product.Trim() })
|
||||
.Distinct(AdobeProductEntryComparer.Instance)
|
||||
.OrderBy(static p => p.Product, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static p => p.Platform, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static p => p.Track, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList()
|
||||
?? new List<AdobeProductEntry>();
|
||||
|
||||
var cveList = cves?.Where(static c => !string.IsNullOrWhiteSpace(c))
|
||||
.Select(static c => c.Trim().ToUpperInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(static c => c, StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
|
||||
return new AdobeBulletinDto(
|
||||
advisoryId.ToUpperInvariant(),
|
||||
title.Trim(),
|
||||
published.ToUniversalTime(),
|
||||
productList,
|
||||
cveList,
|
||||
detailUri.ToString(),
|
||||
string.IsNullOrWhiteSpace(summary) ? null : summary.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record AdobeProductEntry(
|
||||
string Product,
|
||||
string Track,
|
||||
string Platform,
|
||||
string? AffectedVersion,
|
||||
string? UpdatedVersion,
|
||||
string? Priority,
|
||||
string? Availability);
|
||||
|
||||
internal sealed class AdobeProductEntryComparer : IEqualityComparer<AdobeProductEntry>
|
||||
{
|
||||
public static AdobeProductEntryComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(AdobeProductEntry? x, AdobeProductEntry? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.AffectedVersion, y.AffectedVersion, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.UpdatedVersion, y.UpdatedVersion, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Priority, y.Priority, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Availability, y.Availability, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(AdobeProductEntry obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.AffectedVersion, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.UpdatedVersion, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Priority, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Availability, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal sealed record AdobeCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, AdobeFetchCacheEntry>? FetchCache)
|
||||
{
|
||||
public static AdobeCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>(), null);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString()));
|
||||
document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString()));
|
||||
|
||||
if (FetchCache is { Count: > 0 })
|
||||
{
|
||||
var cacheDocument = new BsonDocument();
|
||||
foreach (var (key, entry) in FetchCache)
|
||||
{
|
||||
cacheDocument[key] = entry.ToBson();
|
||||
}
|
||||
|
||||
document["fetchCache"] = cacheDocument;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static AdobeCursor FromBsonDocument(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
DateTimeOffset? lastPublished = null;
|
||||
if (document.TryGetValue("lastPublished", out var lastPublishedValue))
|
||||
{
|
||||
lastPublished = ReadDateTime(lastPublishedValue);
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var fetchCache = ReadFetchCache(document);
|
||||
|
||||
return new AdobeCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache);
|
||||
}
|
||||
|
||||
public AdobeCursor WithLastPublished(DateTimeOffset? value)
|
||||
=> this with { LastPublished = value?.ToUniversalTime() };
|
||||
|
||||
public AdobeCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public AdobeCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public AdobeCursor WithFetchCache(IDictionary<string, AdobeFetchCacheEntry>? cache)
|
||||
{
|
||||
if (cache is null)
|
||||
{
|
||||
return this with { FetchCache = null };
|
||||
}
|
||||
|
||||
var target = new Dictionary<string, AdobeFetchCacheEntry>(cache, StringComparer.Ordinal);
|
||||
return this with { FetchCache = target };
|
||||
}
|
||||
|
||||
public bool TryGetFetchCache(string key, out AdobeFetchCacheEntry entry)
|
||||
{
|
||||
var cache = FetchCache;
|
||||
if (cache is null)
|
||||
{
|
||||
entry = AdobeFetchCacheEntry.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (cache.TryGetValue(key, out var value) && value is not null)
|
||||
{
|
||||
entry = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
entry = AdobeFetchCacheEntry.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ReadDateTime(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var list = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
list.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, AdobeFetchCacheEntry>? ReadFetchCache(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var dictionary = new Dictionary<string, AdobeFetchCacheEntry>(StringComparer.Ordinal);
|
||||
foreach (var element in cacheDocument.Elements)
|
||||
{
|
||||
if (element.Value is BsonDocument entryDocument)
|
||||
{
|
||||
dictionary[element.Name] = AdobeFetchCacheEntry.FromBson(entryDocument);
|
||||
}
|
||||
}
|
||||
|
||||
return dictionary;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record AdobeFetchCacheEntry(string Sha256)
|
||||
{
|
||||
public static AdobeFetchCacheEntry Empty { get; } = new(string.Empty);
|
||||
|
||||
public BsonDocument ToBson()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["sha256"] = Sha256,
|
||||
};
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static AdobeFetchCacheEntry FromBson(BsonDocument document)
|
||||
{
|
||||
var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty;
|
||||
return new AdobeFetchCacheEntry(sha);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal static class AdobeDetailParser
|
||||
{
|
||||
private static readonly HtmlParser Parser = new();
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly string[] DateMarkers = { "date published", "release date", "published" };
|
||||
|
||||
public static AdobeBulletinDto Parse(string html, AdobeDocumentMetadata metadata)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(html);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
|
||||
using var document = Parser.ParseDocument(html);
|
||||
var title = metadata.Title ?? document.QuerySelector("h1")?.TextContent?.Trim() ?? metadata.AdvisoryId;
|
||||
var summary = document.QuerySelector("p")?.TextContent?.Trim();
|
||||
|
||||
var published = metadata.PublishedUtc ?? TryExtractPublished(document) ?? DateTimeOffset.UtcNow;
|
||||
|
||||
var cves = ExtractCves(document.Body?.TextContent ?? string.Empty);
|
||||
var products = ExtractProductEntries(title, document);
|
||||
|
||||
return AdobeBulletinDto.Create(
|
||||
metadata.AdvisoryId,
|
||||
title,
|
||||
published,
|
||||
products,
|
||||
cves,
|
||||
metadata.DetailUri,
|
||||
summary);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCves(string text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(match.Value))
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
|
||||
return set.Count == 0 ? Array.Empty<string>() : set.OrderBy(static cve => cve, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdobeProductEntry> ExtractProductEntries(string title, IDocument document)
|
||||
{
|
||||
var builders = new Dictionary<AdobeProductKey, AdobeProductEntryBuilder>(AdobeProductKeyComparer.Instance);
|
||||
|
||||
foreach (var builder in ParseAffectedTable(document))
|
||||
{
|
||||
builders[builder.Key] = builder;
|
||||
}
|
||||
|
||||
foreach (var updated in ParseUpdatedTable(document))
|
||||
{
|
||||
if (builders.TryGetValue(updated.Key, out var builder))
|
||||
{
|
||||
builder.UpdatedVersion ??= updated.UpdatedVersion;
|
||||
builder.Priority ??= updated.Priority;
|
||||
builder.Availability ??= updated.Availability;
|
||||
}
|
||||
else
|
||||
{
|
||||
builders[updated.Key] = updated;
|
||||
}
|
||||
}
|
||||
|
||||
if (builders.Count == 0 && !string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
var fallback = new AdobeProductEntryBuilder(
|
||||
NormalizeWhitespace(title),
|
||||
string.Empty,
|
||||
string.Empty)
|
||||
{
|
||||
AffectedVersion = null,
|
||||
UpdatedVersion = null,
|
||||
Priority = null,
|
||||
Availability = null
|
||||
};
|
||||
|
||||
builders[fallback.Key] = fallback;
|
||||
}
|
||||
|
||||
return builders.Values
|
||||
.Select(static builder => builder.ToEntry())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static IEnumerable<AdobeProductEntryBuilder> ParseAffectedTable(IDocument document)
|
||||
{
|
||||
var table = FindTableByHeader(document, "Affected Versions");
|
||||
if (table is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var row in table.Rows.Skip(1))
|
||||
{
|
||||
var cells = row.Cells;
|
||||
if (cells.Length < 3)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var product = NormalizeWhitespace(cells[0]?.TextContent);
|
||||
var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent);
|
||||
var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var affectedCell = cells[2];
|
||||
foreach (var line in ExtractLines(affectedCell))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var (platform, versionText) = SplitPlatformLine(line, platformText);
|
||||
var builder = new AdobeProductEntryBuilder(product, track, platform)
|
||||
{
|
||||
AffectedVersion = versionText
|
||||
};
|
||||
|
||||
yield return builder;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<AdobeProductEntryBuilder> ParseUpdatedTable(IDocument document)
|
||||
{
|
||||
var table = FindTableByHeader(document, "Updated Versions");
|
||||
if (table is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var row in table.Rows.Skip(1))
|
||||
{
|
||||
var cells = row.Cells;
|
||||
if (cells.Length < 3)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var product = NormalizeWhitespace(cells[0]?.TextContent);
|
||||
var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent);
|
||||
var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent);
|
||||
var priority = NormalizeWhitespace(cells.ElementAtOrDefault(4)?.TextContent);
|
||||
var availability = NormalizeWhitespace(cells.ElementAtOrDefault(5)?.TextContent);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var updatedCell = cells[2];
|
||||
var lines = ExtractLines(updatedCell);
|
||||
if (lines.Count == 0)
|
||||
{
|
||||
lines.Add(updatedCell.TextContent ?? string.Empty);
|
||||
}
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var (platform, versionText) = SplitPlatformLine(line, platformText);
|
||||
var builder = new AdobeProductEntryBuilder(product, track, platform)
|
||||
{
|
||||
UpdatedVersion = versionText,
|
||||
Priority = priority,
|
||||
Availability = availability
|
||||
};
|
||||
|
||||
yield return builder;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IHtmlTableElement? FindTableByHeader(IDocument document, string headerText)
|
||||
{
|
||||
return document
|
||||
.QuerySelectorAll("table")
|
||||
.OfType<IHtmlTableElement>()
|
||||
.FirstOrDefault(table => table.TextContent.Contains(headerText, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static List<string> ExtractLines(IElement? cell)
|
||||
{
|
||||
var lines = new List<string>();
|
||||
if (cell is null)
|
||||
{
|
||||
return lines;
|
||||
}
|
||||
|
||||
var paragraphs = cell.QuerySelectorAll("p").Select(static p => p.TextContent).ToArray();
|
||||
if (paragraphs.Length > 0)
|
||||
{
|
||||
foreach (var paragraph in paragraphs)
|
||||
{
|
||||
var normalized = NormalizeWhitespace(paragraph);
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
lines.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
var items = cell.QuerySelectorAll("li").Select(static li => li.TextContent).ToArray();
|
||||
if (items.Length > 0)
|
||||
{
|
||||
foreach (var item in items)
|
||||
{
|
||||
var normalized = NormalizeWhitespace(item);
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
lines.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
var raw = NormalizeWhitespace(cell.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
lines.AddRange(raw.Split(new[] { '\n' }, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries));
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
private static (string Platform, string? Version) SplitPlatformLine(string line, string? fallbackPlatform)
|
||||
{
|
||||
var separatorIndex = line.IndexOf('-', StringComparison.Ordinal);
|
||||
if (separatorIndex > 0 && separatorIndex < line.Length - 1)
|
||||
{
|
||||
var prefix = line[..separatorIndex].Trim();
|
||||
var versionText = line[(separatorIndex + 1)..].Trim();
|
||||
return (NormalizePlatform(prefix) ?? NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, versionText);
|
||||
}
|
||||
|
||||
return (NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, line.Trim());
|
||||
}
|
||||
|
||||
private static string? NormalizePlatform(string? platform)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(platform))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = platform.Trim();
|
||||
return trimmed.ToLowerInvariant() switch
|
||||
{
|
||||
"win" or "windows" => "Windows",
|
||||
"mac" or "macos" or "mac os" => "macOS",
|
||||
"windows & macos" or "windows & macos" => "Windows & macOS",
|
||||
_ => trimmed
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTimeOffset? TryExtractPublished(IDocument document)
|
||||
{
|
||||
var candidates = new List<string?>();
|
||||
candidates.Add(document.QuerySelector("time")?.GetAttribute("datetime"));
|
||||
candidates.Add(document.QuerySelector("time")?.TextContent);
|
||||
|
||||
foreach (var marker in DateMarkers)
|
||||
{
|
||||
var element = document.All.FirstOrDefault(node => node.TextContent.Contains(marker, StringComparison.OrdinalIgnoreCase));
|
||||
if (element is not null)
|
||||
{
|
||||
candidates.Add(element.TextContent);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (TryParseDate(candidate, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryParseDate(string? value, out DateTimeOffset result)
|
||||
{
|
||||
result = default;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result))
|
||||
{
|
||||
result = result.ToUniversalTime();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (DateTime.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date))
|
||||
{
|
||||
result = new DateTimeOffset(date, TimeSpan.Zero).ToUniversalTime();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string NormalizeWhitespace(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var sanitized = value ?? string.Empty;
|
||||
return string.Join(" ", sanitized.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
|
||||
}
|
||||
|
||||
private sealed record AdobeProductKey(string Product, string Track, string Platform);
|
||||
|
||||
private sealed class AdobeProductKeyComparer : IEqualityComparer<AdobeProductKey>
|
||||
{
|
||||
public static AdobeProductKeyComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(AdobeProductKey? x, AdobeProductKey? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(AdobeProductKey obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class AdobeProductEntryBuilder
|
||||
{
|
||||
public AdobeProductEntryBuilder(string product, string track, string platform)
|
||||
{
|
||||
Product = NormalizeWhitespace(product);
|
||||
Track = NormalizeWhitespace(track);
|
||||
Platform = NormalizeWhitespace(platform);
|
||||
}
|
||||
|
||||
public AdobeProductKey Key => new(Product, Track, Platform);
|
||||
|
||||
public string Product { get; }
|
||||
public string Track { get; }
|
||||
public string Platform { get; }
|
||||
|
||||
public string? AffectedVersion { get; set; }
|
||||
public string? UpdatedVersion { get; set; }
|
||||
public string? Priority { get; set; }
|
||||
public string? Availability { get; set; }
|
||||
|
||||
public AdobeProductEntry ToEntry()
|
||||
=> new(Product, Track, Platform, AffectedVersion, UpdatedVersion, Priority, Availability);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal sealed record AdobeDocumentMetadata(
|
||||
string AdvisoryId,
|
||||
string? Title,
|
||||
DateTimeOffset? PublishedUtc,
|
||||
Uri DetailUri)
|
||||
{
|
||||
private const string AdvisoryIdKey = "advisoryId";
|
||||
private const string TitleKey = "title";
|
||||
private const string PublishedKey = "published";
|
||||
|
||||
public static AdobeDocumentMetadata FromDocument(DocumentRecord document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (document.Metadata is null)
|
||||
{
|
||||
throw new InvalidOperationException("Adobe document metadata is missing.");
|
||||
}
|
||||
|
||||
var advisoryId = document.Metadata.TryGetValue(AdvisoryIdKey, out var idValue) ? idValue : null;
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
throw new InvalidOperationException("Adobe document advisoryId metadata missing.");
|
||||
}
|
||||
|
||||
var title = document.Metadata.TryGetValue(TitleKey, out var titleValue) ? titleValue : null;
|
||||
DateTimeOffset? published = null;
|
||||
if (document.Metadata.TryGetValue(PublishedKey, out var publishedValue)
|
||||
&& DateTimeOffset.TryParse(publishedValue, out var parsedPublished))
|
||||
{
|
||||
published = parsedPublished.ToUniversalTime();
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri))
|
||||
{
|
||||
throw new InvalidOperationException("Adobe document URI invalid.");
|
||||
}
|
||||
|
||||
return new AdobeDocumentMetadata(advisoryId.Trim(), string.IsNullOrWhiteSpace(title) ? null : title.Trim(), published, detailUri);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal sealed record AdobeIndexEntry(string AdvisoryId, Uri DetailUri, DateTimeOffset PublishedUtc, string? Title);
|
||||
@@ -0,0 +1,159 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal static class AdobeIndexParser
|
||||
{
|
||||
private static readonly HtmlParser Parser = new();
|
||||
private static readonly Regex AdvisoryIdRegex = new("(APSB|APA)\\d{2}-\\d{2,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly string[] ExplicitFormats =
|
||||
{
|
||||
"MMMM d, yyyy",
|
||||
"MMM d, yyyy",
|
||||
"M/d/yyyy",
|
||||
"MM/dd/yyyy",
|
||||
"yyyy-MM-dd",
|
||||
};
|
||||
|
||||
public static IReadOnlyCollection<AdobeIndexEntry> Parse(string html, Uri baseUri)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(html);
|
||||
ArgumentNullException.ThrowIfNull(baseUri);
|
||||
|
||||
var document = Parser.ParseDocument(html);
|
||||
var map = new Dictionary<string, AdobeIndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
var anchors = document.QuerySelectorAll("a[href]");
|
||||
|
||||
foreach (var anchor in anchors)
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!href.Contains("/security/products/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryExtractAdvisoryId(anchor.TextContent, href, out var advisoryId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(baseUri, href, out var detailUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var published = TryResolvePublished(anchor) ?? DateTimeOffset.UtcNow;
|
||||
var entry = new AdobeIndexEntry(advisoryId.ToUpperInvariant(), detailUri, published, anchor.TextContent?.Trim());
|
||||
map[entry.AdvisoryId] = entry;
|
||||
}
|
||||
|
||||
return map.Values
|
||||
.OrderBy(static e => e.PublishedUtc)
|
||||
.ThenBy(static e => e.AdvisoryId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool TryExtractAdvisoryId(string? text, string href, out string advisoryId)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
var match = AdvisoryIdRegex.Match(text);
|
||||
if (match.Success)
|
||||
{
|
||||
advisoryId = match.Value.ToUpperInvariant();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
var hrefMatch = AdvisoryIdRegex.Match(href);
|
||||
if (hrefMatch.Success)
|
||||
{
|
||||
advisoryId = hrefMatch.Value.ToUpperInvariant();
|
||||
return true;
|
||||
}
|
||||
|
||||
advisoryId = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? TryResolvePublished(IElement anchor)
|
||||
{
|
||||
var row = anchor.Closest("tr");
|
||||
if (row is not null)
|
||||
{
|
||||
var cells = row.GetElementsByTagName("td");
|
||||
if (cells.Length >= 2)
|
||||
{
|
||||
for (var idx = 1; idx < cells.Length; idx++)
|
||||
{
|
||||
if (TryParseDate(cells[idx].TextContent, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sibling = anchor.NextElementSibling;
|
||||
while (sibling is not null)
|
||||
{
|
||||
if (TryParseDate(sibling.TextContent, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
sibling = sibling.NextElementSibling;
|
||||
}
|
||||
|
||||
if (TryParseDate(anchor.ParentElement?.TextContent, out var parentDate))
|
||||
{
|
||||
return parentDate;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryParseDate(string? value, out DateTimeOffset result)
|
||||
{
|
||||
result = default;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result))
|
||||
{
|
||||
return Normalize(ref result);
|
||||
}
|
||||
|
||||
foreach (var format in ExplicitFormats)
|
||||
{
|
||||
if (DateTime.TryParseExact(trimmed, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date))
|
||||
{
|
||||
result = new DateTimeOffset(date, TimeSpan.Zero);
|
||||
return Normalize(ref result);
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool Normalize(ref DateTimeOffset value)
|
||||
{
|
||||
value = value.ToUniversalTime();
|
||||
value = new DateTimeOffset(value.Year, value.Month, value.Day, 0, 0, 0, TimeSpan.Zero);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using Json.Schema;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal;
|
||||
|
||||
internal static class AdobeSchemaProvider
|
||||
{
|
||||
private static readonly Lazy<JsonSchema> Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
public static JsonSchema Schema => Cached.Value;
|
||||
|
||||
private static JsonSchema Load()
|
||||
{
|
||||
var assembly = typeof(AdobeSchemaProvider).GetTypeInfo().Assembly;
|
||||
const string resourceName = "StellaOps.Concelier.Connector.Vndr.Adobe.Schemas.adobe-bulletin.schema.json";
|
||||
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName)
|
||||
?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found.");
|
||||
using var reader = new StreamReader(stream);
|
||||
var schemaText = reader.ReadToEnd();
|
||||
return JsonSchema.FromText(schemaText);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://stellaops.example/schemas/adobe-bulletin.schema.json",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"advisoryId",
|
||||
"title",
|
||||
"published",
|
||||
"products",
|
||||
"cves",
|
||||
"detailUrl"
|
||||
],
|
||||
"properties": {
|
||||
"advisoryId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"title": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"published": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"products": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"product",
|
||||
"track",
|
||||
"platform"
|
||||
],
|
||||
"properties": {
|
||||
"product": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"track": {
|
||||
"type": "string"
|
||||
},
|
||||
"platform": {
|
||||
"type": "string"
|
||||
},
|
||||
"affectedVersion": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"updatedVersion": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"priority": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"availability": {
|
||||
"type": ["string", "null"]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"cves": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^CVE-\\d{4}-\\d{4,}$"
|
||||
}
|
||||
},
|
||||
"detailUrl": {
|
||||
"type": "string",
|
||||
"format": "uri"
|
||||
},
|
||||
"summary": {
|
||||
"type": ["string", "null"]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AngleSharp" Version="1.1.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Schemas\adobe-bulletin.schema.json" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
12
src/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md
Normal file
12
src/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Index discovery and sliding window fetch|BE-Conn-Adobe|Source.Common|DONE — Support backfill; honor robots/ToS.|
|
||||
|Detail extractor (products/components/fixes)|BE-Conn-Adobe|Source.Common|DONE — Normalizes metadata and CVE/product capture.|
|
||||
|DTO schema and validation pipeline|BE-Conn-Adobe, QA|Source.Common|DONE — JSON schema enforced during parse.|
|
||||
|Canonical mapping plus psirt_flags|BE-Conn-Adobe|Models|DONE — Emits canonical advisory and Adobe psirt flag.|
|
||||
|SourceState plus sha256 short-circuit|BE-Conn-Adobe|Storage.Mongo|DONE — Idempotence guarantee.|
|
||||
|Golden fixtures and determinism tests|QA|Source.Vndr.Adobe|**DONE** — connector tests assert snapshot determinism for dual advisories.|
|
||||
|Mark failed parse DTOs|BE-Conn-Adobe|Storage.Mongo|**DONE** — parse failures now mark documents `Failed` and tests cover the path.|
|
||||
|Reference dedupe & ordering|BE-Conn-Adobe|Models|**DONE** — mapper groups references by URL with deterministic ordering.|
|
||||
|NormalizedVersions emission|BE-Conn-Adobe|Models|**DONE** (2025-10-11) — EVR-like version metadata now projects `normalizedVersions` with `adobe:<product>:<platform>` notes; regression fixtures refreshed to assert rule output.|
|
||||
Reference in New Issue
Block a user