Rename Concelier Source modules to Connector
This commit is contained in:
		
							
								
								
									
										39
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										39
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,39 @@ | ||||
| # AGENTS | ||||
| ## Role | ||||
| Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS. | ||||
|  | ||||
| ## Scope | ||||
| - Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy. | ||||
| - Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks. | ||||
| - Parse advisories to extract summary, affected products/versions, mitigation, CVEs. | ||||
| - Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions). | ||||
| - Produce deterministic fixtures and regression tests. | ||||
|  | ||||
| ## Participants | ||||
| - `Source.Common` (HTTP/fetch utilities, DTO storage). | ||||
| - `Storage.Mongo` (raw/document/DTO/advisory stores, source state). | ||||
| - `Concelier.Models` (canonical structures + range primitives). | ||||
| - `Concelier.Testing` (integration fixtures/snapshots). | ||||
|  | ||||
| ## Interfaces & Contracts | ||||
| - Job kinds: `apple:fetch`, `apple:parse`, `apple:map`. | ||||
| - Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates. | ||||
| - Alias set should include Apple HT IDs and CVE IDs. | ||||
|  | ||||
| ## In/Out of scope | ||||
| In scope: | ||||
| - Security advisories covering Apple OS/app updates. | ||||
| - Range primitives capturing device/OS version ranges. | ||||
|  | ||||
| Out of scope: | ||||
| - Release notes unrelated to security. | ||||
|  | ||||
| ## Observability & Security Expectations | ||||
| - Log fetch/mapping statistics and failure details. | ||||
| - Sanitize HTML while preserving structured data tables. | ||||
| - Respect upstream rate limits; record failures with backoff. | ||||
|  | ||||
| ## Tests | ||||
| - Add `StellaOps.Concelier.Connector.Vndr.Apple.Tests` covering fetch/parse/map with fixtures. | ||||
| - Snapshot canonical advisories; support fixture regeneration via env flag. | ||||
| - Ensure deterministic ordering/time normalisation. | ||||
							
								
								
									
										439
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										439
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,439 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Net; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Concelier.Connector.Common; | ||||
| using StellaOps.Concelier.Connector.Common.Fetch; | ||||
| using StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
| using StellaOps.Concelier.Storage.Mongo; | ||||
| using StellaOps.Concelier.Storage.Mongo.Advisories; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
| using StellaOps.Concelier.Storage.Mongo.Dtos; | ||||
| using StellaOps.Concelier.Storage.Mongo.PsirtFlags; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| public sealed class AppleConnector : IFeedConnector | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|     }; | ||||
|  | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly IPsirtFlagStore _psirtFlagStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly AppleOptions _options; | ||||
|     private readonly AppleDiagnostics _diagnostics; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<AppleConnector> _logger; | ||||
|  | ||||
|     public AppleConnector( | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         IPsirtFlagStore psirtFlagStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         AppleDiagnostics diagnostics, | ||||
|         IOptions<AppleOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<AppleConnector> logger) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => VndrAppleConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase); | ||||
|         var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue; | ||||
|         var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill); | ||||
|  | ||||
|         SourceFetchContentResult indexResult; | ||||
|         try | ||||
|         { | ||||
|             var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!) | ||||
|             { | ||||
|                 AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" }, | ||||
|             }; | ||||
|  | ||||
|             indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _diagnostics.FetchFailure(); | ||||
|             _logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         if (!indexResult.IsSuccess || indexResult.Content is null) | ||||
|         { | ||||
|             if (indexResult.IsNotModified) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|             } | ||||
|  | ||||
|             await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!); | ||||
|         if (indexEntries.Count == 0) | ||||
|         { | ||||
|             await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var allowlist = _options.AdvisoryAllowlist; | ||||
|         var blocklist = _options.AdvisoryBlocklist; | ||||
|  | ||||
|         var ordered = indexEntries | ||||
|             .Where(entry => ShouldInclude(entry, allowlist, blocklist)) | ||||
|             .OrderBy(entry => entry.PostingDate) | ||||
|             .ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|  | ||||
|         foreach (var entry in ordered) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             if (entry.PostingDate < baseline) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (cursor.LastPosted.HasValue | ||||
|                 && entry.PostingDate <= cursor.LastPosted.Value | ||||
|                 && processedIds.Contains(entry.UpdateId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var metadata = BuildMetadata(entry); | ||||
|             var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             SourceFetchResult result; | ||||
|             try | ||||
|             { | ||||
|                 result = await _fetchService.FetchAsync( | ||||
|                     new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri) | ||||
|                     { | ||||
|                         Metadata = metadata, | ||||
|                         ETag = existing?.Etag, | ||||
|                         LastModified = existing?.LastModified, | ||||
|                         AcceptHeaders = new[] | ||||
|                         { | ||||
|                             "text/html", | ||||
|                             "application/xhtml+xml", | ||||
|                             "text/plain;q=0.5" | ||||
|                         }, | ||||
|                     }, | ||||
|                     cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 _logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri); | ||||
|                 await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             if (result.StatusCode == HttpStatusCode.NotModified) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|             } | ||||
|  | ||||
|             if (!result.IsSuccess || result.Document is null) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             _diagnostics.FetchItem(); | ||||
|  | ||||
|             pendingDocuments.Add(result.Document.Id); | ||||
|             processedIds.Add(entry.UpdateId); | ||||
|  | ||||
|             if (entry.PostingDate > maxPosted) | ||||
|             { | ||||
|                 maxPosted = entry.PostingDate; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updated = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings) | ||||
|             .WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds); | ||||
|  | ||||
|         await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remainingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             AppleDetailDto dto; | ||||
|             try | ||||
|             { | ||||
|                 var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|                 var html = System.Text.Encoding.UTF8.GetString(content); | ||||
|                 var entry = RehydrateIndexEntry(document); | ||||
|                 dto = AppleDetailParser.Parse(html, entry); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var json = JsonSerializer.Serialize(dto, SerializerOptions); | ||||
|             var payload = BsonDocument.Parse(json); | ||||
|             var validatedAt = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|             var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); | ||||
|             var dtoRecord = existingDto is null | ||||
|                 ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt) | ||||
|                 : existingDto with | ||||
|                 { | ||||
|                     Payload = payload, | ||||
|                     SchemaVersion = "apple.security.update.v1", | ||||
|                     ValidatedAt = validatedAt, | ||||
|                 }; | ||||
|  | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             remainingDocuments.Remove(documentId); | ||||
|             pendingMappings.Add(document.Id); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remainingDocuments) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); | ||||
|             if (dtoRecord is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             AppleDetailDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = JsonSerializer.Deserialize<AppleDetailDto>(dtoRecord.Payload.ToJson(), SerializerOptions) | ||||
|                     ?? throw new InvalidOperationException("Unable to deserialize Apple DTO."); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord); | ||||
|             _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); | ||||
|  | ||||
|             await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (flag is not null) | ||||
|             { | ||||
|                 await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|  | ||||
|             pendingMappings.Remove(documentId); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document) | ||||
|     { | ||||
|         var metadata = document.Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         metadata.TryGetValue("apple.articleId", out var articleId); | ||||
|         metadata.TryGetValue("apple.updateId", out var updateId); | ||||
|         metadata.TryGetValue("apple.title", out var title); | ||||
|         metadata.TryGetValue("apple.postingDate", out var postingDateRaw); | ||||
|         metadata.TryGetValue("apple.detailUri", out var detailUriRaw); | ||||
|         metadata.TryGetValue("apple.rapidResponse", out var rapidRaw); | ||||
|         metadata.TryGetValue("apple.products", out var productsJson); | ||||
|  | ||||
|         if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate)) | ||||
|         { | ||||
|             postingDate = document.FetchedAt; | ||||
|         } | ||||
|  | ||||
|         var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri) | ||||
|             ? parsedUri | ||||
|             : new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri); | ||||
|  | ||||
|         var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase); | ||||
|         var products = DeserializeProducts(productsJson); | ||||
|  | ||||
|         return new AppleIndexEntry( | ||||
|             UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId, | ||||
|             ArticleId: articleId ?? document.Uri, | ||||
|             Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update", | ||||
|             PostingDate: postingDate.ToUniversalTime(), | ||||
|             DetailUri: detailUri, | ||||
|             Products: products, | ||||
|             IsRapidSecurityResponse: rapid); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AppleIndexProduct> DeserializeProducts(string? json) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(json)) | ||||
|         { | ||||
|             return Array.Empty<AppleIndexProduct>(); | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var products = JsonSerializer.Deserialize<List<AppleIndexProduct>>(json, SerializerOptions); | ||||
|             return products is { Count: > 0 } ? products : Array.Empty<AppleIndexProduct>(); | ||||
|         } | ||||
|         catch (JsonException) | ||||
|         { | ||||
|             return Array.Empty<AppleIndexProduct>(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static Dictionary<string, string> BuildMetadata(AppleIndexEntry entry) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|         { | ||||
|             ["apple.articleId"] = entry.ArticleId, | ||||
|             ["apple.updateId"] = entry.UpdateId, | ||||
|             ["apple.title"] = entry.Title, | ||||
|             ["apple.postingDate"] = entry.PostingDate.ToString("O"), | ||||
|             ["apple.detailUri"] = entry.DetailUri.ToString(), | ||||
|             ["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false", | ||||
|             ["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions), | ||||
|         }; | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
|  | ||||
|     private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection<string> allowlist, IReadOnlyCollection<string> blocklist) | ||||
|     { | ||||
|         if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         return true; | ||||
|     } | ||||
|  | ||||
|     private async Task<AppleCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBson(); | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "concelier:sources:apple"; | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddAppleConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         services.AddTransient<AppleFetchJob>(); | ||||
|         services.AddTransient<AppleParseJob>(); | ||||
|         services.AddTransient<AppleMapJob>(); | ||||
|  | ||||
|         services.PostConfigure<JobSchedulerOptions>(options => | ||||
|         { | ||||
|             EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob)); | ||||
|             EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob)); | ||||
|             EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob)); | ||||
|         }); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
|  | ||||
|     private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) | ||||
|     { | ||||
|         if (options.Definitions.ContainsKey(kind)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         options.Definitions[kind] = new JobDefinition( | ||||
|             kind, | ||||
|             jobType, | ||||
|             options.DefaultTimeout, | ||||
|             options.DefaultLeaseDuration, | ||||
|             CronExpression: null, | ||||
|             Enabled: true); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										101
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										101
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,101 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.ComponentModel.DataAnnotations; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| public sealed class AppleOptions : IValidatableObject | ||||
| { | ||||
|     public const string HttpClientName = "concelier-vndr-apple"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service). | ||||
|     /// </summary> | ||||
|     public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required. | ||||
|     /// </summary> | ||||
|     public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us". | ||||
|     /// </summary> | ||||
|     public string LocaleSegment { get; set; } = "en-us"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Maximum advisories to fetch per run; defaults to 50. | ||||
|     /// </summary> | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 50; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Sliding backfill window for initial sync (defaults to 90 days). | ||||
|     /// </summary> | ||||
|     public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Tolerance added to the modified timestamp comparisons during resume. | ||||
|     /// </summary> | ||||
|     public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional allowlist of HT identifiers to include; empty means include all. | ||||
|     /// </summary> | ||||
|     public HashSet<string> AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed). | ||||
|     /// </summary> | ||||
|     public HashSet<string> AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|     public IEnumerable<ValidationResult> Validate(ValidationContext validationContext) | ||||
|     { | ||||
|         if (SoftwareLookupUri is null) | ||||
|         { | ||||
|             yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) }); | ||||
|         } | ||||
|         else if (!SoftwareLookupUri.IsAbsoluteUri) | ||||
|         { | ||||
|             yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) }); | ||||
|         } | ||||
|  | ||||
|         if (AdvisoryBaseUri is null) | ||||
|         { | ||||
|             yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) }); | ||||
|         } | ||||
|         else if (!AdvisoryBaseUri.IsAbsoluteUri) | ||||
|         { | ||||
|             yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) }); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(LocaleSegment)) | ||||
|         { | ||||
|             yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) }); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0) | ||||
|         { | ||||
|             yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) }); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfill <= TimeSpan.Zero) | ||||
|         { | ||||
|             yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) }); | ||||
|         } | ||||
|  | ||||
|         if (ModifiedTolerance < TimeSpan.Zero) | ||||
|         { | ||||
|             yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) }); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         var context = new ValidationContext(this); | ||||
|         var results = new List<ValidationResult>(); | ||||
|         if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true)) | ||||
|         { | ||||
|             throw new ValidationException(string.Join("; ", results)); | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,44 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Common.Http; | ||||
| using StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| public static class AppleServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action<AppleOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<AppleOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static opts => opts.Validate()) | ||||
|             .ValidateOnStart(); | ||||
|  | ||||
|         services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<AppleOptions>>().Value; | ||||
|             clientOptions.Timeout = TimeSpan.FromSeconds(30); | ||||
|             clientOptions.UserAgent = "StellaOps.Concelier.Apple/1.0"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             if (options.SoftwareLookupUri is not null) | ||||
|             { | ||||
|                 clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host); | ||||
|             } | ||||
|  | ||||
|             if (options.AdvisoryBaseUri is not null) | ||||
|             { | ||||
|                 clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); | ||||
|             } | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<TimeProvider>(_ => TimeProvider.System); | ||||
|         services.AddSingleton<AppleDiagnostics>(); | ||||
|         services.AddTransient<AppleConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,114 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| internal sealed record AppleCursor( | ||||
|     DateTimeOffset? LastPosted, | ||||
|     IReadOnlyCollection<string> ProcessedIds, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>(); | ||||
|     private static readonly IReadOnlyCollection<string> EmptyStringCollection = Array.Empty<string>(); | ||||
|  | ||||
|     public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection); | ||||
|  | ||||
|     public BsonDocument ToBson() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastPosted.HasValue) | ||||
|         { | ||||
|             document["lastPosted"] = LastPosted.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (ProcessedIds.Count > 0) | ||||
|         { | ||||
|             document["processedIds"] = new BsonArray(ProcessedIds); | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static AppleCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue) | ||||
|             ? ParseDate(lastPostedValue) | ||||
|             : null; | ||||
|  | ||||
|         var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray | ||||
|             ? processedArray.OfType<BsonValue>() | ||||
|                 .Where(static value => value.BsonType == BsonType.String) | ||||
|                 .Select(static value => value.AsString.Trim()) | ||||
|                 .Where(static value => value.Length > 0) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray() | ||||
|             : EmptyStringCollection; | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|  | ||||
|         return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings); | ||||
|     } | ||||
|  | ||||
|     public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable<string>? processedIds = null) | ||||
|     { | ||||
|         var ids = processedIds is null | ||||
|             ? ProcessedIds | ||||
|             : processedIds.Where(static id => !string.IsNullOrWhiteSpace(id)) | ||||
|                 .Select(static id => id.Trim()) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray(); | ||||
|  | ||||
|         return this with | ||||
|         { | ||||
|             LastPosted = timestamp.ToUniversalTime(), | ||||
|             ProcessedIds = ids, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     public AppleCursor WithPendingDocuments(IEnumerable<Guid>? ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; | ||||
|  | ||||
|     public AppleCursor WithPendingMappings(IEnumerable<Guid>? ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key) | ||||
|     { | ||||
|         if (!document.TryGetValue(key, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidCollection; | ||||
|         } | ||||
|  | ||||
|         var results = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 results.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(BsonValue value) | ||||
|         => value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
| } | ||||
| @@ -0,0 +1,78 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| internal sealed record AppleDetailDto( | ||||
|     string AdvisoryId, | ||||
|     string ArticleId, | ||||
|     string Title, | ||||
|     string Summary, | ||||
|     DateTimeOffset Published, | ||||
|     DateTimeOffset? Updated, | ||||
|     IReadOnlyList<string> CveIds, | ||||
|     IReadOnlyList<AppleAffectedProductDto> Affected, | ||||
|     IReadOnlyList<AppleReferenceDto> References, | ||||
|     bool RapidSecurityResponse); | ||||
|  | ||||
| internal sealed record AppleAffectedProductDto( | ||||
|     string Platform, | ||||
|     string Name, | ||||
|     string Version, | ||||
|     string Build); | ||||
|  | ||||
| internal sealed record AppleReferenceDto( | ||||
|     string Url, | ||||
|     string? Title, | ||||
|     string? Kind); | ||||
|  | ||||
| internal static class AppleDetailDtoExtensions | ||||
| { | ||||
|     public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable<AppleIndexProduct> products) | ||||
|     { | ||||
|         if (dto.Affected.Count > 0) | ||||
|         { | ||||
|             return dto; | ||||
|         } | ||||
|  | ||||
|         var fallback = products | ||||
|             .Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build)) | ||||
|             .Select(static product => new AppleAffectedProductDto( | ||||
|                 product.Platform, | ||||
|                 product.Name, | ||||
|                 product.Version, | ||||
|                 product.Build)) | ||||
|             .OrderBy(static product => NormalizeSortKey(product.Platform)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Name)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Version)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Build)) | ||||
|             .ToArray(); | ||||
|  | ||||
|         return fallback.Length == 0 ? dto : dto with { Affected = fallback }; | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeSortKey(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         var span = value.AsSpan(); | ||||
|         var buffer = new char[span.Length]; | ||||
|         var index = 0; | ||||
|  | ||||
|         foreach (var ch in span) | ||||
|         { | ||||
|             if (char.IsWhiteSpace(ch)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             buffer[index++] = char.ToUpperInvariant(ch); | ||||
|         } | ||||
|  | ||||
|         return new string(buffer, 0, index); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,460 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text.RegularExpressions; | ||||
| using AngleSharp.Dom; | ||||
| using AngleSharp.Html.Dom; | ||||
| using AngleSharp.Html.Parser; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| internal static class AppleDetailParser | ||||
| { | ||||
|     private static readonly HtmlParser Parser = new(); | ||||
|     private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase); | ||||
|  | ||||
|     public static AppleDetailDto Parse(string html, AppleIndexEntry entry) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(html)) | ||||
|         { | ||||
|             throw new ArgumentException("HTML content must not be empty.", nameof(html)); | ||||
|         } | ||||
|  | ||||
|         var document = Parser.ParseDocument(html); | ||||
|         var title = ResolveTitle(document, entry.Title); | ||||
|         var summary = ResolveSummary(document); | ||||
|         var (published, updated) = ResolveTimestamps(document, entry.PostingDate); | ||||
|         var cves = ExtractCves(document); | ||||
|         var affected = NormalizeAffectedProducts(ExtractProducts(document)); | ||||
|         var references = ExtractReferences(document, entry.DetailUri); | ||||
|  | ||||
|         var dto = new AppleDetailDto( | ||||
|             entry.ArticleId, | ||||
|             entry.ArticleId, | ||||
|             title, | ||||
|             summary, | ||||
|             published, | ||||
|             updated, | ||||
|             cves, | ||||
|             affected, | ||||
|             references, | ||||
|             entry.IsRapidSecurityResponse); | ||||
|  | ||||
|         return dto.WithAffectedFallback(entry.Products); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AppleAffectedProductDto> NormalizeAffectedProducts(IReadOnlyList<AppleAffectedProductDto> affected) | ||||
|     { | ||||
|         if (affected.Count <= 1) | ||||
|         { | ||||
|             return affected; | ||||
|         } | ||||
|  | ||||
|         return affected | ||||
|             .OrderBy(static product => NormalizeSortKey(product.Platform)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Name)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Version)) | ||||
|             .ThenBy(static product => NormalizeSortKey(product.Build)) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeSortKey(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         var span = value.AsSpan(); | ||||
|         var buffer = new char[span.Length]; | ||||
|         var index = 0; | ||||
|  | ||||
|         foreach (var ch in span) | ||||
|         { | ||||
|             if (char.IsWhiteSpace(ch)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             buffer[index++] = char.ToUpperInvariant(ch); | ||||
|         } | ||||
|  | ||||
|         return new string(buffer, 0, index); | ||||
|     } | ||||
|  | ||||
|     private static string ResolveTitle(IHtmlDocument document, string fallback) | ||||
|     { | ||||
|         var title = document.QuerySelector("[data-testid='update-title']")?.TextContent | ||||
|             ?? document.QuerySelector("h1, h2")?.TextContent | ||||
|             ?? document.Title; | ||||
|  | ||||
|         title = title?.Trim(); | ||||
|         return string.IsNullOrEmpty(title) ? fallback : title; | ||||
|     } | ||||
|  | ||||
|     private static string ResolveSummary(IHtmlDocument document) | ||||
|     { | ||||
|         var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent | ||||
|             ?? document.QuerySelector("meta[name='description']")?.GetAttribute("content") | ||||
|             ?? document.QuerySelector("p")?.TextContent | ||||
|             ?? string.Empty; | ||||
|  | ||||
|         return CleanWhitespace(summary); | ||||
|     } | ||||
|  | ||||
|     private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback) | ||||
|     { | ||||
|         DateTimeOffset published = postingFallback; | ||||
|         DateTimeOffset? updated = null; | ||||
|  | ||||
|         foreach (var time in document.QuerySelectorAll("time")) | ||||
|         { | ||||
|             var raw = time.GetAttribute("datetime") ?? time.TextContent; | ||||
|             if (string.IsNullOrWhiteSpace(raw)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!DateTimeOffset.TryParse(raw, out var parsed)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             parsed = parsed.ToUniversalTime(); | ||||
|  | ||||
|             var itemProp = time.GetAttribute("itemprop") ?? string.Empty; | ||||
|             var dataTestId = time.GetAttribute("data-testid") ?? string.Empty; | ||||
|  | ||||
|             if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase) | ||||
|                 || dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 published = parsed; | ||||
|             } | ||||
|             else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase) | ||||
|                 || dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 updated = parsed; | ||||
|             } | ||||
|             else if (updated is null && parsed > published) | ||||
|             { | ||||
|                 updated = parsed; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return (published, updated); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> ExtractCves(IHtmlDocument document) | ||||
|     { | ||||
|         var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var node in document.All) | ||||
|         { | ||||
|             if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var text = node.TextContent; | ||||
|             if (string.IsNullOrWhiteSpace(text)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             foreach (Match match in CveRegex.Matches(text)) | ||||
|             { | ||||
|                 if (match.Success) | ||||
|                 { | ||||
|                     set.Add(match.Value.ToUpperInvariant()); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (set.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<string>(); | ||||
|         } | ||||
|  | ||||
|         var list = set.ToList(); | ||||
|         list.Sort(StringComparer.OrdinalIgnoreCase); | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AppleAffectedProductDto> ExtractProducts(IHtmlDocument document) | ||||
|     { | ||||
|         var rows = new List<AppleAffectedProductDto>(); | ||||
|  | ||||
|         foreach (var element in document.QuerySelectorAll("[data-testid='product-row']")) | ||||
|         { | ||||
|             var platform = element.GetAttribute("data-platform") ?? string.Empty; | ||||
|             var name = element.GetAttribute("data-product") ?? platform; | ||||
|             var version = element.GetAttribute("data-version") ?? string.Empty; | ||||
|             var build = element.GetAttribute("data-build") ?? string.Empty; | ||||
|  | ||||
|             if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow) | ||||
|             { | ||||
|                 var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); | ||||
|                 if (cells.Length >= 1) | ||||
|                 { | ||||
|                     name = cells[0]; | ||||
|                 } | ||||
|  | ||||
|                 if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version)) | ||||
|                 { | ||||
|                     version = cells[1]; | ||||
|                 } | ||||
|  | ||||
|                 if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build)) | ||||
|                 { | ||||
|                     build = cells[2]; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             if (string.IsNullOrWhiteSpace(name)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             rows.Add(new AppleAffectedProductDto(platform, name, version, build)); | ||||
|         } | ||||
|  | ||||
|         if (rows.Count > 0) | ||||
|         { | ||||
|             return rows; | ||||
|         } | ||||
|  | ||||
|         // fallback for generic tables without data attributes | ||||
|         foreach (var table in document.QuerySelectorAll("table")) | ||||
|         { | ||||
|             var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray(); | ||||
|             if (headers.Length == 0) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase) | ||||
|                 || header.Contains("device", StringComparison.OrdinalIgnoreCase)); | ||||
|             var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase)); | ||||
|             var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase) | ||||
|                 || header.Contains("release", StringComparison.OrdinalIgnoreCase)); | ||||
|  | ||||
|             if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             foreach (var row in table.QuerySelectorAll("tr")) | ||||
|             { | ||||
|                 var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); | ||||
|                 if (cells.Length == 0) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0]; | ||||
|                 string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty; | ||||
|                 string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty; | ||||
|  | ||||
|                 if (string.IsNullOrWhiteSpace(name)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build)); | ||||
|             } | ||||
|  | ||||
|             if (rows.Count > 0) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return rows.Count == 0 ? Array.Empty<AppleAffectedProductDto>() : rows; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AppleReferenceDto> ExtractReferences(IHtmlDocument document, Uri detailUri) | ||||
|     { | ||||
|         var scope = document.QuerySelector("article") | ||||
|             ?? document.QuerySelector("main") | ||||
|             ?? (IElement?)document.Body; | ||||
|  | ||||
|         if (scope is null) | ||||
|         { | ||||
|             return Array.Empty<AppleReferenceDto>(); | ||||
|         } | ||||
|  | ||||
|         var anchors = scope.QuerySelectorAll("a[href]"); | ||||
|         if (anchors.Length == 0) | ||||
|         { | ||||
|             return Array.Empty<AppleReferenceDto>(); | ||||
|         } | ||||
|  | ||||
|         var references = new List<AppleReferenceDto>(anchors.Length); | ||||
|         var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         foreach (var element in anchors) | ||||
|         { | ||||
|             if (element is not IHtmlAnchorElement anchor) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (anchor.HasAttribute("data-globalnav-item-name") | ||||
|                 || anchor.HasAttribute("data-analytics-title")) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var href = anchor.GetAttribute("href"); | ||||
|             if (string.IsNullOrWhiteSpace(href)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!Uri.TryCreate(detailUri, href, out var uri)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!IsRelevantReference(uri)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var normalized = uri.ToString(); | ||||
|             if (!seen.Add(normalized)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var title = CleanWhitespace(anchor.TextContent); | ||||
|             var kind = ResolveReferenceKind(uri); | ||||
|             references.Add(new AppleReferenceDto(normalized, title, kind)); | ||||
|         } | ||||
|  | ||||
|         if (references.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AppleReferenceDto>(); | ||||
|         } | ||||
|  | ||||
|         references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); | ||||
|         return references; | ||||
|     } | ||||
|  | ||||
|     private static bool IsRelevantReference(Uri uri) | ||||
|     { | ||||
|         if (!uri.IsAbsoluteUri) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase) | ||||
|             && !string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(uri.Host)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             if (!uri.AbsolutePath.Contains("/support/", StringComparison.OrdinalIgnoreCase) | ||||
|                 && !uri.AbsolutePath.Contains("/security", StringComparison.OrdinalIgnoreCase) | ||||
|                 && !uri.AbsolutePath.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 return false; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (uri.Host.EndsWith(".apple.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             var supported = | ||||
|                 uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) | ||||
|                 || uri.Host.StartsWith("developer.", StringComparison.OrdinalIgnoreCase) | ||||
|                 || uri.Host.StartsWith("download.", StringComparison.OrdinalIgnoreCase) | ||||
|                 || uri.Host.StartsWith("updates.", StringComparison.OrdinalIgnoreCase) | ||||
|                 || uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase); | ||||
|  | ||||
|             if (!supported) | ||||
|             { | ||||
|                 return false; | ||||
|             } | ||||
|  | ||||
|             if (uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) | ||||
|                 && uri.AbsolutePath == "/" | ||||
|                 && (string.IsNullOrEmpty(uri.Query) | ||||
|                     || uri.Query.Contains("cid=", StringComparison.OrdinalIgnoreCase))) | ||||
|             { | ||||
|                 return false; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return true; | ||||
|     } | ||||
|  | ||||
|     private static string? ResolveReferenceKind(Uri uri) | ||||
|     { | ||||
|         if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 return "download"; | ||||
|             } | ||||
|  | ||||
|             if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 return "document"; | ||||
|             } | ||||
|  | ||||
|             return "advisory"; | ||||
|         } | ||||
|  | ||||
|         if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "nvd"; | ||||
|         } | ||||
|  | ||||
|         if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "kb"; | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static string CleanWhitespace(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         var span = value.AsSpan(); | ||||
|         var buffer = new char[span.Length]; | ||||
|         var index = 0; | ||||
|         var previousWhitespace = false; | ||||
|  | ||||
|         foreach (var ch in span) | ||||
|         { | ||||
|             if (char.IsWhiteSpace(ch)) | ||||
|             { | ||||
|                 if (previousWhitespace) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 buffer[index++] = ' '; | ||||
|                 previousWhitespace = true; | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 buffer[index++] = ch; | ||||
|                 previousWhitespace = false; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return new string(buffer, 0, index).Trim(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,62 @@ | ||||
| using System; | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| public sealed class AppleDiagnostics : IDisposable | ||||
| { | ||||
|     public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Apple"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _fetchItems; | ||||
|     private readonly Counter<long> _fetchFailures; | ||||
|     private readonly Counter<long> _fetchUnchanged; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Histogram<long> _mapAffected; | ||||
|  | ||||
|     public AppleDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _fetchItems = _meter.CreateCounter<long>( | ||||
|             name: "apple.fetch.items", | ||||
|             unit: "documents", | ||||
|             description: "Number of Apple advisories fetched."); | ||||
|         _fetchFailures = _meter.CreateCounter<long>( | ||||
|             name: "apple.fetch.failures", | ||||
|             unit: "operations", | ||||
|             description: "Number of Apple fetch failures."); | ||||
|         _fetchUnchanged = _meter.CreateCounter<long>( | ||||
|             name: "apple.fetch.unchanged", | ||||
|             unit: "documents", | ||||
|             description: "Number of Apple advisories skipped due to 304 responses."); | ||||
|         _parseFailures = _meter.CreateCounter<long>( | ||||
|             name: "apple.parse.failures", | ||||
|             unit: "documents", | ||||
|             description: "Number of Apple documents that failed to parse."); | ||||
|         _mapAffected = _meter.CreateHistogram<long>( | ||||
|             name: "apple.map.affected.count", | ||||
|             unit: "packages", | ||||
|             description: "Distribution of affected package counts emitted per Apple advisory."); | ||||
|     } | ||||
|  | ||||
|     public Meter Meter => _meter; | ||||
|  | ||||
|     public void FetchItem() => _fetchItems.Add(1); | ||||
|  | ||||
|     public void FetchFailure() => _fetchFailures.Add(1); | ||||
|  | ||||
|     public void FetchUnchanged() => _fetchUnchanged.Add(1); | ||||
|  | ||||
|     public void ParseFailure() => _parseFailures.Add(1); | ||||
|  | ||||
|     public void MapAffectedCount(int count) | ||||
|     { | ||||
|         if (count >= 0) | ||||
|         { | ||||
|             _mapAffected.Record(count); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
| } | ||||
| @@ -0,0 +1,144 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| internal sealed record AppleIndexEntry( | ||||
|     string UpdateId, | ||||
|     string ArticleId, | ||||
|     string Title, | ||||
|     DateTimeOffset PostingDate, | ||||
|     Uri DetailUri, | ||||
|     IReadOnlyList<AppleIndexProduct> Products, | ||||
|     bool IsRapidSecurityResponse); | ||||
|  | ||||
| internal sealed record AppleIndexProduct( | ||||
|     string Platform, | ||||
|     string Name, | ||||
|     string Version, | ||||
|     string Build); | ||||
|  | ||||
| internal static class AppleIndexParser | ||||
| { | ||||
|     private sealed record AppleIndexDocument( | ||||
|         [property: JsonPropertyName("updates")] IReadOnlyList<AppleIndexEntryDto>? Updates); | ||||
|  | ||||
|     private sealed record AppleIndexEntryDto( | ||||
|         [property: JsonPropertyName("id")] string? Id, | ||||
|         [property: JsonPropertyName("articleId")] string? ArticleId, | ||||
|         [property: JsonPropertyName("title")] string? Title, | ||||
|         [property: JsonPropertyName("postingDate")] string? PostingDate, | ||||
|         [property: JsonPropertyName("detailUrl")] string? DetailUrl, | ||||
|         [property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse, | ||||
|         [property: JsonPropertyName("products")] IReadOnlyList<AppleIndexProductDto>? Products); | ||||
|  | ||||
|     private sealed record AppleIndexProductDto( | ||||
|         [property: JsonPropertyName("platform")] string? Platform, | ||||
|         [property: JsonPropertyName("name")] string? Name, | ||||
|         [property: JsonPropertyName("version")] string? Version, | ||||
|         [property: JsonPropertyName("build")] string? Build); | ||||
|  | ||||
|     public static IReadOnlyList<AppleIndexEntry> Parse(ReadOnlySpan<byte> payload, Uri baseUri) | ||||
|     { | ||||
|         if (payload.IsEmpty) | ||||
|         { | ||||
|             return Array.Empty<AppleIndexEntry>(); | ||||
|         } | ||||
|  | ||||
|         var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) | ||||
|         { | ||||
|             PropertyNameCaseInsensitive = true, | ||||
|         }; | ||||
|  | ||||
|         AppleIndexDocument? document; | ||||
|         try | ||||
|         { | ||||
|             document = JsonSerializer.Deserialize<AppleIndexDocument>(payload, options); | ||||
|         } | ||||
|         catch (JsonException) | ||||
|         { | ||||
|             return Array.Empty<AppleIndexEntry>(); | ||||
|         } | ||||
|  | ||||
|         if (document?.Updates is null || document.Updates.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AppleIndexEntry>(); | ||||
|         } | ||||
|  | ||||
|         var entries = new List<AppleIndexEntry>(document.Updates.Count); | ||||
|         foreach (var dto in document.Updates) | ||||
|         { | ||||
|             if (dto is null) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id; | ||||
|             if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!TryResolveDetailUri(dto, baseUri, out var detailUri)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var products = dto.Products?.Select(static product => new AppleIndexProduct( | ||||
|                     product.Platform ?? string.Empty, | ||||
|                     product.Name ?? product.Platform ?? string.Empty, | ||||
|                     product.Version ?? string.Empty, | ||||
|                     product.Build ?? string.Empty)) | ||||
|                 .ToArray() ?? Array.Empty<AppleIndexProduct>(); | ||||
|  | ||||
|             entries.Add(new AppleIndexEntry( | ||||
|                 id.Trim(), | ||||
|                 dto.ArticleId!.Trim(), | ||||
|                 dto.Title!.Trim(), | ||||
|                 postingDate.ToUniversalTime(), | ||||
|                 detailUri, | ||||
|                 products, | ||||
|                 dto.RapidSecurityResponse ?? false)); | ||||
|         } | ||||
|  | ||||
|         return entries.Count == 0 ? Array.Empty<AppleIndexEntry>() : entries; | ||||
|     } | ||||
|  | ||||
|     private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri) | ||||
|     { | ||||
|         if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(dto.ArticleId)) | ||||
|         { | ||||
|             uri = default!; | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var article = dto.ArticleId.Trim(); | ||||
|         if (article.Length == 0) | ||||
|         { | ||||
|             uri = default!; | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var combined = new Uri(baseUri, article); | ||||
|         uri = combined; | ||||
|         return true; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,281 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Connector.Common; | ||||
| using StellaOps.Concelier.Connector.Common.Packages; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
| using StellaOps.Concelier.Storage.Mongo.Dtos; | ||||
| using StellaOps.Concelier.Storage.Mongo.PsirtFlags; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; | ||||
|  | ||||
| internal static class AppleMapper | ||||
| { | ||||
|     public static (Advisory Advisory, PsirtFlagRecord? Flag) Map( | ||||
|         AppleDetailDto dto, | ||||
|         DocumentRecord document, | ||||
|         DtoRecord dtoRecord) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|         ArgumentNullException.ThrowIfNull(dtoRecord); | ||||
|  | ||||
|         var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); | ||||
|  | ||||
|         var fetchProvenance = new AdvisoryProvenance( | ||||
|             VndrAppleConnectorPlugin.SourceName, | ||||
|             "document", | ||||
|             document.Uri, | ||||
|             document.FetchedAt.ToUniversalTime()); | ||||
|  | ||||
|         var mapProvenance = new AdvisoryProvenance( | ||||
|             VndrAppleConnectorPlugin.SourceName, | ||||
|             "map", | ||||
|             dto.AdvisoryId, | ||||
|             recordedAt); | ||||
|  | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var affected = BuildAffected(dto, recordedAt); | ||||
|  | ||||
|         var advisory = new Advisory( | ||||
|             advisoryKey: dto.AdvisoryId, | ||||
|             title: dto.Title, | ||||
|             summary: dto.Summary, | ||||
|             language: "en", | ||||
|             published: dto.Published.ToUniversalTime(), | ||||
|             modified: dto.Updated?.ToUniversalTime(), | ||||
|             severity: null, | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             references: references, | ||||
|             affectedPackages: affected, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { fetchProvenance, mapProvenance }); | ||||
|  | ||||
|         PsirtFlagRecord? flag = dto.RapidSecurityResponse | ||||
|             ? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt) | ||||
|             : null; | ||||
|  | ||||
|         return (advisory, flag); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> BuildAliases(AppleDetailDto dto) | ||||
|     { | ||||
|         var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             dto.AdvisoryId, | ||||
|             dto.ArticleId, | ||||
|         }; | ||||
|  | ||||
|         foreach (var cve in dto.CveIds) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(cve)) | ||||
|             { | ||||
|                 set.Add(cve.Trim()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var aliases = set.ToList(); | ||||
|         aliases.Sort(StringComparer.OrdinalIgnoreCase); | ||||
|         return aliases; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.References.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryReference>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<AdvisoryReference>(dto.References.Count); | ||||
|         foreach (var reference in dto.References) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(reference.Url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var provenance = new AdvisoryProvenance( | ||||
|                     VndrAppleConnectorPlugin.SourceName, | ||||
|                     "reference", | ||||
|                     reference.Url, | ||||
|                     recordedAt); | ||||
|  | ||||
|                 list.Add(new AdvisoryReference( | ||||
|                     url: reference.Url, | ||||
|                     kind: reference.Kind, | ||||
|                     sourceTag: null, | ||||
|                     summary: reference.Title, | ||||
|                     provenance: provenance)); | ||||
|             } | ||||
|             catch (ArgumentException) | ||||
|             { | ||||
|                 // ignore invalid URLs | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (list.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryReference>(); | ||||
|         } | ||||
|  | ||||
|         list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Affected.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Affected.Count); | ||||
|         foreach (var product in dto.Affected) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(product.Name)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var provenance = new[] | ||||
|             { | ||||
|                 new AdvisoryProvenance( | ||||
|                     VndrAppleConnectorPlugin.SourceName, | ||||
|                     "affected", | ||||
|                     product.Name, | ||||
|                     recordedAt), | ||||
|             }; | ||||
|  | ||||
|             var ranges = BuildRanges(product, recordedAt); | ||||
|             var normalizedVersions = BuildNormalizedVersions(product, ranges); | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 type: AffectedPackageTypes.Vendor, | ||||
|                 identifier: product.Name, | ||||
|                 platform: product.Platform, | ||||
|                 versionRanges: ranges, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: provenance, | ||||
|                 normalizedVersions: normalizedVersions)); | ||||
|         } | ||||
|  | ||||
|         return packages.Count == 0 ? Array.Empty<AffectedPackage>() : packages; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedVersionRange> BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build)) | ||||
|         { | ||||
|             return Array.Empty<AffectedVersionRange>(); | ||||
|         } | ||||
|  | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             VndrAppleConnectorPlugin.SourceName, | ||||
|             "range", | ||||
|             product.Name, | ||||
|             recordedAt); | ||||
|  | ||||
|         var extensions = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         if (!string.IsNullOrWhiteSpace(product.Version)) | ||||
|         { | ||||
|             extensions["apple.version.raw"] = product.Version; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(product.Build)) | ||||
|         { | ||||
|             extensions["apple.build"] = product.Build; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(product.Platform)) | ||||
|         { | ||||
|             extensions["apple.platform"] = product.Platform; | ||||
|         } | ||||
|  | ||||
|         var primitives = extensions.Count == 0 | ||||
|             ? null | ||||
|             : new RangePrimitives( | ||||
|                 SemVer: TryCreateSemVerPrimitive(product.Version), | ||||
|                 Nevra: null, | ||||
|                 Evr: null, | ||||
|                 VendorExtensions: extensions); | ||||
|  | ||||
|         var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion) | ||||
|             ? normalizedVersion | ||||
|             : product.Version; | ||||
|  | ||||
|         return new[] | ||||
|         { | ||||
|             new AffectedVersionRange( | ||||
|                 rangeKind: "vendor", | ||||
|                 introducedVersion: null, | ||||
|                 fixedVersion: sanitizedVersion, | ||||
|                 lastAffectedVersion: null, | ||||
|                 rangeExpression: product.Version, | ||||
|                 provenance: provenance, | ||||
|                 primitives: primitives), | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(version)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         // treat as fixed version, unknown introduced/last affected | ||||
|         return new SemVerPrimitive( | ||||
|             Introduced: null, | ||||
|             IntroducedInclusive: true, | ||||
|             Fixed: normalized, | ||||
|             FixedInclusive: true, | ||||
|             LastAffected: null, | ||||
|             LastAffectedInclusive: true, | ||||
|             ConstraintExpression: null); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions( | ||||
|         AppleAffectedProductDto product, | ||||
|         IReadOnlyList<AffectedVersionRange> ranges) | ||||
|     { | ||||
|         if (ranges.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<NormalizedVersionRule>(); | ||||
|         } | ||||
|  | ||||
|         var segments = new List<string>(); | ||||
|         if (!string.IsNullOrWhiteSpace(product.Platform)) | ||||
|         { | ||||
|             segments.Add(product.Platform.Trim()); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(product.Name)) | ||||
|         { | ||||
|             segments.Add(product.Name.Trim()); | ||||
|         } | ||||
|  | ||||
|         var note = segments.Count == 0 ? null : $"apple:{string.Join(':', segments)}"; | ||||
|  | ||||
|         var rules = new List<NormalizedVersionRule>(ranges.Count); | ||||
|         foreach (var range in ranges) | ||||
|         { | ||||
|             var rule = range.ToNormalizedVersionRule(note); | ||||
|             if (rule is not null) | ||||
|             { | ||||
|                 rules.Add(rule); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray(); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										46
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										46
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,46 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| internal static class AppleJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:vndr-apple:fetch"; | ||||
|     public const string Parse = "source:vndr-apple:parse"; | ||||
|     public const string Map = "source:vndr-apple:map"; | ||||
| } | ||||
|  | ||||
| internal sealed class AppleFetchJob : IJob | ||||
| { | ||||
|     private readonly AppleConnector _connector; | ||||
|  | ||||
|     public AppleFetchJob(AppleConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class AppleParseJob : IJob | ||||
| { | ||||
|     private readonly AppleConnector _connector; | ||||
|  | ||||
|     public AppleParseJob(AppleConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.ParseAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class AppleMapJob : IJob | ||||
| { | ||||
|     private readonly AppleConnector _connector; | ||||
|  | ||||
|     public AppleMapJob(AppleConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.MapAsync(context.Services, cancellationToken); | ||||
| } | ||||
| @@ -0,0 +1,3 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Apple.Tests")] | ||||
							
								
								
									
										49
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/README.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										49
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/README.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,49 @@ | ||||
| # Apple Security Updates Connector | ||||
|  | ||||
| ## Feed contract | ||||
|  | ||||
| The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes: | ||||
|  | ||||
| - `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`) | ||||
| - Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline` | ||||
| - Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS) | ||||
| - Metadata for download packages, release notes, and signing assets | ||||
|  | ||||
| The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8 | ||||
|  | ||||
| Apple’s new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists: | ||||
|  | ||||
| - CVE identifiers with Apple’s internal tracking IDs | ||||
| - Product version/build applicability tables | ||||
| - Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6 | ||||
|  | ||||
| Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `<section data-component="security-update">` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries. | ||||
|  | ||||
| ## Change detection strategy | ||||
|  | ||||
| 1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches. | ||||
| 2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`). | ||||
| 3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only. | ||||
|  | ||||
| Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3 | ||||
|  | ||||
| ## Parsing & mapping notes | ||||
|  | ||||
| - CVE lists live inside `<ul data-testid="cve-list">` items; each `<li>` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apple’s component name as the package `name` and the OS build as the range primitive seed. | ||||
| - Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`). | ||||
| - Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`. | ||||
|  | ||||
| ## Outstanding questions | ||||
|  | ||||
| - Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`. | ||||
| - Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured. | ||||
|  | ||||
| ## Fixture maintenance | ||||
|  | ||||
| Deterministic regression coverage lives in `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. When Apple publishes new advisories the fixtures must be refreshed using the provided helper scripts: | ||||
|  | ||||
| - Bash: `./scripts/update-apple-fixtures.sh` | ||||
| - PowerShell: `./scripts/update-apple-fixtures.ps1` | ||||
|  | ||||
| Both scripts set `UPDATE_APPLE_FIXTURES=1`, touch a `.update-apple-fixtures` sentinel so test runs inside WSL propagate the flag, fetch the live HT articles referenced in `AppleFixtureManager`, sanitise the HTML, and rewrite the paired `.expected.json` DTO snapshots. Always inspect the resulting diff and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the environment variable to ensure deterministic output before committing. | ||||
|  | ||||
| @@ -0,0 +1,18 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|  | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|  | ||||
							
								
								
									
										11
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										11
									
								
								src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,11 @@ | ||||
| # TASKS | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** – Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.| | ||||
| |Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** – Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.| | ||||
| |Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** – AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.| | ||||
| |Canonical mapping & range primitives|BE-Conn-Apple|Models|**DONE** – Mapper now emits SemVer-derived normalizedVersions with `apple:<platform>:<product>` notes; fixtures updated to assert canonical rules while we continue tracking multi-device coverage in follow-up tasks.<br>2025-10-11 research trail: confirmed payload aligns with `[{"scheme":"semver","type":"range","min":"<build-start>","minInclusive":true,"max":"<build-end>","maxInclusive":false,"notes":"apple:ios:17.1"}]`; continue using `notes` to surface build identifiers for storage provenance.| | ||||
| |Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-12)** – Parser now scopes references to article content, sorts affected rows deterministically, and regenerated fixtures (125326/125328/106355/HT214108/HT215500) produce stable JSON + sanitizer HTML in English.| | ||||
| |Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – OpenTelemetry pipeline exports `StellaOps.Concelier.Connector.Vndr.Apple`; runbook `docs/ops/concelier-apple-operations.md` added with metrics + monitoring guidance.| | ||||
| |Live HTML regression sweep|QA|Source.Common|**DONE (2025-10-12)** – Captured latest support.apple.com articles for 125326/125328/106355/HT214108/HT215500, trimmed nav noise, and committed sanitized HTML + expected DTOs with invariant timestamps.| | ||||
| |Fixture regeneration tooling|DevEx|Testing|**DONE (2025-10-12)** – `scripts/update-apple-fixtures.(sh|ps1)` set the env flag + sentinel, forward through WSLENV, and clean up after regeneration; README references updated usage.| | ||||
| @@ -0,0 +1,24 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Apple; | ||||
|  | ||||
| public sealed class VndrAppleConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "vndr-apple"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return services.GetService<AppleConnector>() is not null; | ||||
|     } | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return services.GetRequiredService<AppleConnector>(); | ||||
|     } | ||||
| } | ||||
		Reference in New Issue
	
	Block a user