Resolve Concelier/Excititor merge conflicts
This commit is contained in:
39
src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md
Normal file
39
src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md
Normal file
@@ -0,0 +1,39 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS.
|
||||
|
||||
## Scope
|
||||
- Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy.
|
||||
- Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks.
|
||||
- Parse advisories to extract summary, affected products/versions, mitigation, CVEs.
|
||||
- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions).
|
||||
- Produce deterministic fixtures and regression tests.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP/fetch utilities, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores, source state).
|
||||
- `Concelier.Models` (canonical structures + range primitives).
|
||||
- `Concelier.Testing` (integration fixtures/snapshots).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `apple:fetch`, `apple:parse`, `apple:map`.
|
||||
- Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates.
|
||||
- Alias set should include Apple HT IDs and CVE IDs.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- Security advisories covering Apple OS/app updates.
|
||||
- Range primitives capturing device/OS version ranges.
|
||||
|
||||
Out of scope:
|
||||
- Release notes unrelated to security.
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch/mapping statistics and failure details.
|
||||
- Sanitize HTML while preserving structured data tables.
|
||||
- Respect upstream rate limits; record failures with backoff.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Vndr.Apple.Tests` covering fetch/parse/map with fixtures.
|
||||
- Snapshot canonical advisories; support fixture regeneration via env flag.
|
||||
- Ensure deterministic ordering/time normalisation.
|
||||
439
src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs
Normal file
439
src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs
Normal file
@@ -0,0 +1,439 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
public sealed class AppleConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly IPsirtFlagStore _psirtFlagStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly AppleOptions _options;
|
||||
private readonly AppleDiagnostics _diagnostics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<AppleConnector> _logger;
|
||||
|
||||
public AppleConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
IPsirtFlagStore psirtFlagStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
AppleDiagnostics diagnostics,
|
||||
IOptions<AppleOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<AppleConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => VndrAppleConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue;
|
||||
var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill);
|
||||
|
||||
SourceFetchContentResult indexResult;
|
||||
try
|
||||
{
|
||||
var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!)
|
||||
{
|
||||
AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" },
|
||||
};
|
||||
|
||||
indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!indexResult.IsSuccess || indexResult.Content is null)
|
||||
{
|
||||
if (indexResult.IsNotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
}
|
||||
|
||||
await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!);
|
||||
if (indexEntries.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var allowlist = _options.AdvisoryAllowlist;
|
||||
var blocklist = _options.AdvisoryBlocklist;
|
||||
|
||||
var ordered = indexEntries
|
||||
.Where(entry => ShouldInclude(entry, allowlist, blocklist))
|
||||
.OrderBy(entry => entry.PostingDate)
|
||||
.ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
foreach (var entry in ordered)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (entry.PostingDate < baseline)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cursor.LastPosted.HasValue
|
||||
&& entry.PostingDate <= cursor.LastPosted.Value
|
||||
&& processedIds.Contains(entry.UpdateId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = BuildMetadata(entry);
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
ETag = existing?.Etag,
|
||||
LastModified = existing?.LastModified,
|
||||
AcceptHeaders = new[]
|
||||
{
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"text/plain;q=0.5"
|
||||
},
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.StatusCode == HttpStatusCode.NotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.FetchItem();
|
||||
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
processedIds.Add(entry.UpdateId);
|
||||
|
||||
if (entry.PostingDate > maxPosted)
|
||||
{
|
||||
maxPosted = entry.PostingDate;
|
||||
}
|
||||
}
|
||||
|
||||
var updated = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds);
|
||||
|
||||
await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AppleDetailDto dto;
|
||||
try
|
||||
{
|
||||
var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
var html = System.Text.Encoding.UTF8.GetString(content);
|
||||
var entry = RehydrateIndexEntry(document);
|
||||
dto = AppleDetailParser.Parse(html, entry);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(dto, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(json);
|
||||
var validatedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
|
||||
var dtoRecord = existingDto is null
|
||||
? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt)
|
||||
: existingDto with
|
||||
{
|
||||
Payload = payload,
|
||||
SchemaVersion = "apple.security.update.v1",
|
||||
ValidatedAt = validatedAt,
|
||||
};
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
AppleDetailDto dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<AppleDetailDto>(dtoRecord.Payload.ToJson(), SerializerOptions)
|
||||
?? throw new InvalidOperationException("Unable to deserialize Apple DTO.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id);
|
||||
pendingMappings.Remove(documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord);
|
||||
_diagnostics.MapAffectedCount(advisory.AffectedPackages.Length);
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (flag is not null)
|
||||
{
|
||||
await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document)
|
||||
{
|
||||
var metadata = document.Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
metadata.TryGetValue("apple.articleId", out var articleId);
|
||||
metadata.TryGetValue("apple.updateId", out var updateId);
|
||||
metadata.TryGetValue("apple.title", out var title);
|
||||
metadata.TryGetValue("apple.postingDate", out var postingDateRaw);
|
||||
metadata.TryGetValue("apple.detailUri", out var detailUriRaw);
|
||||
metadata.TryGetValue("apple.rapidResponse", out var rapidRaw);
|
||||
metadata.TryGetValue("apple.products", out var productsJson);
|
||||
|
||||
if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate))
|
||||
{
|
||||
postingDate = document.FetchedAt;
|
||||
}
|
||||
|
||||
var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri)
|
||||
? parsedUri
|
||||
: new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri);
|
||||
|
||||
var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase);
|
||||
var products = DeserializeProducts(productsJson);
|
||||
|
||||
return new AppleIndexEntry(
|
||||
UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId,
|
||||
ArticleId: articleId ?? document.Uri,
|
||||
Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update",
|
||||
PostingDate: postingDate.ToUniversalTime(),
|
||||
DetailUri: detailUri,
|
||||
Products: products,
|
||||
IsRapidSecurityResponse: rapid);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AppleIndexProduct> DeserializeProducts(string? json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
return Array.Empty<AppleIndexProduct>();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var products = JsonSerializer.Deserialize<List<AppleIndexProduct>>(json, SerializerOptions);
|
||||
return products is { Count: > 0 } ? products : Array.Empty<AppleIndexProduct>();
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return Array.Empty<AppleIndexProduct>();
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildMetadata(AppleIndexEntry entry)
|
||||
{
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["apple.articleId"] = entry.ArticleId,
|
||||
["apple.updateId"] = entry.UpdateId,
|
||||
["apple.title"] = entry.Title,
|
||||
["apple.postingDate"] = entry.PostingDate.ToString("O"),
|
||||
["apple.detailUri"] = entry.DetailUri.ToString(),
|
||||
["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false",
|
||||
["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions),
|
||||
};
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection<string> allowlist, IReadOnlyCollection<string> blocklist)
|
||||
{
|
||||
if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<AppleCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBson();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:apple";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddAppleConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<AppleFetchJob>();
|
||||
services.AddTransient<AppleParseJob>();
|
||||
services.AddTransient<AppleMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob));
|
||||
EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob));
|
||||
EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
101
src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs
Normal file
101
src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs
Normal file
@@ -0,0 +1,101 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
public sealed class AppleOptions : IValidatableObject
|
||||
{
|
||||
public const string HttpClientName = "concelier-vndr-apple";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service).
|
||||
/// </summary>
|
||||
public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv");
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required.
|
||||
/// </summary>
|
||||
public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/");
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us".
|
||||
/// </summary>
|
||||
public string LocaleSegment { get; set; } = "en-us";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum advisories to fetch per run; defaults to 50.
|
||||
/// </summary>
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 50;
|
||||
|
||||
/// <summary>
|
||||
/// Sliding backfill window for initial sync (defaults to 90 days).
|
||||
/// </summary>
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90);
|
||||
|
||||
/// <summary>
|
||||
/// Tolerance added to the modified timestamp comparisons during resume.
|
||||
/// </summary>
|
||||
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Optional allowlist of HT identifiers to include; empty means include all.
|
||||
/// </summary>
|
||||
public HashSet<string> AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed).
|
||||
/// </summary>
|
||||
public HashSet<string> AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
|
||||
{
|
||||
if (SoftwareLookupUri is null)
|
||||
{
|
||||
yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) });
|
||||
}
|
||||
else if (!SoftwareLookupUri.IsAbsoluteUri)
|
||||
{
|
||||
yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) });
|
||||
}
|
||||
|
||||
if (AdvisoryBaseUri is null)
|
||||
{
|
||||
yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) });
|
||||
}
|
||||
else if (!AdvisoryBaseUri.IsAbsoluteUri)
|
||||
{
|
||||
yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(LocaleSegment))
|
||||
{
|
||||
yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) });
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0)
|
||||
{
|
||||
yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) });
|
||||
}
|
||||
|
||||
if (InitialBackfill <= TimeSpan.Zero)
|
||||
{
|
||||
yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) });
|
||||
}
|
||||
|
||||
if (ModifiedTolerance < TimeSpan.Zero)
|
||||
{
|
||||
yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) });
|
||||
}
|
||||
}
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
var context = new ValidationContext(this);
|
||||
var results = new List<ValidationResult>();
|
||||
if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true))
|
||||
{
|
||||
throw new ValidationException(string.Join("; ", results));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
public static class AppleServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action<AppleOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<AppleOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate())
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<AppleOptions>>().Value;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.Apple/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
if (options.SoftwareLookupUri is not null)
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host);
|
||||
}
|
||||
|
||||
if (options.AdvisoryBaseUri is not null)
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host);
|
||||
}
|
||||
});
|
||||
|
||||
services.TryAddSingleton<TimeProvider>(_ => TimeProvider.System);
|
||||
services.AddSingleton<AppleDiagnostics>();
|
||||
services.AddTransient<AppleConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
internal sealed record AppleCursor(
|
||||
DateTimeOffset? LastPosted,
|
||||
IReadOnlyCollection<string> ProcessedIds,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyStringCollection = Array.Empty<string>();
|
||||
|
||||
public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection);
|
||||
|
||||
public BsonDocument ToBson()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPosted.HasValue)
|
||||
{
|
||||
document["lastPosted"] = LastPosted.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (ProcessedIds.Count > 0)
|
||||
{
|
||||
document["processedIds"] = new BsonArray(ProcessedIds);
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static AppleCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue)
|
||||
? ParseDate(lastPostedValue)
|
||||
: null;
|
||||
|
||||
var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray
|
||||
? processedArray.OfType<BsonValue>()
|
||||
.Where(static value => value.BsonType == BsonType.String)
|
||||
.Select(static value => value.AsString.Trim())
|
||||
.Where(static value => value.Length > 0)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray()
|
||||
: EmptyStringCollection;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings);
|
||||
}
|
||||
|
||||
public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable<string>? processedIds = null)
|
||||
{
|
||||
var ids = processedIds is null
|
||||
? ProcessedIds
|
||||
: processedIds.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
return this with
|
||||
{
|
||||
LastPosted = timestamp.ToUniversalTime(),
|
||||
ProcessedIds = ids,
|
||||
};
|
||||
}
|
||||
|
||||
public AppleCursor WithPendingDocuments(IEnumerable<Guid>? ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection };
|
||||
|
||||
public AppleCursor WithPendingMappings(IEnumerable<Guid>? ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection };
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key)
|
||||
{
|
||||
if (!document.TryGetValue(key, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidCollection;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
internal sealed record AppleDetailDto(
|
||||
string AdvisoryId,
|
||||
string ArticleId,
|
||||
string Title,
|
||||
string Summary,
|
||||
DateTimeOffset Published,
|
||||
DateTimeOffset? Updated,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<AppleAffectedProductDto> Affected,
|
||||
IReadOnlyList<AppleReferenceDto> References,
|
||||
bool RapidSecurityResponse);
|
||||
|
||||
internal sealed record AppleAffectedProductDto(
|
||||
string Platform,
|
||||
string Name,
|
||||
string Version,
|
||||
string Build);
|
||||
|
||||
internal sealed record AppleReferenceDto(
|
||||
string Url,
|
||||
string? Title,
|
||||
string? Kind);
|
||||
|
||||
internal static class AppleDetailDtoExtensions
|
||||
{
|
||||
public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable<AppleIndexProduct> products)
|
||||
{
|
||||
if (dto.Affected.Count > 0)
|
||||
{
|
||||
return dto;
|
||||
}
|
||||
|
||||
var fallback = products
|
||||
.Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build))
|
||||
.Select(static product => new AppleAffectedProductDto(
|
||||
product.Platform,
|
||||
product.Name,
|
||||
product.Version,
|
||||
product.Build))
|
||||
.OrderBy(static product => NormalizeSortKey(product.Platform))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Name))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Version))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Build))
|
||||
.ToArray();
|
||||
|
||||
return fallback.Length == 0 ? dto : dto with { Affected = fallback };
|
||||
}
|
||||
|
||||
private static string NormalizeSortKey(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var span = value.AsSpan();
|
||||
var buffer = new char[span.Length];
|
||||
var index = 0;
|
||||
|
||||
foreach (var ch in span)
|
||||
{
|
||||
if (char.IsWhiteSpace(ch))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer[index++] = char.ToUpperInvariant(ch);
|
||||
}
|
||||
|
||||
return new string(buffer, 0, index);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
internal static class AppleDetailParser
|
||||
{
|
||||
private static readonly HtmlParser Parser = new();
|
||||
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
public static AppleDetailDto Parse(string html, AppleIndexEntry entry)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
throw new ArgumentException("HTML content must not be empty.", nameof(html));
|
||||
}
|
||||
|
||||
var document = Parser.ParseDocument(html);
|
||||
var title = ResolveTitle(document, entry.Title);
|
||||
var summary = ResolveSummary(document);
|
||||
var (published, updated) = ResolveTimestamps(document, entry.PostingDate);
|
||||
var cves = ExtractCves(document);
|
||||
var affected = NormalizeAffectedProducts(ExtractProducts(document));
|
||||
var references = ExtractReferences(document, entry.DetailUri);
|
||||
|
||||
var dto = new AppleDetailDto(
|
||||
entry.ArticleId,
|
||||
entry.ArticleId,
|
||||
title,
|
||||
summary,
|
||||
published,
|
||||
updated,
|
||||
cves,
|
||||
affected,
|
||||
references,
|
||||
entry.IsRapidSecurityResponse);
|
||||
|
||||
return dto.WithAffectedFallback(entry.Products);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AppleAffectedProductDto> NormalizeAffectedProducts(IReadOnlyList<AppleAffectedProductDto> affected)
|
||||
{
|
||||
if (affected.Count <= 1)
|
||||
{
|
||||
return affected;
|
||||
}
|
||||
|
||||
return affected
|
||||
.OrderBy(static product => NormalizeSortKey(product.Platform))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Name))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Version))
|
||||
.ThenBy(static product => NormalizeSortKey(product.Build))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string NormalizeSortKey(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var span = value.AsSpan();
|
||||
var buffer = new char[span.Length];
|
||||
var index = 0;
|
||||
|
||||
foreach (var ch in span)
|
||||
{
|
||||
if (char.IsWhiteSpace(ch))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer[index++] = char.ToUpperInvariant(ch);
|
||||
}
|
||||
|
||||
return new string(buffer, 0, index);
|
||||
}
|
||||
|
||||
private static string ResolveTitle(IHtmlDocument document, string fallback)
|
||||
{
|
||||
var title = document.QuerySelector("[data-testid='update-title']")?.TextContent
|
||||
?? document.QuerySelector("h1, h2")?.TextContent
|
||||
?? document.Title;
|
||||
|
||||
title = title?.Trim();
|
||||
return string.IsNullOrEmpty(title) ? fallback : title;
|
||||
}
|
||||
|
||||
private static string ResolveSummary(IHtmlDocument document)
|
||||
{
|
||||
var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent
|
||||
?? document.QuerySelector("meta[name='description']")?.GetAttribute("content")
|
||||
?? document.QuerySelector("p")?.TextContent
|
||||
?? string.Empty;
|
||||
|
||||
return CleanWhitespace(summary);
|
||||
}
|
||||
|
||||
private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback)
|
||||
{
|
||||
DateTimeOffset published = postingFallback;
|
||||
DateTimeOffset? updated = null;
|
||||
|
||||
foreach (var time in document.QuerySelectorAll("time"))
|
||||
{
|
||||
var raw = time.GetAttribute("datetime") ?? time.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(raw, out var parsed))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
parsed = parsed.ToUniversalTime();
|
||||
|
||||
var itemProp = time.GetAttribute("itemprop") ?? string.Empty;
|
||||
var dataTestId = time.GetAttribute("data-testid") ?? string.Empty;
|
||||
|
||||
if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase)
|
||||
|| dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
published = parsed;
|
||||
}
|
||||
else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase)
|
||||
|| dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
updated = parsed;
|
||||
}
|
||||
else if (updated is null && parsed > published)
|
||||
{
|
||||
updated = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
return (published, updated);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCves(IHtmlDocument document)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var node in document.All)
|
||||
{
|
||||
if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var text = node.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (set.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var list = set.ToList();
|
||||
list.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AppleAffectedProductDto> ExtractProducts(IHtmlDocument document)
|
||||
{
|
||||
var rows = new List<AppleAffectedProductDto>();
|
||||
|
||||
foreach (var element in document.QuerySelectorAll("[data-testid='product-row']"))
|
||||
{
|
||||
var platform = element.GetAttribute("data-platform") ?? string.Empty;
|
||||
var name = element.GetAttribute("data-product") ?? platform;
|
||||
var version = element.GetAttribute("data-version") ?? string.Empty;
|
||||
var build = element.GetAttribute("data-build") ?? string.Empty;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow)
|
||||
{
|
||||
var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray();
|
||||
if (cells.Length >= 1)
|
||||
{
|
||||
name = cells[0];
|
||||
}
|
||||
|
||||
if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
version = cells[1];
|
||||
}
|
||||
|
||||
if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build))
|
||||
{
|
||||
build = cells[2];
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.Add(new AppleAffectedProductDto(platform, name, version, build));
|
||||
}
|
||||
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
return rows;
|
||||
}
|
||||
|
||||
// fallback for generic tables without data attributes
|
||||
foreach (var table in document.QuerySelectorAll("table"))
|
||||
{
|
||||
var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray();
|
||||
if (headers.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase)
|
||||
|| header.Contains("device", StringComparison.OrdinalIgnoreCase));
|
||||
var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase));
|
||||
var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase)
|
||||
|| header.Contains("release", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var row in table.QuerySelectorAll("tr"))
|
||||
{
|
||||
var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray();
|
||||
if (cells.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0];
|
||||
string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty;
|
||||
string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build));
|
||||
}
|
||||
|
||||
if (rows.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return rows.Count == 0 ? Array.Empty<AppleAffectedProductDto>() : rows;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AppleReferenceDto> ExtractReferences(IHtmlDocument document, Uri detailUri)
|
||||
{
|
||||
var scope = document.QuerySelector("article")
|
||||
?? document.QuerySelector("main")
|
||||
?? (IElement?)document.Body;
|
||||
|
||||
if (scope is null)
|
||||
{
|
||||
return Array.Empty<AppleReferenceDto>();
|
||||
}
|
||||
|
||||
var anchors = scope.QuerySelectorAll("a[href]");
|
||||
if (anchors.Length == 0)
|
||||
{
|
||||
return Array.Empty<AppleReferenceDto>();
|
||||
}
|
||||
|
||||
var references = new List<AppleReferenceDto>(anchors.Length);
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var element in anchors)
|
||||
{
|
||||
if (element is not IHtmlAnchorElement anchor)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (anchor.HasAttribute("data-globalnav-item-name")
|
||||
|| anchor.HasAttribute("data-analytics-title"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var href = anchor.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(detailUri, href, out var uri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!IsRelevantReference(uri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = uri.ToString();
|
||||
if (!seen.Add(normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = CleanWhitespace(anchor.TextContent);
|
||||
var kind = ResolveReferenceKind(uri);
|
||||
references.Add(new AppleReferenceDto(normalized, title, kind));
|
||||
}
|
||||
|
||||
if (references.Count == 0)
|
||||
{
|
||||
return Array.Empty<AppleReferenceDto>();
|
||||
}
|
||||
|
||||
references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
|
||||
return references;
|
||||
}
|
||||
|
||||
private static bool IsRelevantReference(Uri uri)
|
||||
{
|
||||
if (!uri.IsAbsoluteUri)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(uri.Host))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!uri.AbsolutePath.Contains("/support/", StringComparison.OrdinalIgnoreCase)
|
||||
&& !uri.AbsolutePath.Contains("/security", StringComparison.OrdinalIgnoreCase)
|
||||
&& !uri.AbsolutePath.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (uri.Host.EndsWith(".apple.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var supported =
|
||||
uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase)
|
||||
|| uri.Host.StartsWith("developer.", StringComparison.OrdinalIgnoreCase)
|
||||
|| uri.Host.StartsWith("download.", StringComparison.OrdinalIgnoreCase)
|
||||
|| uri.Host.StartsWith("updates.", StringComparison.OrdinalIgnoreCase)
|
||||
|| uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (!supported)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase)
|
||||
&& uri.AbsolutePath == "/"
|
||||
&& (string.IsNullOrEmpty(uri.Query)
|
||||
|| uri.Query.Contains("cid=", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string? ResolveReferenceKind(Uri uri)
|
||||
{
|
||||
if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "download";
|
||||
}
|
||||
|
||||
if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "document";
|
||||
}
|
||||
|
||||
return "advisory";
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "nvd";
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "kb";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string CleanWhitespace(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var span = value.AsSpan();
|
||||
var buffer = new char[span.Length];
|
||||
var index = 0;
|
||||
var previousWhitespace = false;
|
||||
|
||||
foreach (var ch in span)
|
||||
{
|
||||
if (char.IsWhiteSpace(ch))
|
||||
{
|
||||
if (previousWhitespace)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer[index++] = ' ';
|
||||
previousWhitespace = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer[index++] = ch;
|
||||
previousWhitespace = false;
|
||||
}
|
||||
}
|
||||
|
||||
return new string(buffer, 0, index).Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
public sealed class AppleDiagnostics : IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Apple";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchItems;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _mapAffected;
|
||||
|
||||
public AppleDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchItems = _meter.CreateCounter<long>(
|
||||
name: "apple.fetch.items",
|
||||
unit: "documents",
|
||||
description: "Number of Apple advisories fetched.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "apple.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of Apple fetch failures.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "apple.fetch.unchanged",
|
||||
unit: "documents",
|
||||
description: "Number of Apple advisories skipped due to 304 responses.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "apple.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of Apple documents that failed to parse.");
|
||||
_mapAffected = _meter.CreateHistogram<long>(
|
||||
name: "apple.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected package counts emitted per Apple advisory.");
|
||||
}
|
||||
|
||||
public Meter Meter => _meter;
|
||||
|
||||
public void FetchItem() => _fetchItems.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void MapAffectedCount(int count)
|
||||
{
|
||||
if (count >= 0)
|
||||
{
|
||||
_mapAffected.Record(count);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
internal sealed record AppleIndexEntry(
|
||||
string UpdateId,
|
||||
string ArticleId,
|
||||
string Title,
|
||||
DateTimeOffset PostingDate,
|
||||
Uri DetailUri,
|
||||
IReadOnlyList<AppleIndexProduct> Products,
|
||||
bool IsRapidSecurityResponse);
|
||||
|
||||
internal sealed record AppleIndexProduct(
|
||||
string Platform,
|
||||
string Name,
|
||||
string Version,
|
||||
string Build);
|
||||
|
||||
internal static class AppleIndexParser
|
||||
{
|
||||
private sealed record AppleIndexDocument(
|
||||
[property: JsonPropertyName("updates")] IReadOnlyList<AppleIndexEntryDto>? Updates);
|
||||
|
||||
private sealed record AppleIndexEntryDto(
|
||||
[property: JsonPropertyName("id")] string? Id,
|
||||
[property: JsonPropertyName("articleId")] string? ArticleId,
|
||||
[property: JsonPropertyName("title")] string? Title,
|
||||
[property: JsonPropertyName("postingDate")] string? PostingDate,
|
||||
[property: JsonPropertyName("detailUrl")] string? DetailUrl,
|
||||
[property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse,
|
||||
[property: JsonPropertyName("products")] IReadOnlyList<AppleIndexProductDto>? Products);
|
||||
|
||||
private sealed record AppleIndexProductDto(
|
||||
[property: JsonPropertyName("platform")] string? Platform,
|
||||
[property: JsonPropertyName("name")] string? Name,
|
||||
[property: JsonPropertyName("version")] string? Version,
|
||||
[property: JsonPropertyName("build")] string? Build);
|
||||
|
||||
public static IReadOnlyList<AppleIndexEntry> Parse(ReadOnlySpan<byte> payload, Uri baseUri)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return Array.Empty<AppleIndexEntry>();
|
||||
}
|
||||
|
||||
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
AppleIndexDocument? document;
|
||||
try
|
||||
{
|
||||
document = JsonSerializer.Deserialize<AppleIndexDocument>(payload, options);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return Array.Empty<AppleIndexEntry>();
|
||||
}
|
||||
|
||||
if (document?.Updates is null || document.Updates.Count == 0)
|
||||
{
|
||||
return Array.Empty<AppleIndexEntry>();
|
||||
}
|
||||
|
||||
var entries = new List<AppleIndexEntry>(document.Updates.Count);
|
||||
foreach (var dto in document.Updates)
|
||||
{
|
||||
if (dto is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id;
|
||||
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryResolveDetailUri(dto, baseUri, out var detailUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var products = dto.Products?.Select(static product => new AppleIndexProduct(
|
||||
product.Platform ?? string.Empty,
|
||||
product.Name ?? product.Platform ?? string.Empty,
|
||||
product.Version ?? string.Empty,
|
||||
product.Build ?? string.Empty))
|
||||
.ToArray() ?? Array.Empty<AppleIndexProduct>();
|
||||
|
||||
entries.Add(new AppleIndexEntry(
|
||||
id.Trim(),
|
||||
dto.ArticleId!.Trim(),
|
||||
dto.Title!.Trim(),
|
||||
postingDate.ToUniversalTime(),
|
||||
detailUri,
|
||||
products,
|
||||
dto.RapidSecurityResponse ?? false));
|
||||
}
|
||||
|
||||
return entries.Count == 0 ? Array.Empty<AppleIndexEntry>() : entries;
|
||||
}
|
||||
|
||||
private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(dto.ArticleId))
|
||||
{
|
||||
uri = default!;
|
||||
return false;
|
||||
}
|
||||
|
||||
var article = dto.ArticleId.Trim();
|
||||
if (article.Length == 0)
|
||||
{
|
||||
uri = default!;
|
||||
return false;
|
||||
}
|
||||
|
||||
var combined = new Uri(baseUri, article);
|
||||
uri = combined;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Packages;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
|
||||
|
||||
internal static class AppleMapper
|
||||
{
|
||||
public static (Advisory Advisory, PsirtFlagRecord? Flag) Map(
|
||||
AppleDetailDto dto,
|
||||
DocumentRecord document,
|
||||
DtoRecord dtoRecord)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
|
||||
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(
|
||||
VndrAppleConnectorPlugin.SourceName,
|
||||
"document",
|
||||
document.Uri,
|
||||
document.FetchedAt.ToUniversalTime());
|
||||
|
||||
var mapProvenance = new AdvisoryProvenance(
|
||||
VndrAppleConnectorPlugin.SourceName,
|
||||
"map",
|
||||
dto.AdvisoryId,
|
||||
recordedAt);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var affected = BuildAffected(dto, recordedAt);
|
||||
|
||||
var advisory = new Advisory(
|
||||
advisoryKey: dto.AdvisoryId,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: "en",
|
||||
published: dto.Published.ToUniversalTime(),
|
||||
modified: dto.Updated?.ToUniversalTime(),
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affected,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mapProvenance });
|
||||
|
||||
PsirtFlagRecord? flag = dto.RapidSecurityResponse
|
||||
? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt)
|
||||
: null;
|
||||
|
||||
return (advisory, flag);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(AppleDetailDto dto)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
dto.AdvisoryId,
|
||||
dto.ArticleId,
|
||||
};
|
||||
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
set.Add(cve.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
var aliases = set.ToList();
|
||||
aliases.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
return aliases;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.References.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var list = new List<AdvisoryReference>(dto.References.Count);
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
VndrAppleConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt);
|
||||
|
||||
list.Add(new AdvisoryReference(
|
||||
url: reference.Url,
|
||||
kind: reference.Kind,
|
||||
sourceTag: null,
|
||||
summary: reference.Title,
|
||||
provenance: provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore invalid URLs
|
||||
}
|
||||
}
|
||||
|
||||
if (list.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Affected.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Affected.Count);
|
||||
foreach (var product in dto.Affected)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(
|
||||
VndrAppleConnectorPlugin.SourceName,
|
||||
"affected",
|
||||
product.Name,
|
||||
recordedAt),
|
||||
};
|
||||
|
||||
var ranges = BuildRanges(product, recordedAt);
|
||||
var normalizedVersions = BuildNormalizedVersions(product, ranges);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
type: AffectedPackageTypes.Vendor,
|
||||
identifier: product.Name,
|
||||
platform: product.Platform,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance,
|
||||
normalizedVersions: normalizedVersions));
|
||||
}
|
||||
|
||||
return packages.Count == 0 ? Array.Empty<AffectedPackage>() : packages;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build))
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
VndrAppleConnectorPlugin.SourceName,
|
||||
"range",
|
||||
product.Name,
|
||||
recordedAt);
|
||||
|
||||
var extensions = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (!string.IsNullOrWhiteSpace(product.Version))
|
||||
{
|
||||
extensions["apple.version.raw"] = product.Version;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Build))
|
||||
{
|
||||
extensions["apple.build"] = product.Build;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Platform))
|
||||
{
|
||||
extensions["apple.platform"] = product.Platform;
|
||||
}
|
||||
|
||||
var primitives = extensions.Count == 0
|
||||
? null
|
||||
: new RangePrimitives(
|
||||
SemVer: TryCreateSemVerPrimitive(product.Version),
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: extensions);
|
||||
|
||||
var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion)
|
||||
? normalizedVersion
|
||||
: product.Version;
|
||||
|
||||
return new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: null,
|
||||
fixedVersion: sanitizedVersion,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.Version,
|
||||
provenance: provenance,
|
||||
primitives: primitives),
|
||||
};
|
||||
}
|
||||
|
||||
private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// treat as fixed version, unknown introduced/last affected
|
||||
return new SemVerPrimitive(
|
||||
Introduced: null,
|
||||
IntroducedInclusive: true,
|
||||
Fixed: normalized,
|
||||
FixedInclusive: true,
|
||||
LastAffected: null,
|
||||
LastAffectedInclusive: true,
|
||||
ConstraintExpression: null);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
AppleAffectedProductDto product,
|
||||
IReadOnlyList<AffectedVersionRange> ranges)
|
||||
{
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var segments = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Platform))
|
||||
{
|
||||
segments.Add(product.Platform.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Name))
|
||||
{
|
||||
segments.Add(product.Name.Trim());
|
||||
}
|
||||
|
||||
var note = segments.Count == 0 ? null : $"apple:{string.Join(':', segments)}";
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(ranges.Count);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var rule = range.ToNormalizedVersionRule(note);
|
||||
if (rule is not null)
|
||||
{
|
||||
rules.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
|
||||
}
|
||||
}
|
||||
46
src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs
Normal file
46
src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs
Normal file
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
internal static class AppleJobKinds
|
||||
{
|
||||
public const string Fetch = "source:vndr-apple:fetch";
|
||||
public const string Parse = "source:vndr-apple:parse";
|
||||
public const string Map = "source:vndr-apple:map";
|
||||
}
|
||||
|
||||
internal sealed class AppleFetchJob : IJob
|
||||
{
|
||||
private readonly AppleConnector _connector;
|
||||
|
||||
public AppleFetchJob(AppleConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AppleParseJob : IJob
|
||||
{
|
||||
private readonly AppleConnector _connector;
|
||||
|
||||
public AppleParseJob(AppleConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class AppleMapJob : IJob
|
||||
{
|
||||
private readonly AppleConnector _connector;
|
||||
|
||||
public AppleMapJob(AppleConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Apple.Tests")]
|
||||
49
src/StellaOps.Concelier.Connector.Vndr.Apple/README.md
Normal file
49
src/StellaOps.Concelier.Connector.Vndr.Apple/README.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# Apple Security Updates Connector
|
||||
|
||||
## Feed contract
|
||||
|
||||
The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes:
|
||||
|
||||
- `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`)
|
||||
- Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline`
|
||||
- Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS)
|
||||
- Metadata for download packages, release notes, and signing assets
|
||||
|
||||
The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8
|
||||
|
||||
Apple’s new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists:
|
||||
|
||||
- CVE identifiers with Apple’s internal tracking IDs
|
||||
- Product version/build applicability tables
|
||||
- Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6
|
||||
|
||||
Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `<section data-component="security-update">` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries.
|
||||
|
||||
## Change detection strategy
|
||||
|
||||
1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches.
|
||||
2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`).
|
||||
3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only.
|
||||
|
||||
Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3
|
||||
|
||||
## Parsing & mapping notes
|
||||
|
||||
- CVE lists live inside `<ul data-testid="cve-list">` items; each `<li>` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apple’s component name as the package `name` and the OS build as the range primitive seed.
|
||||
- Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`).
|
||||
- Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`.
|
||||
|
||||
## Outstanding questions
|
||||
|
||||
- Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`.
|
||||
- Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured.
|
||||
|
||||
## Fixture maintenance
|
||||
|
||||
Deterministic regression coverage lives in `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. When Apple publishes new advisories the fixtures must be refreshed using the provided helper scripts:
|
||||
|
||||
- Bash: `./scripts/update-apple-fixtures.sh`
|
||||
- PowerShell: `./scripts/update-apple-fixtures.ps1`
|
||||
|
||||
Both scripts set `UPDATE_APPLE_FIXTURES=1`, touch a `.update-apple-fixtures` sentinel so test runs inside WSL propagate the flag, fetch the live HT articles referenced in `AppleFixtureManager`, sanitise the HTML, and rewrite the paired `.expected.json` DTO snapshots. Always inspect the resulting diff and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the environment variable to ensure deterministic output before committing.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
11
src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md
Normal file
11
src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** – Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.|
|
||||
|Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** – Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.|
|
||||
|Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** – AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.|
|
||||
|Canonical mapping & range primitives|BE-Conn-Apple|Models|**DONE** – Mapper now emits SemVer-derived normalizedVersions with `apple:<platform>:<product>` notes; fixtures updated to assert canonical rules while we continue tracking multi-device coverage in follow-up tasks.<br>2025-10-11 research trail: confirmed payload aligns with `[{"scheme":"semver","type":"range","min":"<build-start>","minInclusive":true,"max":"<build-end>","maxInclusive":false,"notes":"apple:ios:17.1"}]`; continue using `notes` to surface build identifiers for storage provenance.|
|
||||
|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-12)** – Parser now scopes references to article content, sorts affected rows deterministically, and regenerated fixtures (125326/125328/106355/HT214108/HT215500) produce stable JSON + sanitizer HTML in English.|
|
||||
|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – OpenTelemetry pipeline exports `StellaOps.Concelier.Connector.Vndr.Apple`; runbook `docs/ops/concelier-apple-operations.md` added with metrics + monitoring guidance.|
|
||||
|Live HTML regression sweep|QA|Source.Common|**DONE (2025-10-12)** – Captured latest support.apple.com articles for 125326/125328/106355/HT214108/HT215500, trimmed nav noise, and committed sanitized HTML + expected DTOs with invariant timestamps.|
|
||||
|Fixture regeneration tooling|DevEx|Testing|**DONE (2025-10-12)** – `scripts/update-apple-fixtures.(sh|ps1)` set the env flag + sentinel, forward through WSLENV, and clean up after regeneration; README references updated usage.|
|
||||
@@ -0,0 +1,24 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Apple;
|
||||
|
||||
public sealed class VndrAppleConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "vndr-apple";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetService<AppleConnector>() is not null;
|
||||
}
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<AppleConnector>();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user