Rename Concelier Source modules to Connector

This commit is contained in:
2025-10-18 20:11:18 +03:00
parent 0137856fdb
commit 6524626230
789 changed files with 1489 additions and 1489 deletions

View File

@@ -0,0 +1,39 @@
# AGENTS
## Role
Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS.
## Scope
- Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy.
- Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks.
- Parse advisories to extract summary, affected products/versions, mitigation, CVEs.
- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions).
- Produce deterministic fixtures and regression tests.
## Participants
- `Source.Common` (HTTP/fetch utilities, DTO storage).
- `Storage.Mongo` (raw/document/DTO/advisory stores, source state).
- `Concelier.Models` (canonical structures + range primitives).
- `Concelier.Testing` (integration fixtures/snapshots).
## Interfaces & Contracts
- Job kinds: `apple:fetch`, `apple:parse`, `apple:map`.
- Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates.
- Alias set should include Apple HT IDs and CVE IDs.
## In/Out of scope
In scope:
- Security advisories covering Apple OS/app updates.
- Range primitives capturing device/OS version ranges.
Out of scope:
- Release notes unrelated to security.
## Observability & Security Expectations
- Log fetch/mapping statistics and failure details.
- Sanitize HTML while preserving structured data tables.
- Respect upstream rate limits; record failures with backoff.
## Tests
- Add `StellaOps.Concelier.Connector.Vndr.Apple.Tests` covering fetch/parse/map with fixtures.
- Snapshot canonical advisories; support fixture regeneration via env flag.
- Ensure deterministic ordering/time normalisation.

View File

@@ -0,0 +1,439 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Vndr.Apple.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
public sealed class AppleConnector : IFeedConnector
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNameCaseInsensitive = true,
};
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
private readonly IDtoStore _dtoStore;
private readonly IAdvisoryStore _advisoryStore;
private readonly IPsirtFlagStore _psirtFlagStore;
private readonly ISourceStateRepository _stateRepository;
private readonly AppleOptions _options;
private readonly AppleDiagnostics _diagnostics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<AppleConnector> _logger;
public AppleConnector(
SourceFetchService fetchService,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
IPsirtFlagStore psirtFlagStore,
ISourceStateRepository stateRepository,
AppleDiagnostics diagnostics,
IOptions<AppleOptions> options,
TimeProvider? timeProvider,
ILogger<AppleConnector> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => VndrAppleConnectorPlugin.SourceName;
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase);
var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue;
var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill);
SourceFetchContentResult indexResult;
try
{
var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!)
{
AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" },
};
indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.FetchFailure();
_logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri);
await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
if (!indexResult.IsSuccess || indexResult.Content is null)
{
if (indexResult.IsNotModified)
{
_diagnostics.FetchUnchanged();
}
await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false);
return;
}
var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!);
if (indexEntries.Count == 0)
{
await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false);
return;
}
var allowlist = _options.AdvisoryAllowlist;
var blocklist = _options.AdvisoryBlocklist;
var ordered = indexEntries
.Where(entry => ShouldInclude(entry, allowlist, blocklist))
.OrderBy(entry => entry.PostingDate)
.ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase)
.ToArray();
foreach (var entry in ordered)
{
cancellationToken.ThrowIfCancellationRequested();
if (entry.PostingDate < baseline)
{
continue;
}
if (cursor.LastPosted.HasValue
&& entry.PostingDate <= cursor.LastPosted.Value
&& processedIds.Contains(entry.UpdateId))
{
continue;
}
var metadata = BuildMetadata(entry);
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false);
SourceFetchResult result;
try
{
result = await _fetchService.FetchAsync(
new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri)
{
Metadata = metadata,
ETag = existing?.Etag,
LastModified = existing?.LastModified,
AcceptHeaders = new[]
{
"text/html",
"application/xhtml+xml",
"text/plain;q=0.5"
},
},
cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.FetchFailure();
_logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri);
await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
if (result.StatusCode == HttpStatusCode.NotModified)
{
_diagnostics.FetchUnchanged();
}
if (!result.IsSuccess || result.Document is null)
{
continue;
}
_diagnostics.FetchItem();
pendingDocuments.Add(result.Document.Id);
processedIds.Add(entry.UpdateId);
if (entry.PostingDate > maxPosted)
{
maxPosted = entry.PostingDate;
}
}
var updated = cursor
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(pendingMappings)
.WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds);
await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false);
}
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingDocuments.Count == 0)
{
return;
}
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
foreach (var documentId in cursor.PendingDocuments)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
if (!document.GridFsId.HasValue)
{
_diagnostics.ParseFailure();
_logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
AppleDetailDto dto;
try
{
var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
var html = System.Text.Encoding.UTF8.GetString(content);
var entry = RehydrateIndexEntry(document);
dto = AppleDetailParser.Parse(html, entry);
}
catch (Exception ex)
{
_diagnostics.ParseFailure();
_logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = BsonDocument.Parse(json);
var validatedAt = _timeProvider.GetUtcNow();
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt)
: existingDto with
{
Payload = payload,
SchemaVersion = "apple.security.update.v1",
ValidatedAt = validatedAt,
};
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Add(document.Id);
}
var updatedCursor = cursor
.WithPendingDocuments(remainingDocuments)
.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingMappings.Count == 0)
{
return;
}
var pendingMappings = cursor.PendingMappings.ToHashSet();
foreach (var documentId in cursor.PendingMappings)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
pendingMappings.Remove(documentId);
continue;
}
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
if (dtoRecord is null)
{
pendingMappings.Remove(documentId);
continue;
}
AppleDetailDto dto;
try
{
dto = JsonSerializer.Deserialize<AppleDetailDto>(dtoRecord.Payload.ToJson(), SerializerOptions)
?? throw new InvalidOperationException("Unable to deserialize Apple DTO.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id);
pendingMappings.Remove(documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
continue;
}
var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord);
_diagnostics.MapAffectedCount(advisory.AffectedPackages.Length);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
if (flag is not null)
{
await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document)
{
var metadata = document.Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal);
metadata.TryGetValue("apple.articleId", out var articleId);
metadata.TryGetValue("apple.updateId", out var updateId);
metadata.TryGetValue("apple.title", out var title);
metadata.TryGetValue("apple.postingDate", out var postingDateRaw);
metadata.TryGetValue("apple.detailUri", out var detailUriRaw);
metadata.TryGetValue("apple.rapidResponse", out var rapidRaw);
metadata.TryGetValue("apple.products", out var productsJson);
if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate))
{
postingDate = document.FetchedAt;
}
var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri)
? parsedUri
: new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri);
var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase);
var products = DeserializeProducts(productsJson);
return new AppleIndexEntry(
UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId,
ArticleId: articleId ?? document.Uri,
Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update",
PostingDate: postingDate.ToUniversalTime(),
DetailUri: detailUri,
Products: products,
IsRapidSecurityResponse: rapid);
}
private static IReadOnlyList<AppleIndexProduct> DeserializeProducts(string? json)
{
if (string.IsNullOrWhiteSpace(json))
{
return Array.Empty<AppleIndexProduct>();
}
try
{
var products = JsonSerializer.Deserialize<List<AppleIndexProduct>>(json, SerializerOptions);
return products is { Count: > 0 } ? products : Array.Empty<AppleIndexProduct>();
}
catch (JsonException)
{
return Array.Empty<AppleIndexProduct>();
}
}
private static Dictionary<string, string> BuildMetadata(AppleIndexEntry entry)
{
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["apple.articleId"] = entry.ArticleId,
["apple.updateId"] = entry.UpdateId,
["apple.title"] = entry.Title,
["apple.postingDate"] = entry.PostingDate.ToString("O"),
["apple.detailUri"] = entry.DetailUri.ToString(),
["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false",
["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions),
};
return metadata;
}
private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection<string> allowlist, IReadOnlyCollection<string> blocklist)
{
if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId))
{
return false;
}
if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId))
{
return false;
}
return true;
}
private async Task<AppleCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor);
}
private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBson();
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,53 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:apple";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddAppleConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<AppleFetchJob>();
services.AddTransient<AppleParseJob>();
services.AddTransient<AppleMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob));
EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob));
EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -0,0 +1,101 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
public sealed class AppleOptions : IValidatableObject
{
public const string HttpClientName = "concelier-vndr-apple";
/// <summary>
/// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service).
/// </summary>
public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv");
/// <summary>
/// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required.
/// </summary>
public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/");
/// <summary>
/// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us".
/// </summary>
public string LocaleSegment { get; set; } = "en-us";
/// <summary>
/// Maximum advisories to fetch per run; defaults to 50.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 50;
/// <summary>
/// Sliding backfill window for initial sync (defaults to 90 days).
/// </summary>
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90);
/// <summary>
/// Tolerance added to the modified timestamp comparisons during resume.
/// </summary>
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1);
/// <summary>
/// Optional allowlist of HT identifiers to include; empty means include all.
/// </summary>
public HashSet<string> AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed).
/// </summary>
public HashSet<string> AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase);
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (SoftwareLookupUri is null)
{
yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) });
}
else if (!SoftwareLookupUri.IsAbsoluteUri)
{
yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) });
}
if (AdvisoryBaseUri is null)
{
yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) });
}
else if (!AdvisoryBaseUri.IsAbsoluteUri)
{
yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) });
}
if (string.IsNullOrWhiteSpace(LocaleSegment))
{
yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) });
}
if (MaxAdvisoriesPerFetch <= 0)
{
yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) });
}
if (InitialBackfill <= TimeSpan.Zero)
{
yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) });
}
if (ModifiedTolerance < TimeSpan.Zero)
{
yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) });
}
}
public void Validate()
{
var context = new ValidationContext(this);
var results = new List<ValidationResult>();
if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true))
{
throw new ValidationException(string.Join("; ", results));
}
}
}

View File

@@ -0,0 +1,44 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Vndr.Apple.Internal;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
public static class AppleServiceCollectionExtensions
{
public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action<AppleOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<AppleOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate())
.ValidateOnStart();
services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<AppleOptions>>().Value;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.Apple/1.0";
clientOptions.AllowedHosts.Clear();
if (options.SoftwareLookupUri is not null)
{
clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host);
}
if (options.AdvisoryBaseUri is not null)
{
clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host);
}
});
services.TryAddSingleton<TimeProvider>(_ => TimeProvider.System);
services.AddSingleton<AppleDiagnostics>();
services.AddTransient<AppleConnector>();
return services;
}
}

View File

@@ -0,0 +1,114 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
internal sealed record AppleCursor(
DateTimeOffset? LastPosted,
IReadOnlyCollection<string> ProcessedIds,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStringCollection = Array.Empty<string>();
public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection);
public BsonDocument ToBson()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPosted.HasValue)
{
document["lastPosted"] = LastPosted.Value.UtcDateTime;
}
if (ProcessedIds.Count > 0)
{
document["processedIds"] = new BsonArray(ProcessedIds);
}
return document;
}
public static AppleCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue)
? ParseDate(lastPostedValue)
: null;
var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray
? processedArray.OfType<BsonValue>()
.Where(static value => value.BsonType == BsonType.String)
.Select(static value => value.AsString.Trim())
.Where(static value => value.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray()
: EmptyStringCollection;
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings);
}
public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable<string>? processedIds = null)
{
var ids = processedIds is null
? ProcessedIds
: processedIds.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
return this with
{
LastPosted = timestamp.ToUniversalTime(),
ProcessedIds = ids,
};
}
public AppleCursor WithPendingDocuments(IEnumerable<Guid>? ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection };
public AppleCursor WithPendingMappings(IEnumerable<Guid>? ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection };
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key)
{
if (!document.TryGetValue(key, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,78 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
internal sealed record AppleDetailDto(
string AdvisoryId,
string ArticleId,
string Title,
string Summary,
DateTimeOffset Published,
DateTimeOffset? Updated,
IReadOnlyList<string> CveIds,
IReadOnlyList<AppleAffectedProductDto> Affected,
IReadOnlyList<AppleReferenceDto> References,
bool RapidSecurityResponse);
internal sealed record AppleAffectedProductDto(
string Platform,
string Name,
string Version,
string Build);
internal sealed record AppleReferenceDto(
string Url,
string? Title,
string? Kind);
internal static class AppleDetailDtoExtensions
{
public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable<AppleIndexProduct> products)
{
if (dto.Affected.Count > 0)
{
return dto;
}
var fallback = products
.Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build))
.Select(static product => new AppleAffectedProductDto(
product.Platform,
product.Name,
product.Version,
product.Build))
.OrderBy(static product => NormalizeSortKey(product.Platform))
.ThenBy(static product => NormalizeSortKey(product.Name))
.ThenBy(static product => NormalizeSortKey(product.Version))
.ThenBy(static product => NormalizeSortKey(product.Build))
.ToArray();
return fallback.Length == 0 ? dto : dto with { Affected = fallback };
}
private static string NormalizeSortKey(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var span = value.AsSpan();
var buffer = new char[span.Length];
var index = 0;
foreach (var ch in span)
{
if (char.IsWhiteSpace(ch))
{
continue;
}
buffer[index++] = char.ToUpperInvariant(ch);
}
return new string(buffer, 0, index);
}
}

View File

@@ -0,0 +1,460 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
internal static class AppleDetailParser
{
private static readonly HtmlParser Parser = new();
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public static AppleDetailDto Parse(string html, AppleIndexEntry entry)
{
if (string.IsNullOrWhiteSpace(html))
{
throw new ArgumentException("HTML content must not be empty.", nameof(html));
}
var document = Parser.ParseDocument(html);
var title = ResolveTitle(document, entry.Title);
var summary = ResolveSummary(document);
var (published, updated) = ResolveTimestamps(document, entry.PostingDate);
var cves = ExtractCves(document);
var affected = NormalizeAffectedProducts(ExtractProducts(document));
var references = ExtractReferences(document, entry.DetailUri);
var dto = new AppleDetailDto(
entry.ArticleId,
entry.ArticleId,
title,
summary,
published,
updated,
cves,
affected,
references,
entry.IsRapidSecurityResponse);
return dto.WithAffectedFallback(entry.Products);
}
private static IReadOnlyList<AppleAffectedProductDto> NormalizeAffectedProducts(IReadOnlyList<AppleAffectedProductDto> affected)
{
if (affected.Count <= 1)
{
return affected;
}
return affected
.OrderBy(static product => NormalizeSortKey(product.Platform))
.ThenBy(static product => NormalizeSortKey(product.Name))
.ThenBy(static product => NormalizeSortKey(product.Version))
.ThenBy(static product => NormalizeSortKey(product.Build))
.ToArray();
}
private static string NormalizeSortKey(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var span = value.AsSpan();
var buffer = new char[span.Length];
var index = 0;
foreach (var ch in span)
{
if (char.IsWhiteSpace(ch))
{
continue;
}
buffer[index++] = char.ToUpperInvariant(ch);
}
return new string(buffer, 0, index);
}
private static string ResolveTitle(IHtmlDocument document, string fallback)
{
var title = document.QuerySelector("[data-testid='update-title']")?.TextContent
?? document.QuerySelector("h1, h2")?.TextContent
?? document.Title;
title = title?.Trim();
return string.IsNullOrEmpty(title) ? fallback : title;
}
private static string ResolveSummary(IHtmlDocument document)
{
var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent
?? document.QuerySelector("meta[name='description']")?.GetAttribute("content")
?? document.QuerySelector("p")?.TextContent
?? string.Empty;
return CleanWhitespace(summary);
}
private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback)
{
DateTimeOffset published = postingFallback;
DateTimeOffset? updated = null;
foreach (var time in document.QuerySelectorAll("time"))
{
var raw = time.GetAttribute("datetime") ?? time.TextContent;
if (string.IsNullOrWhiteSpace(raw))
{
continue;
}
if (!DateTimeOffset.TryParse(raw, out var parsed))
{
continue;
}
parsed = parsed.ToUniversalTime();
var itemProp = time.GetAttribute("itemprop") ?? string.Empty;
var dataTestId = time.GetAttribute("data-testid") ?? string.Empty;
if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase)
|| dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase))
{
published = parsed;
}
else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase)
|| dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase))
{
updated = parsed;
}
else if (updated is null && parsed > published)
{
updated = parsed;
}
}
return (published, updated);
}
private static IReadOnlyList<string> ExtractCves(IHtmlDocument document)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var node in document.All)
{
if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element)
{
continue;
}
var text = node.TextContent;
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
if (set.Count == 0)
{
return Array.Empty<string>();
}
var list = set.ToList();
list.Sort(StringComparer.OrdinalIgnoreCase);
return list;
}
private static IReadOnlyList<AppleAffectedProductDto> ExtractProducts(IHtmlDocument document)
{
var rows = new List<AppleAffectedProductDto>();
foreach (var element in document.QuerySelectorAll("[data-testid='product-row']"))
{
var platform = element.GetAttribute("data-platform") ?? string.Empty;
var name = element.GetAttribute("data-product") ?? platform;
var version = element.GetAttribute("data-version") ?? string.Empty;
var build = element.GetAttribute("data-build") ?? string.Empty;
if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow)
{
var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray();
if (cells.Length >= 1)
{
name = cells[0];
}
if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version))
{
version = cells[1];
}
if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build))
{
build = cells[2];
}
}
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
rows.Add(new AppleAffectedProductDto(platform, name, version, build));
}
if (rows.Count > 0)
{
return rows;
}
// fallback for generic tables without data attributes
foreach (var table in document.QuerySelectorAll("table"))
{
var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray();
if (headers.Length == 0)
{
continue;
}
var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase)
|| header.Contains("device", StringComparison.OrdinalIgnoreCase));
var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase));
var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase)
|| header.Contains("release", StringComparison.OrdinalIgnoreCase));
if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1)
{
continue;
}
foreach (var row in table.QuerySelectorAll("tr"))
{
var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray();
if (cells.Length == 0)
{
continue;
}
string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0];
string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty;
string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty;
if (string.IsNullOrWhiteSpace(name))
{
continue;
}
rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build));
}
if (rows.Count > 0)
{
break;
}
}
return rows.Count == 0 ? Array.Empty<AppleAffectedProductDto>() : rows;
}
private static IReadOnlyList<AppleReferenceDto> ExtractReferences(IHtmlDocument document, Uri detailUri)
{
var scope = document.QuerySelector("article")
?? document.QuerySelector("main")
?? (IElement?)document.Body;
if (scope is null)
{
return Array.Empty<AppleReferenceDto>();
}
var anchors = scope.QuerySelectorAll("a[href]");
if (anchors.Length == 0)
{
return Array.Empty<AppleReferenceDto>();
}
var references = new List<AppleReferenceDto>(anchors.Length);
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var element in anchors)
{
if (element is not IHtmlAnchorElement anchor)
{
continue;
}
if (anchor.HasAttribute("data-globalnav-item-name")
|| anchor.HasAttribute("data-analytics-title"))
{
continue;
}
var href = anchor.GetAttribute("href");
if (string.IsNullOrWhiteSpace(href))
{
continue;
}
if (!Uri.TryCreate(detailUri, href, out var uri))
{
continue;
}
if (!IsRelevantReference(uri))
{
continue;
}
var normalized = uri.ToString();
if (!seen.Add(normalized))
{
continue;
}
var title = CleanWhitespace(anchor.TextContent);
var kind = ResolveReferenceKind(uri);
references.Add(new AppleReferenceDto(normalized, title, kind));
}
if (references.Count == 0)
{
return Array.Empty<AppleReferenceDto>();
}
references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
return references;
}
private static bool IsRelevantReference(Uri uri)
{
if (!uri.IsAbsoluteUri)
{
return false;
}
if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (string.IsNullOrWhiteSpace(uri.Host))
{
return false;
}
if (uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase))
{
if (!uri.AbsolutePath.Contains("/support/", StringComparison.OrdinalIgnoreCase)
&& !uri.AbsolutePath.Contains("/security", StringComparison.OrdinalIgnoreCase)
&& !uri.AbsolutePath.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase))
{
return false;
}
}
if (uri.Host.EndsWith(".apple.com", StringComparison.OrdinalIgnoreCase))
{
var supported =
uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase)
|| uri.Host.StartsWith("developer.", StringComparison.OrdinalIgnoreCase)
|| uri.Host.StartsWith("download.", StringComparison.OrdinalIgnoreCase)
|| uri.Host.StartsWith("updates.", StringComparison.OrdinalIgnoreCase)
|| uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase);
if (!supported)
{
return false;
}
if (uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase)
&& uri.AbsolutePath == "/"
&& (string.IsNullOrEmpty(uri.Query)
|| uri.Query.Contains("cid=", StringComparison.OrdinalIgnoreCase)))
{
return false;
}
}
return true;
}
private static string? ResolveReferenceKind(Uri uri)
{
if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase))
{
if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase))
{
return "download";
}
if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase))
{
return "document";
}
return "advisory";
}
if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase))
{
return "nvd";
}
if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase))
{
return "kb";
}
return null;
}
private static string CleanWhitespace(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var span = value.AsSpan();
var buffer = new char[span.Length];
var index = 0;
var previousWhitespace = false;
foreach (var ch in span)
{
if (char.IsWhiteSpace(ch))
{
if (previousWhitespace)
{
continue;
}
buffer[index++] = ' ';
previousWhitespace = true;
}
else
{
buffer[index++] = ch;
previousWhitespace = false;
}
}
return new string(buffer, 0, index).Trim();
}
}

View File

@@ -0,0 +1,62 @@
using System;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
public sealed class AppleDiagnostics : IDisposable
{
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Apple";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchItems;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _mapAffected;
public AppleDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchItems = _meter.CreateCounter<long>(
name: "apple.fetch.items",
unit: "documents",
description: "Number of Apple advisories fetched.");
_fetchFailures = _meter.CreateCounter<long>(
name: "apple.fetch.failures",
unit: "operations",
description: "Number of Apple fetch failures.");
_fetchUnchanged = _meter.CreateCounter<long>(
name: "apple.fetch.unchanged",
unit: "documents",
description: "Number of Apple advisories skipped due to 304 responses.");
_parseFailures = _meter.CreateCounter<long>(
name: "apple.parse.failures",
unit: "documents",
description: "Number of Apple documents that failed to parse.");
_mapAffected = _meter.CreateHistogram<long>(
name: "apple.map.affected.count",
unit: "packages",
description: "Distribution of affected package counts emitted per Apple advisory.");
}
public Meter Meter => _meter;
public void FetchItem() => _fetchItems.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void MapAffectedCount(int count)
{
if (count >= 0)
{
_mapAffected.Record(count);
}
}
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,144 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
internal sealed record AppleIndexEntry(
string UpdateId,
string ArticleId,
string Title,
DateTimeOffset PostingDate,
Uri DetailUri,
IReadOnlyList<AppleIndexProduct> Products,
bool IsRapidSecurityResponse);
internal sealed record AppleIndexProduct(
string Platform,
string Name,
string Version,
string Build);
internal static class AppleIndexParser
{
private sealed record AppleIndexDocument(
[property: JsonPropertyName("updates")] IReadOnlyList<AppleIndexEntryDto>? Updates);
private sealed record AppleIndexEntryDto(
[property: JsonPropertyName("id")] string? Id,
[property: JsonPropertyName("articleId")] string? ArticleId,
[property: JsonPropertyName("title")] string? Title,
[property: JsonPropertyName("postingDate")] string? PostingDate,
[property: JsonPropertyName("detailUrl")] string? DetailUrl,
[property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse,
[property: JsonPropertyName("products")] IReadOnlyList<AppleIndexProductDto>? Products);
private sealed record AppleIndexProductDto(
[property: JsonPropertyName("platform")] string? Platform,
[property: JsonPropertyName("name")] string? Name,
[property: JsonPropertyName("version")] string? Version,
[property: JsonPropertyName("build")] string? Build);
public static IReadOnlyList<AppleIndexEntry> Parse(ReadOnlySpan<byte> payload, Uri baseUri)
{
if (payload.IsEmpty)
{
return Array.Empty<AppleIndexEntry>();
}
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
};
AppleIndexDocument? document;
try
{
document = JsonSerializer.Deserialize<AppleIndexDocument>(payload, options);
}
catch (JsonException)
{
return Array.Empty<AppleIndexEntry>();
}
if (document?.Updates is null || document.Updates.Count == 0)
{
return Array.Empty<AppleIndexEntry>();
}
var entries = new List<AppleIndexEntry>(document.Updates.Count);
foreach (var dto in document.Updates)
{
if (dto is null)
{
continue;
}
var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id;
if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId))
{
continue;
}
if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate))
{
continue;
}
if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate))
{
continue;
}
if (!TryResolveDetailUri(dto, baseUri, out var detailUri))
{
continue;
}
var products = dto.Products?.Select(static product => new AppleIndexProduct(
product.Platform ?? string.Empty,
product.Name ?? product.Platform ?? string.Empty,
product.Version ?? string.Empty,
product.Build ?? string.Empty))
.ToArray() ?? Array.Empty<AppleIndexProduct>();
entries.Add(new AppleIndexEntry(
id.Trim(),
dto.ArticleId!.Trim(),
dto.Title!.Trim(),
postingDate.ToUniversalTime(),
detailUri,
products,
dto.RapidSecurityResponse ?? false));
}
return entries.Count == 0 ? Array.Empty<AppleIndexEntry>() : entries;
}
private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri)
{
if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri))
{
return true;
}
if (string.IsNullOrWhiteSpace(dto.ArticleId))
{
uri = default!;
return false;
}
var article = dto.ArticleId.Trim();
if (article.Length == 0)
{
uri = default!;
return false;
}
var combined = new Uri(baseUri, article);
uri = combined;
return true;
}
}

View File

@@ -0,0 +1,281 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Packages;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Storage.Mongo.PsirtFlags;
namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal;
internal static class AppleMapper
{
public static (Advisory Advisory, PsirtFlagRecord? Flag) Map(
AppleDetailDto dto,
DocumentRecord document,
DtoRecord dtoRecord)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
var fetchProvenance = new AdvisoryProvenance(
VndrAppleConnectorPlugin.SourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime());
var mapProvenance = new AdvisoryProvenance(
VndrAppleConnectorPlugin.SourceName,
"map",
dto.AdvisoryId,
recordedAt);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affected = BuildAffected(dto, recordedAt);
var advisory = new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: "en",
published: dto.Published.ToUniversalTime(),
modified: dto.Updated?.ToUniversalTime(),
severity: null,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affected,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { fetchProvenance, mapProvenance });
PsirtFlagRecord? flag = dto.RapidSecurityResponse
? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt)
: null;
return (advisory, flag);
}
private static IReadOnlyList<string> BuildAliases(AppleDetailDto dto)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
dto.AdvisoryId,
dto.ArticleId,
};
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
set.Add(cve.Trim());
}
}
var aliases = set.ToList();
aliases.Sort(StringComparer.OrdinalIgnoreCase);
return aliases;
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt)
{
if (dto.References.Count == 0)
{
return Array.Empty<AdvisoryReference>();
}
var list = new List<AdvisoryReference>(dto.References.Count);
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
try
{
var provenance = new AdvisoryProvenance(
VndrAppleConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt);
list.Add(new AdvisoryReference(
url: reference.Url,
kind: reference.Kind,
sourceTag: null,
summary: reference.Title,
provenance: provenance));
}
catch (ArgumentException)
{
// ignore invalid URLs
}
}
if (list.Count == 0)
{
return Array.Empty<AdvisoryReference>();
}
list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url));
return list;
}
private static IReadOnlyList<AffectedPackage> BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt)
{
if (dto.Affected.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Affected.Count);
foreach (var product in dto.Affected)
{
if (string.IsNullOrWhiteSpace(product.Name))
{
continue;
}
var provenance = new[]
{
new AdvisoryProvenance(
VndrAppleConnectorPlugin.SourceName,
"affected",
product.Name,
recordedAt),
};
var ranges = BuildRanges(product, recordedAt);
var normalizedVersions = BuildNormalizedVersions(product, ranges);
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: product.Name,
platform: product.Platform,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: normalizedVersions));
}
return packages.Count == 0 ? Array.Empty<AffectedPackage>() : packages;
}
private static IReadOnlyList<AffectedVersionRange> BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build))
{
return Array.Empty<AffectedVersionRange>();
}
var provenance = new AdvisoryProvenance(
VndrAppleConnectorPlugin.SourceName,
"range",
product.Name,
recordedAt);
var extensions = new Dictionary<string, string>(StringComparer.Ordinal);
if (!string.IsNullOrWhiteSpace(product.Version))
{
extensions["apple.version.raw"] = product.Version;
}
if (!string.IsNullOrWhiteSpace(product.Build))
{
extensions["apple.build"] = product.Build;
}
if (!string.IsNullOrWhiteSpace(product.Platform))
{
extensions["apple.platform"] = product.Platform;
}
var primitives = extensions.Count == 0
? null
: new RangePrimitives(
SemVer: TryCreateSemVerPrimitive(product.Version),
Nevra: null,
Evr: null,
VendorExtensions: extensions);
var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion)
? normalizedVersion
: product.Version;
return new[]
{
new AffectedVersionRange(
rangeKind: "vendor",
introducedVersion: null,
fixedVersion: sanitizedVersion,
lastAffectedVersion: null,
rangeExpression: product.Version,
provenance: provenance,
primitives: primitives),
};
}
private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version)
{
if (string.IsNullOrWhiteSpace(version))
{
return null;
}
if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized))
{
return null;
}
// treat as fixed version, unknown introduced/last affected
return new SemVerPrimitive(
Introduced: null,
IntroducedInclusive: true,
Fixed: normalized,
FixedInclusive: true,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null);
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
AppleAffectedProductDto product,
IReadOnlyList<AffectedVersionRange> ranges)
{
if (ranges.Count == 0)
{
return Array.Empty<NormalizedVersionRule>();
}
var segments = new List<string>();
if (!string.IsNullOrWhiteSpace(product.Platform))
{
segments.Add(product.Platform.Trim());
}
if (!string.IsNullOrWhiteSpace(product.Name))
{
segments.Add(product.Name.Trim());
}
var note = segments.Count == 0 ? null : $"apple:{string.Join(':', segments)}";
var rules = new List<NormalizedVersionRule>(ranges.Count);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(note);
if (rule is not null)
{
rules.Add(rule);
}
}
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules.ToArray();
}
}

View File

@@ -0,0 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
internal static class AppleJobKinds
{
public const string Fetch = "source:vndr-apple:fetch";
public const string Parse = "source:vndr-apple:parse";
public const string Map = "source:vndr-apple:map";
}
internal sealed class AppleFetchJob : IJob
{
private readonly AppleConnector _connector;
public AppleFetchJob(AppleConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class AppleParseJob : IJob
{
private readonly AppleConnector _connector;
public AppleParseJob(AppleConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class AppleMapJob : IJob
{
private readonly AppleConnector _connector;
public AppleMapJob(AppleConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Apple.Tests")]

View File

@@ -0,0 +1,49 @@
# Apple Security Updates Connector
## Feed contract
The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes:
- `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`)
- Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline`
- Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS)
- Metadata for download packages, release notes, and signing assets
The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8
Apples new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists:
- CVE identifiers with Apples internal tracking IDs
- Product version/build applicability tables
- Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6
Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `<section data-component="security-update">` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries.
## Change detection strategy
1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches.
2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`).
3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only.
Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3
## Parsing & mapping notes
- CVE lists live inside `<ul data-testid="cve-list">` items; each `<li>` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apples component name as the package `name` and the OS build as the range primitive seed.
- Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`).
- Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`.
## Outstanding questions
- Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`.
- Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured.
## Fixture maintenance
Deterministic regression coverage lives in `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. When Apple publishes new advisories the fixtures must be refreshed using the provided helper scripts:
- Bash: `./scripts/update-apple-fixtures.sh`
- PowerShell: `./scripts/update-apple-fixtures.ps1`
Both scripts set `UPDATE_APPLE_FIXTURES=1`, touch a `.update-apple-fixtures` sentinel so test runs inside WSL propagate the flag, fetch the live HT articles referenced in `AppleFixtureManager`, sanitise the HTML, and rewrite the paired `.expected.json` DTO snapshots. Always inspect the resulting diff and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the environment variable to ensure deterministic output before committing.

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,11 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.|
|Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.|
|Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.|
|Canonical mapping & range primitives|BE-Conn-Apple|Models|**DONE** Mapper now emits SemVer-derived normalizedVersions with `apple:<platform>:<product>` notes; fixtures updated to assert canonical rules while we continue tracking multi-device coverage in follow-up tasks.<br>2025-10-11 research trail: confirmed payload aligns with `[{"scheme":"semver","type":"range","min":"<build-start>","minInclusive":true,"max":"<build-end>","maxInclusive":false,"notes":"apple:ios:17.1"}]`; continue using `notes` to surface build identifiers for storage provenance.|
|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-12)** Parser now scopes references to article content, sorts affected rows deterministically, and regenerated fixtures (125326/125328/106355/HT214108/HT215500) produce stable JSON + sanitizer HTML in English.|
|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** OpenTelemetry pipeline exports `StellaOps.Concelier.Connector.Vndr.Apple`; runbook `docs/ops/concelier-apple-operations.md` added with metrics + monitoring guidance.|
|Live HTML regression sweep|QA|Source.Common|**DONE (2025-10-12)** Captured latest support.apple.com articles for 125326/125328/106355/HT214108/HT215500, trimmed nav noise, and committed sanitized HTML + expected DTOs with invariant timestamps.|
|Fixture regeneration tooling|DevEx|Testing|**DONE (2025-10-12)** `scripts/update-apple-fixtures.(sh|ps1)` set the env flag + sentinel, forward through WSLENV, and clean up after regeneration; README references updated usage.|

View File

@@ -0,0 +1,24 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Vndr.Apple;
public sealed class VndrAppleConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "vndr-apple";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetService<AppleConnector>() is not null;
}
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<AppleConnector>();
}
}