Rename Concelier Source modules to Connector
This commit is contained in:
		| @@ -0,0 +1,143 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal sealed record ChromiumCursor( | ||||
|     DateTimeOffset? LastPublished, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings, | ||||
|     IReadOnlyDictionary<string, ChromiumFetchCacheEntry> FetchCache) | ||||
| { | ||||
|     public static ChromiumCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>(), new Dictionary<string, ChromiumFetchCacheEntry>(StringComparer.Ordinal)); | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument(); | ||||
|         if (LastPublished.HasValue) | ||||
|         { | ||||
|             document["lastPublished"] = LastPublished.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); | ||||
|         document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); | ||||
|  | ||||
|         if (FetchCache.Count > 0) | ||||
|         { | ||||
|             var cacheDocument = new BsonDocument(); | ||||
|             foreach (var (key, entry) in FetchCache) | ||||
|             { | ||||
|                 cacheDocument[key] = entry.ToBson(); | ||||
|             } | ||||
|  | ||||
|             document["fetchCache"] = cacheDocument; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static ChromiumCursor FromBsonDocument(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset? lastPublished = null; | ||||
|         if (document.TryGetValue("lastPublished", out var lastPublishedValue)) | ||||
|         { | ||||
|             lastPublished = ReadDateTime(lastPublishedValue); | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|         var fetchCache = ReadFetchCache(document); | ||||
|  | ||||
|         return new ChromiumCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); | ||||
|     } | ||||
|  | ||||
|     public ChromiumCursor WithLastPublished(DateTimeOffset? lastPublished) | ||||
|         => this with { LastPublished = lastPublished?.ToUniversalTime() }; | ||||
|  | ||||
|     public ChromiumCursor WithPendingDocuments(IEnumerable<Guid> ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() }; | ||||
|  | ||||
|     public ChromiumCursor WithPendingMappings(IEnumerable<Guid> ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() }; | ||||
|  | ||||
|     public ChromiumCursor WithFetchCache(IDictionary<string, ChromiumFetchCacheEntry> cache) | ||||
|         => this with { FetchCache = cache is null ? new Dictionary<string, ChromiumFetchCacheEntry>(StringComparer.Ordinal) : new Dictionary<string, ChromiumFetchCacheEntry>(cache, StringComparer.Ordinal) }; | ||||
|  | ||||
|     public bool TryGetFetchCache(string key, out ChromiumFetchCacheEntry entry) | ||||
|         => FetchCache.TryGetValue(key, out entry); | ||||
|  | ||||
|     private static DateTimeOffset? ReadDateTime(BsonValue value) | ||||
|     { | ||||
|         return value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return Array.Empty<Guid>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 list.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, ChromiumFetchCacheEntry> ReadFetchCache(BsonDocument document) | ||||
|     { | ||||
|         if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument) | ||||
|         { | ||||
|             return new Dictionary<string, ChromiumFetchCacheEntry>(StringComparer.Ordinal); | ||||
|         } | ||||
|  | ||||
|         var dictionary = new Dictionary<string, ChromiumFetchCacheEntry>(StringComparer.Ordinal); | ||||
|         foreach (var element in cacheDocument.Elements) | ||||
|         { | ||||
|             if (element.Value is BsonDocument entryDocument) | ||||
|             { | ||||
|                 dictionary[element.Name] = ChromiumFetchCacheEntry.FromBson(entryDocument); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return dictionary; | ||||
|     } | ||||
| } | ||||
|  | ||||
| internal sealed record ChromiumFetchCacheEntry(string Sha256) | ||||
| { | ||||
|     public static ChromiumFetchCacheEntry Empty { get; } = new(string.Empty); | ||||
|  | ||||
|     public BsonDocument ToBson() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["sha256"] = Sha256, | ||||
|         }; | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static ChromiumFetchCacheEntry FromBson(BsonDocument document) | ||||
|     { | ||||
|         var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty; | ||||
|         return new ChromiumFetchCacheEntry(sha); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,78 @@ | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal sealed record ChromiumDocumentMetadata( | ||||
|     string PostId, | ||||
|     string Title, | ||||
|     Uri DetailUrl, | ||||
|     DateTimeOffset Published, | ||||
|     DateTimeOffset? Updated, | ||||
|     string? Summary) | ||||
| { | ||||
|     private const string PostIdKey = "postId"; | ||||
|     private const string TitleKey = "title"; | ||||
|     private const string PublishedKey = "published"; | ||||
|     private const string UpdatedKey = "updated"; | ||||
|     private const string SummaryKey = "summary"; | ||||
|  | ||||
|     public static ChromiumDocumentMetadata FromDocument(DocumentRecord document) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         var metadata = document.Metadata ?? throw new InvalidOperationException("Chromium document metadata missing."); | ||||
|  | ||||
|         if (!metadata.TryGetValue(PostIdKey, out var postId) || string.IsNullOrWhiteSpace(postId)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Chromium document metadata missing postId."); | ||||
|         } | ||||
|  | ||||
|         if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Chromium document metadata missing title."); | ||||
|         } | ||||
|  | ||||
|         if (!metadata.TryGetValue(PublishedKey, out var publishedString) || !DateTimeOffset.TryParse(publishedString, out var published)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Chromium document metadata missing published timestamp."); | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset? updated = null; | ||||
|         if (metadata.TryGetValue(UpdatedKey, out var updatedString) && DateTimeOffset.TryParse(updatedString, out var updatedValue)) | ||||
|         { | ||||
|             updated = updatedValue; | ||||
|         } | ||||
|  | ||||
|         metadata.TryGetValue(SummaryKey, out var summary); | ||||
|  | ||||
|         return new ChromiumDocumentMetadata( | ||||
|             postId.Trim(), | ||||
|             title.Trim(), | ||||
|             new Uri(document.Uri, UriKind.Absolute), | ||||
|             published.ToUniversalTime(), | ||||
|             updated?.ToUniversalTime(), | ||||
|             string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); | ||||
|     } | ||||
|  | ||||
|     public static IReadOnlyDictionary<string, string> CreateMetadata(string postId, string title, DateTimeOffset published, DateTimeOffset? updated, string? summary) | ||||
|     { | ||||
|         var dictionary = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|         { | ||||
|             [PostIdKey] = postId, | ||||
|             [TitleKey] = title, | ||||
|             [PublishedKey] = published.ToUniversalTime().ToString("O"), | ||||
|         }; | ||||
|  | ||||
|         if (updated.HasValue) | ||||
|         { | ||||
|             dictionary[UpdatedKey] = updated.Value.ToUniversalTime().ToString("O"); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(summary)) | ||||
|         { | ||||
|             dictionary[SummaryKey] = summary.Trim(); | ||||
|         } | ||||
|  | ||||
|         return dictionary; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,39 @@ | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal sealed record ChromiumDto( | ||||
|     [property: JsonPropertyName("postId")] string PostId, | ||||
|     [property: JsonPropertyName("title")] string Title, | ||||
|     [property: JsonPropertyName("detailUrl")] string DetailUrl, | ||||
|     [property: JsonPropertyName("published")] DateTimeOffset Published, | ||||
|     [property: JsonPropertyName("updated")] DateTimeOffset? Updated, | ||||
|     [property: JsonPropertyName("summary")] string? Summary, | ||||
|     [property: JsonPropertyName("cves")] IReadOnlyList<string> Cves, | ||||
|     [property: JsonPropertyName("platforms")] IReadOnlyList<string> Platforms, | ||||
|     [property: JsonPropertyName("versions")] IReadOnlyList<ChromiumVersionInfo> Versions, | ||||
|     [property: JsonPropertyName("references")] IReadOnlyList<ChromiumReference> References) | ||||
| { | ||||
|     public static ChromiumDto From(ChromiumDocumentMetadata metadata, IReadOnlyList<string> cves, IReadOnlyList<string> platforms, IReadOnlyList<ChromiumVersionInfo> versions, IReadOnlyList<ChromiumReference> references) | ||||
|         => new( | ||||
|             metadata.PostId, | ||||
|             metadata.Title, | ||||
|             metadata.DetailUrl.ToString(), | ||||
|             metadata.Published, | ||||
|             metadata.Updated, | ||||
|             metadata.Summary, | ||||
|             cves, | ||||
|             platforms, | ||||
|             versions, | ||||
|             references); | ||||
| } | ||||
|  | ||||
| internal sealed record ChromiumVersionInfo( | ||||
|     [property: JsonPropertyName("platform")] string Platform, | ||||
|     [property: JsonPropertyName("channel")] string Channel, | ||||
|     [property: JsonPropertyName("version")] string Version); | ||||
|  | ||||
| internal sealed record ChromiumReference( | ||||
|     [property: JsonPropertyName("url")] string Url, | ||||
|     [property: JsonPropertyName("kind")] string Kind, | ||||
|     [property: JsonPropertyName("label")] string? Label); | ||||
| @@ -0,0 +1,24 @@ | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| public sealed record ChromiumFeedEntry( | ||||
|     string EntryId, | ||||
|     string PostId, | ||||
|     string Title, | ||||
|     Uri DetailUri, | ||||
|     DateTimeOffset Published, | ||||
|     DateTimeOffset? Updated, | ||||
|     string? Summary, | ||||
|     IReadOnlyCollection<string> Categories) | ||||
| { | ||||
|     public bool IsSecurityUpdate() | ||||
|     { | ||||
|         if (Categories.Count > 0 && Categories.Contains("Stable updates", StringComparer.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return Title.Contains("Stable Channel Update", StringComparison.OrdinalIgnoreCase) | ||||
|             || Title.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase) | ||||
|             || Title.Contains("Stable Channel Desktop", StringComparison.OrdinalIgnoreCase); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,147 @@ | ||||
| using System.ServiceModel.Syndication; | ||||
| using System.Xml; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| public sealed class ChromiumFeedLoader | ||||
| { | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly ChromiumOptions _options; | ||||
|     private readonly ILogger<ChromiumFeedLoader> _logger; | ||||
|  | ||||
|     public ChromiumFeedLoader(IHttpClientFactory httpClientFactory, IOptions<ChromiumOptions> options, ILogger<ChromiumFeedLoader> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<IReadOnlyList<ChromiumFeedEntry>> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = _httpClientFactory.CreateClient(ChromiumOptions.HttpClientName); | ||||
|         var results = new List<ChromiumFeedEntry>(); | ||||
|         var startIndex = 1; | ||||
|  | ||||
|         for (var page = 0; page < _options.MaxFeedPages; page++) | ||||
|         { | ||||
|             var requestUri = BuildRequestUri(startIndex); | ||||
|             using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); | ||||
|             response.EnsureSuccessStatusCode(); | ||||
|  | ||||
|             await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); | ||||
|             using var reader = XmlReader.Create(stream); | ||||
|             var feed = SyndicationFeed.Load(reader); | ||||
|             if (feed is null || feed.Items is null) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var pageEntries = new List<ChromiumFeedEntry>(); | ||||
|             foreach (var entry in feed.Items) | ||||
|             { | ||||
|                 var published = entry.PublishDate != DateTimeOffset.MinValue | ||||
|                     ? entry.PublishDate.ToUniversalTime() | ||||
|                     : entry.LastUpdatedTime.ToUniversalTime(); | ||||
|  | ||||
|                 if (published > windowEnd || published < windowStart - _options.WindowOverlap) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var detailUri = entry.Links.FirstOrDefault(link => string.Equals(link.RelationshipType, "alternate", StringComparison.OrdinalIgnoreCase))?.Uri; | ||||
|                 if (detailUri is null) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var postId = ExtractPostId(detailUri); | ||||
|                 if (string.IsNullOrEmpty(postId)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var categories = entry.Categories.Select(static cat => cat.Name).Where(static name => !string.IsNullOrWhiteSpace(name)).ToArray(); | ||||
|                 var chromiumEntry = new ChromiumFeedEntry( | ||||
|                     entry.Id ?? detailUri.ToString(), | ||||
|                     postId, | ||||
|                     entry.Title?.Text?.Trim() ?? postId, | ||||
|                     detailUri, | ||||
|                     published, | ||||
|                     entry.LastUpdatedTime == DateTimeOffset.MinValue ? null : entry.LastUpdatedTime.ToUniversalTime(), | ||||
|                     entry.Summary?.Text?.Trim(), | ||||
|                     categories); | ||||
|  | ||||
|                 if (chromiumEntry.Published >= windowStart && chromiumEntry.Published <= windowEnd) | ||||
|                 { | ||||
|                     pageEntries.Add(chromiumEntry); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             if (pageEntries.Count == 0) | ||||
|             { | ||||
|                 var oldest = feed.Items?.Select(static item => item.PublishDate).Where(static dt => dt != DateTimeOffset.MinValue).OrderBy(static dt => dt).FirstOrDefault(); | ||||
|                 if (oldest.HasValue && oldest.Value.ToUniversalTime() < windowStart) | ||||
|                 { | ||||
|                     break; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             results.AddRange(pageEntries); | ||||
|  | ||||
|             if (feed.Items?.Any() != true) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var nextLink = feed.Links?.FirstOrDefault(link => string.Equals(link.RelationshipType, "next", StringComparison.OrdinalIgnoreCase))?.Uri; | ||||
|             if (nextLink is null) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             startIndex += _options.MaxEntriesPerPage; | ||||
|         } | ||||
|  | ||||
|         return results | ||||
|             .DistinctBy(static entry => entry.DetailUri) | ||||
|             .OrderBy(static entry => entry.Published) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private Uri BuildRequestUri(int startIndex) | ||||
|     { | ||||
|         var builder = new UriBuilder(_options.FeedUri); | ||||
|         var query = new List<string>(); | ||||
|  | ||||
|         if (!string.IsNullOrEmpty(builder.Query)) | ||||
|         { | ||||
|             query.Add(builder.Query.TrimStart('?')); | ||||
|         } | ||||
|  | ||||
|         query.Add($"max-results={_options.MaxEntriesPerPage}"); | ||||
|         query.Add($"start-index={startIndex}"); | ||||
|         query.Add("redirect=false"); | ||||
|         builder.Query = string.Join('&', query); | ||||
|         return builder.Uri; | ||||
|     } | ||||
|  | ||||
|     private static string ExtractPostId(Uri detailUri) | ||||
|     { | ||||
|         var segments = detailUri.Segments; | ||||
|         if (segments.Length == 0) | ||||
|         { | ||||
|             return detailUri.AbsoluteUri; | ||||
|         } | ||||
|  | ||||
|         var last = segments[^1].Trim('/'); | ||||
|         if (last.EndsWith(".html", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             last = last[..^5]; | ||||
|         } | ||||
|  | ||||
|         return last.Replace('/', '-'); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,174 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Globalization; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Storage.Mongo.PsirtFlags; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal static class ChromiumMapper | ||||
| { | ||||
|     private const string VendorIdentifier = "google:chrome"; | ||||
|  | ||||
|     public static (Advisory Advisory, PsirtFlagRecord Flag) Map(ChromiumDto dto, string sourceName, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentException.ThrowIfNullOrEmpty(sourceName); | ||||
|  | ||||
|         var advisoryKey = $"chromium/post/{dto.PostId}"; | ||||
|         var provenance = new AdvisoryProvenance(sourceName, "document", dto.PostId, recordedAt.ToUniversalTime()); | ||||
|  | ||||
|         var aliases = BuildAliases(dto).ToArray(); | ||||
|         var references = BuildReferences(dto, provenance).ToArray(); | ||||
|         var affectedPackages = BuildAffected(dto, provenance).ToArray(); | ||||
|  | ||||
|         var advisory = new Advisory( | ||||
|             advisoryKey, | ||||
|             dto.Title, | ||||
|             dto.Summary, | ||||
|             language: "en", | ||||
|             dto.Published.ToUniversalTime(), | ||||
|             dto.Updated?.ToUniversalTime(), | ||||
|             severity: null, | ||||
|             exploitKnown: false, | ||||
|             aliases, | ||||
|             references, | ||||
|             affectedPackages, | ||||
|             Array.Empty<CvssMetric>(), | ||||
|             new[] { provenance }); | ||||
|  | ||||
|         var flag = new PsirtFlagRecord( | ||||
|             advisoryKey, | ||||
|             "Google", | ||||
|             sourceName, | ||||
|             dto.PostId, | ||||
|             recordedAt.ToUniversalTime()); | ||||
|  | ||||
|         return (advisory, flag); | ||||
|     } | ||||
|  | ||||
|     private static IEnumerable<string> BuildAliases(ChromiumDto dto) | ||||
|     { | ||||
|         yield return $"CHROMIUM-POST:{dto.PostId}"; | ||||
|         yield return $"CHROMIUM-POST:{dto.Published:yyyy-MM-dd}"; | ||||
|  | ||||
|         foreach (var cve in dto.Cves) | ||||
|         { | ||||
|             yield return cve; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static IEnumerable<AdvisoryReference> BuildReferences(ChromiumDto dto, AdvisoryProvenance provenance) | ||||
|     { | ||||
|         var comparer = StringComparer.OrdinalIgnoreCase; | ||||
|         var references = new List<(AdvisoryReference Reference, int Priority)> | ||||
|         { | ||||
|             (new AdvisoryReference(dto.DetailUrl, "advisory", "chromium-blog", summary: null, provenance), 0), | ||||
|         }; | ||||
|  | ||||
|         foreach (var reference in dto.References) | ||||
|         { | ||||
|             var summary = string.IsNullOrWhiteSpace(reference.Label) ? null : reference.Label; | ||||
|             var sourceTag = string.IsNullOrWhiteSpace(reference.Kind) ? null : reference.Kind; | ||||
|             var advisoryReference = new AdvisoryReference(reference.Url, reference.Kind, sourceTag, summary, provenance); | ||||
|             references.Add((advisoryReference, 1)); | ||||
|         } | ||||
|  | ||||
|         return references | ||||
|             .GroupBy(tuple => tuple.Reference.Url, comparer) | ||||
|             .Select(group => group | ||||
|                 .OrderBy(t => t.Priority) | ||||
|                 .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) | ||||
|                 .ThenBy(t => t.Reference.SourceTag ?? string.Empty, comparer) | ||||
|                 .ThenBy(t => t.Reference.Url, comparer) | ||||
|                 .First()) | ||||
|             .OrderBy(t => t.Priority) | ||||
|             .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) | ||||
|             .ThenBy(t => t.Reference.Url, comparer) | ||||
|             .Select(t => t.Reference); | ||||
|     } | ||||
|  | ||||
|     private static IEnumerable<AffectedPackage> BuildAffected(ChromiumDto dto, AdvisoryProvenance provenance) | ||||
|     { | ||||
|         foreach (var version in dto.Versions) | ||||
|         { | ||||
|             var identifier = version.Channel switch | ||||
|             { | ||||
|                 "extended-stable" => $"{VendorIdentifier}:extended-stable", | ||||
|                 "beta" => $"{VendorIdentifier}:beta", | ||||
|                 "dev" => $"{VendorIdentifier}:dev", | ||||
|                 _ => VendorIdentifier, | ||||
|             }; | ||||
|  | ||||
|             var range = new AffectedVersionRange( | ||||
|                 rangeKind: "vendor", | ||||
|                 introducedVersion: null, | ||||
|                 fixedVersion: version.Version, | ||||
|                 lastAffectedVersion: null, | ||||
|                 rangeExpression: null, | ||||
|                 provenance, | ||||
|                 primitives: BuildRangePrimitives(version)); | ||||
|  | ||||
|             yield return new AffectedPackage( | ||||
|                 AffectedPackageTypes.Vendor, | ||||
|                 identifier, | ||||
|                 version.Platform, | ||||
|                 new[] { range }, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: new[] { provenance }); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static RangePrimitives? BuildRangePrimitives(ChromiumVersionInfo version) | ||||
|     { | ||||
|         var extensions = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         AddExtension(extensions, "chromium.channel", version.Channel); | ||||
|         AddExtension(extensions, "chromium.platform", version.Platform); | ||||
|         AddExtension(extensions, "chromium.version.raw", version.Version); | ||||
|  | ||||
|         if (Version.TryParse(version.Version, out var parsed)) | ||||
|         { | ||||
|             AddExtension(extensions, "chromium.version.normalized", BuildNormalizedVersion(parsed)); | ||||
|             extensions["chromium.version.major"] = parsed.Major.ToString(CultureInfo.InvariantCulture); | ||||
|             extensions["chromium.version.minor"] = parsed.Minor.ToString(CultureInfo.InvariantCulture); | ||||
|  | ||||
|             if (parsed.Build >= 0) | ||||
|             { | ||||
|                 extensions["chromium.version.build"] = parsed.Build.ToString(CultureInfo.InvariantCulture); | ||||
|             } | ||||
|  | ||||
|             if (parsed.Revision >= 0) | ||||
|             { | ||||
|                 extensions["chromium.version.patch"] = parsed.Revision.ToString(CultureInfo.InvariantCulture); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return extensions.Count == 0 ? null : new RangePrimitives(null, null, null, extensions); | ||||
|     } | ||||
|  | ||||
|     private static string BuildNormalizedVersion(Version version) | ||||
|     { | ||||
|         if (version.Build >= 0 && version.Revision >= 0) | ||||
|         { | ||||
|             return $"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"; | ||||
|         } | ||||
|  | ||||
|         if (version.Build >= 0) | ||||
|         { | ||||
|             return $"{version.Major}.{version.Minor}.{version.Build}"; | ||||
|         } | ||||
|  | ||||
|         return $"{version.Major}.{version.Minor}"; | ||||
|     } | ||||
|  | ||||
|     private static void AddExtension(Dictionary<string, string> extensions, string key, string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         extensions[key] = value.Trim(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,282 @@ | ||||
| using System.Text.RegularExpressions; | ||||
| using AngleSharp.Dom; | ||||
| using AngleSharp.Html.Parser; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal static class ChromiumParser | ||||
| { | ||||
|     private static readonly HtmlParser HtmlParser = new(); | ||||
|     private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); | ||||
|     private static readonly Regex VersionRegex = new("(?<version>\\d+\\.\\d+\\.\\d+\\.\\d+)", RegexOptions.Compiled); | ||||
|  | ||||
|     public static ChromiumDto Parse(string html, ChromiumDocumentMetadata metadata) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrEmpty(html); | ||||
|         ArgumentNullException.ThrowIfNull(metadata); | ||||
|  | ||||
|         var document = HtmlParser.ParseDocument(html); | ||||
|         var body = document.QuerySelector("div.post-body") ?? document.Body; | ||||
|         if (body is null) | ||||
|         { | ||||
|             throw new InvalidOperationException("Chromium post body not found."); | ||||
|         } | ||||
|  | ||||
|         var cves = ExtractCves(body); | ||||
|         var versions = ExtractVersions(body); | ||||
|         var platforms = versions.Select(static v => v.Platform).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); | ||||
|         var references = ExtractReferences(body, metadata.DetailUrl); | ||||
|  | ||||
|         return ChromiumDto.From(metadata, cves, platforms, versions, references); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> ExtractCves(IElement body) | ||||
|     { | ||||
|         var matches = CveRegex.Matches(body.TextContent ?? string.Empty); | ||||
|         return matches | ||||
|             .Select(static match => match.Value.ToUpperInvariant()) | ||||
|             .Distinct(StringComparer.Ordinal) | ||||
|             .OrderBy(static cve => cve, StringComparer.Ordinal) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<ChromiumVersionInfo> ExtractVersions(IElement body) | ||||
|     { | ||||
|         var results = new Dictionary<string, ChromiumVersionInfo>(StringComparer.OrdinalIgnoreCase); | ||||
|         var elements = body.QuerySelectorAll("p,li"); | ||||
|         if (elements.Length == 0) | ||||
|         { | ||||
|             elements = body.QuerySelectorAll("div,span"); | ||||
|         } | ||||
|  | ||||
|         foreach (var element in elements) | ||||
|         { | ||||
|             var text = element.TextContent?.Trim(); | ||||
|             if (string.IsNullOrEmpty(text)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var channel = DetermineChannel(text); | ||||
|             foreach (Match match in VersionRegex.Matches(text)) | ||||
|             { | ||||
|                 var version = match.Groups["version"].Value; | ||||
|                 var platform = DeterminePlatform(text, match); | ||||
|                 var key = string.Join('|', platform.ToLowerInvariant(), channel.ToLowerInvariant(), version); | ||||
|                 if (!results.ContainsKey(key)) | ||||
|                 { | ||||
|                     results[key] = new ChromiumVersionInfo(platform, channel, version); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results.Values | ||||
|             .OrderBy(static v => v.Platform, StringComparer.OrdinalIgnoreCase) | ||||
|             .ThenBy(static v => v.Channel, StringComparer.OrdinalIgnoreCase) | ||||
|             .ThenBy(static v => v.Version, StringComparer.Ordinal) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string DeterminePlatform(string text, Match match) | ||||
|     { | ||||
|         var after = ExtractSlice(text, match.Index + match.Length, Math.Min(120, text.Length - (match.Index + match.Length))); | ||||
|         var segment = ExtractPlatformSegment(after); | ||||
|         var normalized = NormalizePlatform(segment); | ||||
|         if (!string.IsNullOrEmpty(normalized)) | ||||
|         { | ||||
|             return normalized!; | ||||
|         } | ||||
|  | ||||
|         var before = ExtractSlice(text, Math.Max(0, match.Index - 80), Math.Min(80, match.Index)); | ||||
|         normalized = NormalizePlatform(before + " " + after); | ||||
|         return string.IsNullOrEmpty(normalized) ? "desktop" : normalized!; | ||||
|     } | ||||
|  | ||||
|     private static string DetermineChannel(string text) | ||||
|     { | ||||
|         if (text.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "extended-stable"; | ||||
|         } | ||||
|  | ||||
|         if (text.Contains("Beta", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "beta"; | ||||
|         } | ||||
|  | ||||
|         if (text.Contains("Dev", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "dev"; | ||||
|         } | ||||
|  | ||||
|         return "stable"; | ||||
|     } | ||||
|  | ||||
|     private static string ExtractSlice(string text, int start, int length) | ||||
|     { | ||||
|         if (length <= 0) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         return text.Substring(start, length); | ||||
|     } | ||||
|  | ||||
|     private static string ExtractPlatformSegment(string after) | ||||
|     { | ||||
|         if (string.IsNullOrEmpty(after)) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         var forIndex = after.IndexOf("for ", StringComparison.OrdinalIgnoreCase); | ||||
|         if (forIndex < 0) | ||||
|         { | ||||
|             return string.Empty; | ||||
|         } | ||||
|  | ||||
|         var remainder = after[(forIndex + 4)..]; | ||||
|         var terminatorIndex = remainder.IndexOfAny(new[] { '.', ';', '\n', '(', ')' }); | ||||
|         if (terminatorIndex >= 0) | ||||
|         { | ||||
|             remainder = remainder[..terminatorIndex]; | ||||
|         } | ||||
|  | ||||
|         var digitIndex = remainder.IndexOfAny("0123456789".ToCharArray()); | ||||
|         if (digitIndex >= 0) | ||||
|         { | ||||
|             remainder = remainder[..digitIndex]; | ||||
|         } | ||||
|  | ||||
|         var whichIndex = remainder.IndexOf(" which", StringComparison.OrdinalIgnoreCase); | ||||
|         if (whichIndex >= 0) | ||||
|         { | ||||
|             remainder = remainder[..whichIndex]; | ||||
|         } | ||||
|  | ||||
|         return remainder.Trim(); | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizePlatform(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var normalized = value.Replace("/", " ", StringComparison.OrdinalIgnoreCase) | ||||
|             .Replace(" and ", " ", StringComparison.OrdinalIgnoreCase) | ||||
|             .Replace("&", " ", StringComparison.OrdinalIgnoreCase) | ||||
|             .Trim(); | ||||
|  | ||||
|         if (normalized.Contains("android", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "android"; | ||||
|         } | ||||
|  | ||||
|         if (normalized.Contains("chromeos flex", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "chromeos-flex"; | ||||
|         } | ||||
|  | ||||
|         if (normalized.Contains("chromeos", StringComparison.OrdinalIgnoreCase) || normalized.Contains("chrome os", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "chromeos"; | ||||
|         } | ||||
|  | ||||
|         if (normalized.Contains("linux", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "linux"; | ||||
|         } | ||||
|  | ||||
|         var hasWindows = normalized.Contains("windows", StringComparison.OrdinalIgnoreCase); | ||||
|         var hasMac = normalized.Contains("mac", StringComparison.OrdinalIgnoreCase); | ||||
|  | ||||
|         if (hasWindows && hasMac) | ||||
|         { | ||||
|             return "windows-mac"; | ||||
|         } | ||||
|  | ||||
|         if (hasWindows) | ||||
|         { | ||||
|             return "windows"; | ||||
|         } | ||||
|  | ||||
|         if (hasMac) | ||||
|         { | ||||
|             return "mac"; | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<ChromiumReference> ExtractReferences(IElement body, Uri detailUri) | ||||
|     { | ||||
|         var references = new Dictionary<string, ChromiumReference>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var anchor in body.QuerySelectorAll("a[href]")) | ||||
|         { | ||||
|             var href = anchor.GetAttribute("href"); | ||||
|             if (string.IsNullOrWhiteSpace(href)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!Uri.TryCreate(href.Trim(), UriKind.Absolute, out var linkUri)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (string.Equals(linkUri.AbsoluteUri, detailUri.AbsoluteUri, StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!string.Equals(linkUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) | ||||
|                 && !string.Equals(linkUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var kind = ClassifyReference(linkUri); | ||||
|             var label = anchor.TextContent?.Trim(); | ||||
|  | ||||
|             if (!references.ContainsKey(linkUri.AbsoluteUri)) | ||||
|             { | ||||
|                 references[linkUri.AbsoluteUri] = new ChromiumReference(linkUri.AbsoluteUri, kind, string.IsNullOrWhiteSpace(label) ? null : label); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return references.Values | ||||
|             .OrderBy(static r => r.Url, StringComparer.Ordinal) | ||||
|             .ThenBy(static r => r.Kind, StringComparer.Ordinal) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string ClassifyReference(Uri uri) | ||||
|     { | ||||
|         var host = uri.Host; | ||||
|         if (host.Contains("googlesource.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "changelog"; | ||||
|         } | ||||
|  | ||||
|         if (host.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase) | ||||
|             || host.Contains("bugs.chromium.org", StringComparison.OrdinalIgnoreCase) | ||||
|             || host.Contains("crbug.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "bug"; | ||||
|         } | ||||
|  | ||||
|         if (host.Contains("chromium.org", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "doc"; | ||||
|         } | ||||
|  | ||||
|         if (host.Contains("google.com", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return "google"; | ||||
|         } | ||||
|  | ||||
|         return "reference"; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,25 @@ | ||||
| using System.IO; | ||||
| using System.Reflection; | ||||
| using System.Threading; | ||||
| using Json.Schema; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; | ||||
|  | ||||
| internal static class ChromiumSchemaProvider | ||||
| { | ||||
|     private static readonly Lazy<JsonSchema> Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication); | ||||
|  | ||||
|     public static JsonSchema Schema => Cached.Value; | ||||
|  | ||||
|     private static JsonSchema Load() | ||||
|     { | ||||
|         var assembly = typeof(ChromiumSchemaProvider).GetTypeInfo().Assembly; | ||||
|         const string resourceName = "StellaOps.Concelier.Connector.Vndr.Chromium.Schemas.chromium-post.schema.json"; | ||||
|  | ||||
|         using var stream = assembly.GetManifestResourceStream(resourceName) | ||||
|             ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); | ||||
|         using var reader = new StreamReader(stream); | ||||
|         var schemaText = reader.ReadToEnd(); | ||||
|         return JsonSchema.FromText(schemaText); | ||||
|     } | ||||
| } | ||||
		Reference in New Issue
	
	Block a user