up
This commit is contained in:
		| @@ -1,29 +0,0 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| public sealed class VndrMsrcConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public string Name => "vndr-msrc"; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => true; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); | ||||
|  | ||||
|     private sealed class StubConnector : IFeedConnector | ||||
|     { | ||||
|         public StubConnector(string sourceName) => SourceName = sourceName; | ||||
|  | ||||
|         public string SourceName { get; } | ||||
|  | ||||
|         public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|     } | ||||
| } | ||||
|  | ||||
| @@ -0,0 +1,132 @@ | ||||
| using System.Globalization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
|  | ||||
| public sealed class MsrcOptions | ||||
| { | ||||
|     public const string HttpClientName = "feedser.source.vndr.msrc"; | ||||
|     public const string TokenClientName = "feedser.source.vndr.msrc.token"; | ||||
|  | ||||
|     public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute); | ||||
|  | ||||
|     public string Locale { get; set; } = "en-US"; | ||||
|  | ||||
|     public string ApiVersion { get; set; } = "2024-08-01"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Azure AD tenant identifier used for client credential flow. | ||||
|     /// </summary> | ||||
|     public string TenantId { get; set; } = string.Empty; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Azure AD application (client) identifier. | ||||
|     /// </summary> | ||||
|     public string ClientId { get; set; } = string.Empty; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Azure AD client secret used for token acquisition. | ||||
|     /// </summary> | ||||
|     public string ClientSecret { get; set; } = string.Empty; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Scope requested during client-credential token acquisition. | ||||
|     /// </summary> | ||||
|     public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Maximum advisories to fetch per cycle. | ||||
|     /// </summary> | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 200; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Page size used when iterating the MSRC API. | ||||
|     /// </summary> | ||||
|     public int PageSize { get; set; } = 100; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Overlap window added when resuming from the last modified cursor. | ||||
|     /// </summary> | ||||
|     public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// When enabled the connector downloads the CVRF artefact referenced by each advisory. | ||||
|     /// </summary> | ||||
|     public bool DownloadCvrf { get; set; } = false; | ||||
|  | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); | ||||
|  | ||||
|     public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional lower bound for the initial sync if the cursor is empty. | ||||
|     /// </summary> | ||||
|     public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30); | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (BaseUri is null || !BaseUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("MSRC base URI must be absolute."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(Locale)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Locale must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase))) | ||||
|         { | ||||
|             throw new InvalidOperationException($"Locale '{Locale}' is not recognised."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ApiVersion)) | ||||
|         { | ||||
|             throw new InvalidOperationException("API version must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (!Guid.TryParse(TenantId, out _)) | ||||
|         { | ||||
|             throw new InvalidOperationException("TenantId must be a valid GUID."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ClientId)) | ||||
|         { | ||||
|             throw new InvalidOperationException("ClientId must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ClientSecret)) | ||||
|         { | ||||
|             throw new InvalidOperationException("ClientSecret must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(Scope)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Scope must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (PageSize <= 0 || PageSize > 500) | ||||
|         { | ||||
|             throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500."); | ||||
|         } | ||||
|  | ||||
|         if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6)) | ||||
|         { | ||||
|             throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (FailureBackoff <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,49 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed record MsrcAdvisoryDto | ||||
| { | ||||
|     public string AdvisoryId { get; init; } = string.Empty; | ||||
|  | ||||
|     public string Title { get; init; } = string.Empty; | ||||
|  | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     public string? Severity { get; init; } | ||||
|  | ||||
|     public DateTimeOffset? ReleaseDate { get; init; } | ||||
|  | ||||
|     public DateTimeOffset? LastModifiedDate { get; init; } | ||||
|  | ||||
|     public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>(); | ||||
|  | ||||
|     public IReadOnlyList<string> KbIds { get; init; } = Array.Empty<string>(); | ||||
|  | ||||
|     public IReadOnlyList<MsrcAdvisoryThreat> Threats { get; init; } = Array.Empty<MsrcAdvisoryThreat>(); | ||||
|  | ||||
|     public IReadOnlyList<MsrcAdvisoryRemediation> Remediations { get; init; } = Array.Empty<MsrcAdvisoryRemediation>(); | ||||
|  | ||||
|     public IReadOnlyList<MsrcAdvisoryProduct> Products { get; init; } = Array.Empty<MsrcAdvisoryProduct>(); | ||||
|  | ||||
|     public double? CvssBaseScore { get; init; } | ||||
|  | ||||
|     public string? CvssVector { get; init; } | ||||
|  | ||||
|     public string? ReleaseNoteUrl { get; init; } | ||||
|  | ||||
|     public string? CvrfUrl { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity); | ||||
|  | ||||
| public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb); | ||||
|  | ||||
| public sealed record MsrcAdvisoryProduct( | ||||
|     string Identifier, | ||||
|     string? ProductName, | ||||
|     string? Platform, | ||||
|     string? Architecture, | ||||
|     string? BuildNumber, | ||||
|     string? Cpe); | ||||
							
								
								
									
										138
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcApiClient.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										138
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcApiClient.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,138 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Net.Http; | ||||
| using System.Net.Http.Json; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed class MsrcApiClient | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         WriteIndented = false, | ||||
|     }; | ||||
|  | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly IMsrcTokenProvider _tokenProvider; | ||||
|     private readonly MsrcOptions _options; | ||||
|     private readonly ILogger<MsrcApiClient> _logger; | ||||
|  | ||||
|     public MsrcApiClient( | ||||
|         IHttpClientFactory httpClientFactory, | ||||
|         IMsrcTokenProvider tokenProvider, | ||||
|         IOptions<MsrcOptions> options, | ||||
|         ILogger<MsrcApiClient> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<IReadOnlyList<MsrcVulnerabilitySummary>> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         var results = new List<MsrcVulnerabilitySummary>(); | ||||
|         var requestUri = BuildSummaryUri(fromInclusive, toExclusive); | ||||
|  | ||||
|         while (requestUri is not null) | ||||
|         { | ||||
|             using var request = new HttpRequestMessage(HttpMethod.Get, requestUri); | ||||
|             using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (!response.IsSuccessStatusCode) | ||||
|             { | ||||
|                 var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); | ||||
|                 throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}"); | ||||
|             } | ||||
|  | ||||
|             var payload = await response.Content.ReadFromJsonAsync<MsrcSummaryResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false) | ||||
|                 ?? new MsrcSummaryResponse(); | ||||
|  | ||||
|             results.AddRange(payload.Value); | ||||
|  | ||||
|             if (string.IsNullOrWhiteSpace(payload.NextLink)) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             requestUri = new Uri(payload.NextLink, UriKind.Absolute); | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
|  | ||||
|     public Uri BuildDetailUri(string vulnerabilityId) | ||||
|     { | ||||
|         var uri = CreateDetailUriInternal(vulnerabilityId); | ||||
|         return uri; | ||||
|     } | ||||
|  | ||||
|     public async Task<byte[]> FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var uri = CreateDetailUriInternal(vulnerabilityId); | ||||
|  | ||||
|         using var request = new HttpRequestMessage(HttpMethod.Get, uri); | ||||
|         using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (!response.IsSuccessStatusCode) | ||||
|         { | ||||
|             var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); | ||||
|             throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}"); | ||||
|         } | ||||
|  | ||||
|         return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private async Task<HttpClient> CreateAuthenticatedClientAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName); | ||||
|         client.DefaultRequestHeaders.Remove("Authorization"); | ||||
|         client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}"); | ||||
|         client.DefaultRequestHeaders.Remove("Accept"); | ||||
|         client.DefaultRequestHeaders.Add("Accept", "application/json"); | ||||
|         client.DefaultRequestHeaders.Remove("api-version"); | ||||
|         client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion); | ||||
|         client.DefaultRequestHeaders.Remove("Accept-Language"); | ||||
|         client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale); | ||||
|         return client; | ||||
|     } | ||||
|  | ||||
|     private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive) | ||||
|     { | ||||
|         var builder = new StringBuilder(); | ||||
|         builder.Append(_options.BaseUri.ToString().TrimEnd('/')); | ||||
|         builder.Append("/vulnerabilities?"); | ||||
|         builder.Append("$top=").Append(_options.PageSize); | ||||
|         builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O"))); | ||||
|         builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O"))); | ||||
|         builder.Append("&$orderby=lastModifiedDate"); | ||||
|         builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale)); | ||||
|         builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion)); | ||||
|  | ||||
|         return new Uri(builder.ToString(), UriKind.Absolute); | ||||
|     } | ||||
|  | ||||
|     private Uri CreateDetailUriInternal(string vulnerabilityId) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(vulnerabilityId)) | ||||
|         { | ||||
|             throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId)); | ||||
|         } | ||||
|  | ||||
|         var baseUri = _options.BaseUri.ToString().TrimEnd('/'); | ||||
|         var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}"; | ||||
|         return new Uri(path, UriKind.Absolute); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,87 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| internal sealed record MsrcCursor( | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings, | ||||
|     DateTimeOffset? LastModifiedCursor) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidSet = Array.Empty<Guid>(); | ||||
|  | ||||
|     public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null); | ||||
|  | ||||
|     public MsrcCursor WithPendingDocuments(IEnumerable<Guid> documents) | ||||
|         => this with { PendingDocuments = Distinct(documents) }; | ||||
|  | ||||
|     public MsrcCursor WithPendingMappings(IEnumerable<Guid> mappings) | ||||
|         => this with { PendingMappings = Distinct(mappings) }; | ||||
|  | ||||
|     public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp) | ||||
|         => this with { LastModifiedCursor = timestamp }; | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastModifiedCursor.HasValue) | ||||
|         { | ||||
|             document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static MsrcCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|         var lastModified = document.TryGetValue("lastModifiedCursor", out var value) | ||||
|             ? ParseDate(value) | ||||
|             : null; | ||||
|  | ||||
|         return new MsrcCursor(pendingDocuments, pendingMappings, lastModified); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values) | ||||
|         => values?.Distinct().ToArray() ?? EmptyGuidSet; | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidSet; | ||||
|         } | ||||
|  | ||||
|         var items = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element?.ToString(), out var id)) | ||||
|             { | ||||
|                 items.Add(id); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return items; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(BsonValue value) | ||||
|         => value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
| } | ||||
							
								
								
									
										113
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailDto.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										113
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailDto.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,113 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed record MsrcVulnerabilityDetailDto | ||||
| { | ||||
|     [JsonPropertyName("id")] | ||||
|     public string Id { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("vulnerabilityId")] | ||||
|     public string VulnerabilityId { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("cveNumber")] | ||||
|     public string? CveNumber { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cveNumbers")] | ||||
|     public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>(); | ||||
|  | ||||
|     [JsonPropertyName("title")] | ||||
|     public string Title { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("description")] | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("releaseDate")] | ||||
|     public DateTimeOffset? ReleaseDate { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("lastModifiedDate")] | ||||
|     public DateTimeOffset? LastModifiedDate { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("severity")] | ||||
|     public string? Severity { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("threats")] | ||||
|     public IReadOnlyList<MsrcThreatDto> Threats { get; init; } = Array.Empty<MsrcThreatDto>(); | ||||
|  | ||||
|     [JsonPropertyName("remediations")] | ||||
|     public IReadOnlyList<MsrcRemediationDto> Remediations { get; init; } = Array.Empty<MsrcRemediationDto>(); | ||||
|  | ||||
|     [JsonPropertyName("affectedProducts")] | ||||
|     public IReadOnlyList<MsrcAffectedProductDto> AffectedProducts { get; init; } = Array.Empty<MsrcAffectedProductDto>(); | ||||
|  | ||||
|     [JsonPropertyName("cvssV3")] | ||||
|     public MsrcCvssDto? Cvss { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("releaseNoteUrl")] | ||||
|     public string? ReleaseNoteUrl { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cvrfUrl")] | ||||
|     public string? CvrfUrl { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcThreatDto | ||||
| { | ||||
|     [JsonPropertyName("type")] | ||||
|     public string? Type { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("description")] | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("severity")] | ||||
|     public string? Severity { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcRemediationDto | ||||
| { | ||||
|     [JsonPropertyName("id")] | ||||
|     public string? Id { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("type")] | ||||
|     public string? Type { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("description")] | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("url")] | ||||
|     public string? Url { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("kbNumber")] | ||||
|     public string? KbNumber { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcAffectedProductDto | ||||
| { | ||||
|     [JsonPropertyName("productId")] | ||||
|     public string? ProductId { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("productName")] | ||||
|     public string? ProductName { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cpe")] | ||||
|     public string? Cpe { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("platform")] | ||||
|     public string? Platform { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("architecture")] | ||||
|     public string? Architecture { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("buildNumber")] | ||||
|     public string? BuildNumber { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcCvssDto | ||||
| { | ||||
|     [JsonPropertyName("baseScore")] | ||||
|     public double? BaseScore { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("vectorString")] | ||||
|     public string? VectorString { get; init; } | ||||
| } | ||||
| @@ -0,0 +1,71 @@ | ||||
| using System; | ||||
| using System.Linq; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed class MsrcDetailParser | ||||
| { | ||||
|     public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(detail); | ||||
|  | ||||
|         var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId; | ||||
|         var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray() | ||||
|             ?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty<string>() : new[] { detail.CveNumber! }); | ||||
|  | ||||
|         var kbIds = detail.Remediations? | ||||
|             .Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber)) | ||||
|             .Select(static remediation => remediation.KbNumber!.Trim()) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray() ?? Array.Empty<string>(); | ||||
|  | ||||
|         return new MsrcAdvisoryDto | ||||
|         { | ||||
|             AdvisoryId = advisoryId, | ||||
|             Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(), | ||||
|             Description = detail.Description, | ||||
|             Severity = detail.Severity, | ||||
|             ReleaseDate = detail.ReleaseDate, | ||||
|             LastModifiedDate = detail.LastModifiedDate, | ||||
|             CveIds = cveIds, | ||||
|             KbIds = kbIds, | ||||
|             Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat( | ||||
|                 threat.Type ?? "unspecified", | ||||
|                 threat.Description, | ||||
|                 threat.Severity)).ToArray() ?? Array.Empty<MsrcAdvisoryThreat>(), | ||||
|             Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation( | ||||
|                 remediation.Type ?? "unspecified", | ||||
|                 remediation.Description, | ||||
|                 remediation.Url, | ||||
|                 remediation.KbNumber)).ToArray() ?? Array.Empty<MsrcAdvisoryRemediation>(), | ||||
|             Products = detail.AffectedProducts?.Select(product => | ||||
|                 new MsrcAdvisoryProduct( | ||||
|                     BuildProductIdentifier(product), | ||||
|                     product.ProductName, | ||||
|                     product.Platform, | ||||
|                     product.Architecture, | ||||
|                     product.BuildNumber, | ||||
|                     product.Cpe)).ToArray() ?? Array.Empty<MsrcAdvisoryProduct>(), | ||||
|             CvssBaseScore = detail.Cvss?.BaseScore, | ||||
|             CvssVector = detail.Cvss?.VectorString, | ||||
|             ReleaseNoteUrl = detail.ReleaseNoteUrl, | ||||
|             CvrfUrl = detail.CvrfUrl, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static string BuildProductIdentifier(MsrcAffectedProductDto product) | ||||
|     { | ||||
|         var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName; | ||||
|         if (string.IsNullOrWhiteSpace(name)) | ||||
|         { | ||||
|             name = "Unknown Product"; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(product.BuildNumber)) | ||||
|         { | ||||
|             return $"{name} build {product.BuildNumber}"; | ||||
|         } | ||||
|  | ||||
|         return name; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,129 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed class MsrcDiagnostics : IDisposable | ||||
| { | ||||
|     private const string MeterName = "StellaOps.Feedser.Source.Vndr.Msrc"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _summaryFetchAttempts; | ||||
|     private readonly Counter<long> _summaryFetchSuccess; | ||||
|     private readonly Counter<long> _summaryFetchFailures; | ||||
|     private readonly Histogram<long> _summaryItemCount; | ||||
|     private readonly Histogram<double> _summaryWindowHours; | ||||
|     private readonly Counter<long> _detailFetchAttempts; | ||||
|     private readonly Counter<long> _detailFetchSuccess; | ||||
|     private readonly Counter<long> _detailFetchNotModified; | ||||
|     private readonly Counter<long> _detailFetchFailures; | ||||
|     private readonly Histogram<long> _detailEnqueued; | ||||
|     private readonly Counter<long> _parseSuccess; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Histogram<long> _parseProductCount; | ||||
|     private readonly Histogram<long> _parseKbCount; | ||||
|     private readonly Counter<long> _mapSuccess; | ||||
|     private readonly Counter<long> _mapFailures; | ||||
|     private readonly Histogram<long> _mapAliasCount; | ||||
|     private readonly Histogram<long> _mapAffectedCount; | ||||
|  | ||||
|     public MsrcDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _summaryFetchAttempts = _meter.CreateCounter<long>("msrc.summary.fetch.attempts", "operations"); | ||||
|         _summaryFetchSuccess = _meter.CreateCounter<long>("msrc.summary.fetch.success", "operations"); | ||||
|         _summaryFetchFailures = _meter.CreateCounter<long>("msrc.summary.fetch.failures", "operations"); | ||||
|         _summaryItemCount = _meter.CreateHistogram<long>("msrc.summary.items.count", "items"); | ||||
|         _summaryWindowHours = _meter.CreateHistogram<double>("msrc.summary.window.hours", "hours"); | ||||
|         _detailFetchAttempts = _meter.CreateCounter<long>("msrc.detail.fetch.attempts", "operations"); | ||||
|         _detailFetchSuccess = _meter.CreateCounter<long>("msrc.detail.fetch.success", "operations"); | ||||
|         _detailFetchNotModified = _meter.CreateCounter<long>("msrc.detail.fetch.not_modified", "operations"); | ||||
|         _detailFetchFailures = _meter.CreateCounter<long>("msrc.detail.fetch.failures", "operations"); | ||||
|         _detailEnqueued = _meter.CreateHistogram<long>("msrc.detail.enqueued.count", "documents"); | ||||
|         _parseSuccess = _meter.CreateCounter<long>("msrc.parse.success", "documents"); | ||||
|         _parseFailures = _meter.CreateCounter<long>("msrc.parse.failures", "documents"); | ||||
|         _parseProductCount = _meter.CreateHistogram<long>("msrc.parse.products.count", "products"); | ||||
|         _parseKbCount = _meter.CreateHistogram<long>("msrc.parse.kb.count", "kb"); | ||||
|         _mapSuccess = _meter.CreateCounter<long>("msrc.map.success", "advisories"); | ||||
|         _mapFailures = _meter.CreateCounter<long>("msrc.map.failures", "advisories"); | ||||
|         _mapAliasCount = _meter.CreateHistogram<long>("msrc.map.aliases.count", "aliases"); | ||||
|         _mapAffectedCount = _meter.CreateHistogram<long>("msrc.map.affected.count", "packages"); | ||||
|     } | ||||
|  | ||||
|     public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1); | ||||
|  | ||||
|     public void SummaryFetchSuccess(int count, double? windowHours) | ||||
|     { | ||||
|         _summaryFetchSuccess.Add(1); | ||||
|         if (count >= 0) | ||||
|         { | ||||
|             _summaryItemCount.Record(count); | ||||
|         } | ||||
|  | ||||
|         if (windowHours is { } value && value >= 0) | ||||
|         { | ||||
|             _summaryWindowHours.Record(value); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void SummaryFetchFailure(string reason) | ||||
|         => _summaryFetchFailures.Add(1, ReasonTag(reason)); | ||||
|  | ||||
|     public void DetailFetchAttempt() => _detailFetchAttempts.Add(1); | ||||
|  | ||||
|     public void DetailFetchSuccess() => _detailFetchSuccess.Add(1); | ||||
|  | ||||
|     public void DetailFetchNotModified() => _detailFetchNotModified.Add(1); | ||||
|  | ||||
|     public void DetailFetchFailure(string reason) | ||||
|         => _detailFetchFailures.Add(1, ReasonTag(reason)); | ||||
|  | ||||
|     public void DetailEnqueued(int count) | ||||
|     { | ||||
|         if (count >= 0) | ||||
|         { | ||||
|             _detailEnqueued.Record(count); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void ParseSuccess(int productCount, int kbCount) | ||||
|     { | ||||
|         _parseSuccess.Add(1); | ||||
|         if (productCount >= 0) | ||||
|         { | ||||
|             _parseProductCount.Record(productCount); | ||||
|         } | ||||
|  | ||||
|         if (kbCount >= 0) | ||||
|         { | ||||
|             _parseKbCount.Record(kbCount); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void ParseFailure(string reason) | ||||
|         => _parseFailures.Add(1, ReasonTag(reason)); | ||||
|  | ||||
|     public void MapSuccess(int aliasCount, int packageCount) | ||||
|     { | ||||
|         _mapSuccess.Add(1); | ||||
|         if (aliasCount >= 0) | ||||
|         { | ||||
|             _mapAliasCount.Record(aliasCount); | ||||
|         } | ||||
|  | ||||
|         if (packageCount >= 0) | ||||
|         { | ||||
|             _mapAffectedCount.Record(packageCount); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void MapFailure(string reason) | ||||
|         => _mapFailures.Add(1, ReasonTag(reason)); | ||||
|  | ||||
|     private static KeyValuePair<string, object?> ReasonTag(string reason) | ||||
|         => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant()); | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
| } | ||||
| @@ -0,0 +1,45 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| internal static class MsrcDocumentMetadata | ||||
| { | ||||
|     public static Dictionary<string, string> CreateMetadata(MsrcVulnerabilitySummary summary) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, | ||||
|             ["msrc.id"] = summary.Id, | ||||
|         }; | ||||
|  | ||||
|         if (summary.LastModifiedDate.HasValue) | ||||
|         { | ||||
|             metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O"); | ||||
|         } | ||||
|  | ||||
|         if (summary.ReleaseDate.HasValue) | ||||
|         { | ||||
|             metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O"); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(summary.CvrfUrl)) | ||||
|         { | ||||
|             metadata["msrc.cvrfUrl"] = summary.CvrfUrl!; | ||||
|         } | ||||
|  | ||||
|         if (summary.CveNumbers.Count > 0) | ||||
|         { | ||||
|             metadata["msrc.cves"] = string.Join(",", summary.CveNumbers); | ||||
|         } | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
|  | ||||
|     public static Dictionary<string, string> CreateCvrfMetadata(MsrcVulnerabilitySummary summary) | ||||
|     { | ||||
|         var metadata = CreateMetadata(summary); | ||||
|         metadata["msrc.cvrf"] = "true"; | ||||
|         return metadata; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										239
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcMapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										239
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcMapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,239 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| internal static class MsrcMapper | ||||
| { | ||||
|     public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         var advisoryKey = dto.AdvisoryId; | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var affectedPackages = BuildPackages(dto, recordedAt); | ||||
|         var cvssMetrics = BuildCvss(dto, recordedAt); | ||||
|  | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             source: MsrcConnectorPlugin.SourceName, | ||||
|             kind: "advisory", | ||||
|             value: advisoryKey, | ||||
|             recordedAt, | ||||
|             new[] { ProvenanceFieldMasks.Advisory }); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey: advisoryKey, | ||||
|             title: dto.Title, | ||||
|             summary: dto.Description, | ||||
|             language: "en", | ||||
|             published: dto.ReleaseDate, | ||||
|             modified: dto.LastModifiedDate, | ||||
|             severity: NormalizeSeverity(dto.Severity), | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             references: references, | ||||
|             affectedPackages: affectedPackages, | ||||
|             cvssMetrics: cvssMetrics, | ||||
|             provenance: new[] { provenance }); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> BuildAliases(MsrcAdvisoryDto dto) | ||||
|     { | ||||
|         var aliases = new List<string> { dto.AdvisoryId }; | ||||
|         foreach (var cve in dto.CveIds) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(cve)) | ||||
|             { | ||||
|                 aliases.Add(cve); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         foreach (var kb in dto.KbIds) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(kb)) | ||||
|             { | ||||
|                 aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}"); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return aliases | ||||
|             .Where(static alias => !string.IsNullOrWhiteSpace(alias)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var references = new List<AdvisoryReference>(); | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl)) | ||||
|         { | ||||
|             references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt)); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(dto.CvrfUrl)) | ||||
|         { | ||||
|             references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt)); | ||||
|         } | ||||
|  | ||||
|         foreach (var remediation in dto.Remediations) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(remediation.Url)) | ||||
|             { | ||||
|                 references.Add(CreateReference( | ||||
|                     remediation.Url!, | ||||
|                     string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference", | ||||
|                     recordedAt, | ||||
|                     remediation.Description)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return references | ||||
|             .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null) | ||||
|         => new( | ||||
|             url, | ||||
|             kind: kind.ToLowerInvariant(), | ||||
|             sourceTag: "msrc", | ||||
|             summary: summary, | ||||
|             provenance: new AdvisoryProvenance( | ||||
|                 MsrcConnectorPlugin.SourceName, | ||||
|                 "reference", | ||||
|                 url, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.References })); | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Products.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Products.Count); | ||||
|         foreach (var product in dto.Products) | ||||
|         { | ||||
|             var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier; | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 MsrcConnectorPlugin.SourceName, | ||||
|                 "package", | ||||
|                 identifier, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.AffectedPackages }); | ||||
|  | ||||
|             var notes = new List<string>(); | ||||
|             if (!string.IsNullOrWhiteSpace(product.Platform)) | ||||
|             { | ||||
|                 notes.Add($"platform:{product.Platform}"); | ||||
|             } | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(product.Architecture)) | ||||
|             { | ||||
|                 notes.Add($"arch:{product.Architecture}"); | ||||
|             } | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(product.Cpe)) | ||||
|             { | ||||
|                 notes.Add($"cpe:{product.Cpe}"); | ||||
|             } | ||||
|  | ||||
|             var range = !string.IsNullOrWhiteSpace(product.BuildNumber) | ||||
|                 ? new[] | ||||
|                 { | ||||
|                     new AffectedVersionRange( | ||||
|                         rangeKind: "custom", | ||||
|                         introducedVersion: null, | ||||
|                         fixedVersion: null, | ||||
|                         lastAffectedVersion: null, | ||||
|                         rangeExpression: $"build:{product.BuildNumber}", | ||||
|                         provenance: new AdvisoryProvenance( | ||||
|                             MsrcConnectorPlugin.SourceName, | ||||
|                             "package-range", | ||||
|                             identifier, | ||||
|                             recordedAt, | ||||
|                             new[] { ProvenanceFieldMasks.VersionRanges })), | ||||
|                 } | ||||
|                 : Array.Empty<AffectedVersionRange>(); | ||||
|  | ||||
|             var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber) | ||||
|                 ? new[] | ||||
|                 { | ||||
|                     new NormalizedVersionRule( | ||||
|                         scheme: "msrc.build", | ||||
|                         type: NormalizedVersionRuleTypes.Exact, | ||||
|                         value: product.BuildNumber, | ||||
|                         notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n)))) | ||||
|                 } | ||||
|                 : Array.Empty<NormalizedVersionRule>(); | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 type: AffectedPackageTypes.Vendor, | ||||
|                 identifier: identifier, | ||||
|                 platform: product.Platform, | ||||
|                 versionRanges: range, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: new[] { provenance }, | ||||
|                 normalizedVersions: normalizedRules)); | ||||
|         } | ||||
|  | ||||
|         return packages | ||||
|             .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) | ||||
|             .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<CvssMetric> BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector)) | ||||
|         { | ||||
|             return Array.Empty<CvssMetric>(); | ||||
|         } | ||||
|  | ||||
|         var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value); | ||||
|  | ||||
|         return new[] | ||||
|         { | ||||
|             new CvssMetric( | ||||
|                 version: "3.1", | ||||
|                 vector: dto.CvssVector!, | ||||
|                 baseScore: dto.CvssBaseScore.Value, | ||||
|                 baseSeverity: severity, | ||||
|                 provenance: new AdvisoryProvenance( | ||||
|                     MsrcConnectorPlugin.SourceName, | ||||
|                     "cvss", | ||||
|                     dto.AdvisoryId, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.CvssMetrics })), | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static string CvssSeverityFromScore(double score) | ||||
|         => score switch | ||||
|         { | ||||
|             < 0 => "none", | ||||
|             < 4 => "low", | ||||
|             < 7 => "medium", | ||||
|             < 9 => "high", | ||||
|             _ => "critical", | ||||
|         }; | ||||
|  | ||||
|     private static string? NormalizeSeverity(string? severity) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(severity)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return severity.Trim().ToLowerInvariant(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,47 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public sealed record MsrcSummaryResponse | ||||
| { | ||||
|     [JsonPropertyName("value")] | ||||
|     public List<MsrcVulnerabilitySummary> Value { get; init; } = new(); | ||||
|  | ||||
|     [JsonPropertyName("@odata.nextLink")] | ||||
|     public string? NextLink { get; init; } | ||||
| } | ||||
|  | ||||
| public sealed record MsrcVulnerabilitySummary | ||||
| { | ||||
|     [JsonPropertyName("id")] | ||||
|     public string Id { get; init; } = string.Empty; | ||||
|  | ||||
|     [JsonPropertyName("vulnerabilityId")] | ||||
|     public string? VulnerabilityId { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cveNumber")] | ||||
|     public string? CveNumber { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cveNumbers")] | ||||
|     public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>(); | ||||
|  | ||||
|     [JsonPropertyName("title")] | ||||
|     public string? Title { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("description")] | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("releaseDate")] | ||||
|     public DateTimeOffset? ReleaseDate { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("lastModifiedDate")] | ||||
|     public DateTimeOffset? LastModifiedDate { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("severity")] | ||||
|     public string? Severity { get; init; } | ||||
|  | ||||
|     [JsonPropertyName("cvrfUrl")] | ||||
|     public string? CvrfUrl { get; init; } | ||||
| } | ||||
| @@ -0,0 +1,106 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Net.Http; | ||||
| using System.Net.Http.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| public interface IMsrcTokenProvider | ||||
| { | ||||
|     Task<string> GetAccessTokenAsync(CancellationToken cancellationToken); | ||||
| } | ||||
|  | ||||
| public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable | ||||
| { | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly MsrcOptions _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<MsrcTokenProvider> _logger; | ||||
|     private readonly SemaphoreSlim _refreshLock = new(1, 1); | ||||
|  | ||||
|     private AccessToken? _currentToken; | ||||
|  | ||||
|     public MsrcTokenProvider( | ||||
|         IHttpClientFactory httpClientFactory, | ||||
|         IOptions<MsrcOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<MsrcTokenProvider> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var token = _currentToken; | ||||
|         if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) | ||||
|         { | ||||
|             return token.Token; | ||||
|         } | ||||
|  | ||||
|         await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); | ||||
|         try | ||||
|         { | ||||
|             token = _currentToken; | ||||
|             if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) | ||||
|             { | ||||
|                 return token.Token; | ||||
|             } | ||||
|  | ||||
|             _logger.LogInformation("Requesting new MSRC access token"); | ||||
|             var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName); | ||||
|             var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) | ||||
|             { | ||||
|                 Content = new FormUrlEncodedContent(new Dictionary<string, string> | ||||
|                 { | ||||
|                     ["client_id"] = _options.ClientId, | ||||
|                     ["client_secret"] = _options.ClientSecret, | ||||
|                     ["grant_type"] = "client_credentials", | ||||
|                     ["scope"] = _options.Scope, | ||||
|                 }), | ||||
|             }; | ||||
|  | ||||
|             using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|             response.EnsureSuccessStatusCode(); | ||||
|  | ||||
|             var payload = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false) | ||||
|                 ?? throw new InvalidOperationException("AAD token response was null."); | ||||
|  | ||||
|             var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60); | ||||
|             _currentToken = new AccessToken(payload.AccessToken, expiresAt); | ||||
|             return payload.AccessToken; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             _refreshLock.Release(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private Uri BuildTokenUri() | ||||
|         => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); | ||||
|  | ||||
|     public void Dispose() => _refreshLock.Dispose(); | ||||
|  | ||||
|     private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt) | ||||
|     { | ||||
|         public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; | ||||
|     } | ||||
|  | ||||
|     private sealed record TokenResponse | ||||
|     { | ||||
|         [JsonPropertyName("access_token")] | ||||
|         public string AccessToken { get; init; } = string.Empty; | ||||
|  | ||||
|         [JsonPropertyName("expires_in")] | ||||
|         public int ExpiresIn { get; init; } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										22
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										22
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,22 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| internal static class MsrcJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:vndr.msrc:fetch"; | ||||
| } | ||||
|  | ||||
| internal sealed class MsrcFetchJob : IJob | ||||
| { | ||||
|     private readonly MsrcConnector _connector; | ||||
|  | ||||
|     public MsrcFetchJob(MsrcConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
							
								
								
									
										447
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										447
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,447 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Security.Cryptography; | ||||
| using System.Net.Http; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| public sealed class MsrcConnector : IFeedConnector | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         WriteIndented = false, | ||||
|     }; | ||||
|  | ||||
|     private readonly MsrcApiClient _apiClient; | ||||
|     private readonly MsrcDetailParser _detailParser; | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly MsrcOptions _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<MsrcConnector> _logger; | ||||
|     private readonly MsrcDiagnostics _diagnostics; | ||||
|  | ||||
|     public MsrcConnector( | ||||
|         MsrcApiClient apiClient, | ||||
|         MsrcDetailParser detailParser, | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IOptions<MsrcOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         MsrcDiagnostics diagnostics, | ||||
|         ILogger<MsrcConnector> logger) | ||||
|     { | ||||
|         _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); | ||||
|         _detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser)); | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => MsrcConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var from = cursor.LastModifiedCursor ?? _options.InitialLastModified ?? now.AddDays(-30); | ||||
|         from = from.Add(-_options.CursorOverlap); | ||||
|         var to = now; | ||||
|  | ||||
|         _diagnostics.SummaryFetchAttempt(); | ||||
|         IReadOnlyList<MsrcVulnerabilitySummary> summaries; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             summaries = await _apiClient.FetchSummariesAsync(from, to, cancellationToken).ConfigureAwait(false); | ||||
|             var windowHours = (to - from).TotalHours; | ||||
|             _diagnostics.SummaryFetchSuccess(summaries.Count, windowHours); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _diagnostics.SummaryFetchFailure("exception"); | ||||
|             _logger.LogError(ex, "MSRC summary fetch failed"); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         if (summaries.Count == 0) | ||||
|         { | ||||
|             await UpdateCursorAsync(cursor.WithLastModifiedCursor(to), cancellationToken).ConfigureAwait(false); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var processed = 0; | ||||
|         var failures = 0; | ||||
|  | ||||
|         foreach (var summary in summaries.OrderBy(static s => s.LastModifiedDate ?? DateTimeOffset.MinValue)) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|             if (processed >= _options.MaxAdvisoriesPerFetch) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var vulnerabilityId = string.IsNullOrWhiteSpace(summary.VulnerabilityId) ? summary.Id : summary.VulnerabilityId!; | ||||
|             var detailUri = _apiClient.BuildDetailUri(vulnerabilityId).ToString(); | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri, cancellationToken).ConfigureAwait(false); | ||||
|                 if (existing is not null && !ShouldRefresh(summary, existing)) | ||||
|                 { | ||||
|                     _diagnostics.DetailFetchNotModified(); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 _diagnostics.DetailFetchAttempt(); | ||||
|                 if (existing?.GridFsId is { } oldGridId) | ||||
|                 { | ||||
|                     await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|  | ||||
|                 var bytes = await _apiClient.FetchDetailAsync(vulnerabilityId, cancellationToken).ConfigureAwait(false); | ||||
|                 var sha = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); | ||||
|  | ||||
|                 var gridId = await _rawDocumentStorage.UploadAsync(SourceName, detailUri, bytes, "application/json", cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|                 var metadata = MsrcDocumentMetadata.CreateMetadata(summary); | ||||
|                 var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|                 { | ||||
|                     ["content-type"] = "application/json", | ||||
|                 }; | ||||
|  | ||||
|                 var documentId = existing?.Id ?? Guid.NewGuid(); | ||||
|                 var record = new DocumentRecord( | ||||
|                     documentId, | ||||
|                     SourceName, | ||||
|                     detailUri, | ||||
|                     now, | ||||
|                     sha, | ||||
|                     DocumentStatuses.PendingParse, | ||||
|                     ContentType: "application/json", | ||||
|                     Headers: headers, | ||||
|                     metadata, | ||||
|                     existing?.Etag, | ||||
|                     summary.LastModifiedDate, | ||||
|                     gridId); | ||||
|  | ||||
|                 var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|                 pendingDocuments.Add(upserted.Id); | ||||
|                 pendingMappings.Remove(upserted.Id); | ||||
|                 _diagnostics.DetailFetchSuccess(); | ||||
|                 processed++; | ||||
|  | ||||
|                 if (_options.DownloadCvrf) | ||||
|                 { | ||||
|                     await FetchCvrfAsync(summary, now, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|  | ||||
|                 if (_options.RequestDelay > TimeSpan.Zero) | ||||
|                 { | ||||
|                     await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.DetailFetchFailure("exception"); | ||||
|                 failures++; | ||||
|                 _logger.LogError(ex, "MSRC detail fetch failed for {VulnerabilityId}", vulnerabilityId); | ||||
|                 await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                 throw; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         _diagnostics.DetailEnqueued(processed); | ||||
|  | ||||
|         if (processed > 0 || failures > 0) | ||||
|         { | ||||
|             _logger.LogInformation("MSRC fetch cycle enqueued {Processed} advisories (failures={Failures}, pendingDocuments={PendingDocuments}, pendingMappings={PendingMappings})", processed, failures, pendingDocuments.Count, pendingMappings.Count); | ||||
|         } | ||||
|  | ||||
|         var latestCursor = summaries | ||||
|             .Where(static s => s.LastModifiedDate.HasValue) | ||||
|             .Select(static s => s.LastModifiedDate!.Value) | ||||
|             .DefaultIfEmpty(to) | ||||
|             .Max(); | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings) | ||||
|             .WithLastModifiedCursor(latestCursor); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remainingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure("missing_payload"); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             byte[] payload; | ||||
|             try | ||||
|             { | ||||
|                 payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure("download_failed"); | ||||
|                 _logger.LogError(ex, "MSRC unable to download document {DocumentId}", document.Id); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             MsrcVulnerabilityDetailDto? detail; | ||||
|             try | ||||
|             { | ||||
|                 detail = JsonSerializer.Deserialize<MsrcVulnerabilityDetailDto>(payload, SerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure("deserialize_failed"); | ||||
|                 _logger.LogError(ex, "MSRC failed to deserialize detail payload for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (detail is null) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure("empty_payload"); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var dto = _detailParser.Parse(detail); | ||||
|             var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); | ||||
|             var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "msrc.detail.v1", bson, now); | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|             remainingDocuments.Remove(documentId); | ||||
|             pendingMappings.Add(document.Id); | ||||
|             _diagnostics.ParseSuccess(dto.Products.Count, dto.KbIds.Count); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remainingDocuments) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (dtoRecord is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure("missing_dto"); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             MsrcAdvisoryDto? dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = JsonSerializer.Deserialize<MsrcAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure("deserialize_dto_failed"); | ||||
|                 _logger.LogError(ex, "MSRC failed to deserialize DTO for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (dto is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure("null_dto"); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var advisory = MsrcMapper.Map(dto, document, dtoRecord.ValidatedAt); | ||||
|                 await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.MapSuccess(advisory.Aliases.Length, advisory.AffectedPackages.Length); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure("exception"); | ||||
|                 _logger.LogError(ex, "MSRC mapping failed for document {DocumentId}", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private bool ShouldRefresh(MsrcVulnerabilitySummary summary, DocumentRecord existing) | ||||
|     { | ||||
|         if (existing.Status == DocumentStatuses.Failed) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (summary.LastModifiedDate is null) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (existing.Metadata is null || !existing.Metadata.TryGetValue("msrc.lastModified", out var stored)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return !string.Equals(stored, summary.LastModifiedDate.Value.ToString("O"), StringComparison.OrdinalIgnoreCase); | ||||
|     } | ||||
|  | ||||
|     private async Task<MsrcCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? MsrcCursor.Empty : MsrcCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task FetchCvrfAsync(MsrcVulnerabilitySummary summary, DateTimeOffset fetchedAt, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var uri = new Uri(summary.CvrfUrl); | ||||
|             var metadata = MsrcDocumentMetadata.CreateCvrfMetadata(summary); | ||||
|             var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); | ||||
|             var request = new SourceFetchRequest( | ||||
|                 MsrcOptions.HttpClientName, | ||||
|                 SourceName, | ||||
|                 HttpMethod.Get, | ||||
|                 uri, | ||||
|                 metadata, | ||||
|                 existing?.Etag, | ||||
|                 existing?.LastModified, | ||||
|                 AcceptHeaders: new[] { "application/zip", "application/xml", "application/json" }); | ||||
|  | ||||
|             var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|             if (result.IsNotModified || !result.IsSuccess || result.Document is null) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|             _logger.LogInformation("MSRC CVRF artefact captured for {AdvisoryId} ({Uri})", summary.VulnerabilityId ?? summary.Id, uri); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _logger.LogWarning(ex, "MSRC CVRF download failed for {CvrfUrl}", summary.CvrfUrl); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private Task UpdateCursorAsync(MsrcCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBsonDocument(); | ||||
|         var completedAt = _timeProvider.GetUtcNow(); | ||||
|         return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,21 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| public sealed class MsrcConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "vndr.msrc"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) | ||||
|         => services.GetService<MsrcConnector>() is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return services.GetRequiredService<MsrcConnector>(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,50 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "feedser:sources:vndr:msrc"; | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddMsrcConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         services.AddTransient<MsrcFetchJob>(); | ||||
|  | ||||
|         services.PostConfigure<JobSchedulerOptions>(options => | ||||
|         { | ||||
|             EnsureJob(options, MsrcJobKinds.Fetch, typeof(MsrcFetchJob)); | ||||
|         }); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
|  | ||||
|     private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) | ||||
|     { | ||||
|         if (options.Definitions.ContainsKey(kind)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         options.Definitions[kind] = new JobDefinition( | ||||
|             kind, | ||||
|             jobType, | ||||
|             options.DefaultTimeout, | ||||
|             options.DefaultLeaseDuration, | ||||
|             CronExpression: null, | ||||
|             Enabled: true); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,55 @@ | ||||
| using System; | ||||
| using System.Net; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Common.Http; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; | ||||
| using StellaOps.Feedser.Source.Vndr.Msrc.Internal; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Msrc; | ||||
|  | ||||
| public static class MsrcServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddMsrcConnector(this IServiceCollection services, Action<MsrcOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<MsrcOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static options => options.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(MsrcOptions.HttpClientName, static (provider, clientOptions) => | ||||
|         { | ||||
|             var options = provider.GetRequiredService<IOptions<MsrcOptions>>().Value; | ||||
|             clientOptions.Timeout = TimeSpan.FromSeconds(30); | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.BaseUri.Host); | ||||
|             clientOptions.AllowedHosts.Add("download.microsoft.com"); | ||||
|             clientOptions.ConfigureHandler = handler => | ||||
|             { | ||||
|                 handler.AutomaticDecompression = DecompressionMethods.All; | ||||
|             }; | ||||
|         }); | ||||
|  | ||||
|         services.AddSourceHttpClient(MsrcOptions.TokenClientName, static (_, clientOptions) => | ||||
|         { | ||||
|             clientOptions.Timeout = TimeSpan.FromSeconds(30); | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add("login.microsoftonline.com"); | ||||
|             clientOptions.ConfigureHandler = handler => | ||||
|             { | ||||
|                 handler.AutomaticDecompression = DecompressionMethods.All; | ||||
|             }; | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<IMsrcTokenProvider, MsrcTokenProvider>(); | ||||
|         services.TryAddSingleton<MsrcApiClient>(); | ||||
|         services.TryAddSingleton<MsrcDetailParser>(); | ||||
|         services.TryAddSingleton<MsrcDiagnostics>(); | ||||
|         services.AddTransient<MsrcConnector>(); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										19
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/README.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										19
									
								
								src/StellaOps.Feedser.Source.Vndr.Msrc/README.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,19 @@ | ||||
| # MSRC Security Updates – Connector Notes | ||||
|  | ||||
| ## API endpoints | ||||
| - **Vulnerability summaries** – `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerabilities` (requires `api-version=2024-08-01`, client credential bearer token). | ||||
| - **Vulnerability detail** – `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerability/{id}` (same headers/scopes). | ||||
| - **CVRF package** – the detail payload contains `cvrfUrl` pointing to a ZIP/JSON asset that is stable per revision. We surface the URL as a reference and capture it in metadata for future offline bundling. | ||||
|  | ||||
| ## Cursor behaviour | ||||
| - Connector keeps a `lastModifiedCursor` and replays the previous 10 minutes on every fetch to cover late revisions. | ||||
| - MSRC limits requests to ~60/minute; `requestDelay` defaults to 250 ms and is configurable. | ||||
|  | ||||
| ## Authentication | ||||
| - Uses Azure AD client credential flow against `https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token` with scope `api://api.msrc.microsoft.com/.default`. | ||||
| - Token refresh happens lazily and is cached until 60 seconds before expiry. | ||||
| - Configuration values (`tenantId`, `clientId`, `clientSecret`) must be supplied via `feedser:sources:vndr:msrc`. | ||||
|  | ||||
| ## CVRF handling | ||||
| - Detail payload is persisted with the `cvrfUrl` in metadata (`msrc.cvrfUrl`). | ||||
| - Mapping stage emits the CVRF link as a reference so offline runs can fetch it later. When `DownloadCvrf` is enabled the connector also saves the ZIP artefact to the documents store (marked as `msrc.cvrf=true`) for Offline Kit staging. | ||||
| @@ -2,10 +2,10 @@ | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Confirmed REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) + CVRF ZIP download flow, required Azure AD client-credentials scope (`api://api.msrc.microsoft.com/.default`), mandatory `api-version=2024-08-01` header, and delta params (`lastModifiedStartDateTime`, `lastModifiedEndDateTime`). Findings recorded in `docs/feedser-connector-research-20251011.md`.| | ||||
| |FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**TODO** – Implement fetch job that loops over `lastModifiedStartDateTime` cursor, handles `Retry-After` on throttling (default quota 60 req/min), and persists both REST JSON + optional CVRF attachments. Maintain source_state cursor at minute precision with overlap to cover delayed revisions.| | ||||
| |FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**TODO** – Extract `vulnerabilityId`, `cveNumber`, `title`, `description`, `threats[]`, `remediations[]`, KB list, CVSS data, and `affectedProducts`. Map products into package identifiers (Windows build numbers, Office version) and capture `releaseNotes` URLs as references.| | ||||
| |FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**TODO** – Map advisories to canonical records with aliases, references, range primitives for product/build coverage. Coordinate scheme naming and normalized outputs with `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: normalized array exemplar `[{"scheme":"semver","type":"range","min":"<build-start>","minInclusive":true,"max":"<build-end>","maxInclusive":false,"notes":"msrc:KB<id>"}]`; if monthly rollups require `msrc.patch` scheme, gather samples and align with Models before emitting.| | ||||
| |FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**TODO** – Add regression tests with fixtures; support `UPDATE_MSRC_FIXTURES=1`.| | ||||
| |FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics and documentation; update backlog once connector is production-ready.| | ||||
| |FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**DONE (2025-10-15)** – Added `MsrcApiClient` + token provider, cursor overlap handling, and detail persistence via GridFS (metadata carries CVRF URL + timestamps). State tracks `lastModifiedCursor` with configurable overlap/backoff. **Next:** coordinate with Tools on shared state-seeding helper once CVRF download flag stabilises.| | ||||
| |FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**DONE (2025-10-15)** – Implemented `MsrcDetailParser`/DTOs capturing threats, remediations, KB IDs, CVEs, CVSS, and affected products (build/platform metadata preserved).| | ||||
| |FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**DONE (2025-10-15)** – `MsrcMapper` emits aliases (MSRC ID/CVE/KB), references (release notes + CVRF), vendor packages with `msrc.build` normalized rules, and CVSS provenance.| | ||||
| |FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-15)** – Added `StellaOps.Feedser.Source.Vndr.Msrc.Tests` with canned token/summary/detail responses and snapshot assertions via Mongo2Go. Fixtures regenerate via `UPDATE_MSRC_FIXTURES`.| | ||||
| |FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Introduced `MsrcDiagnostics` meter (summary/detail/parse/map metrics), structured fetch logs, README updates, and Ops brief `docs/ops/feedser-msrc-operations.md` covering AAD onboarding + CVRF handling.| | ||||
| |FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Completed memo outline recommending dual-path (REST for incremental, CVRF for offline); implementation hinges on `FEEDCONN-MSRC-02-008` AAD onboarding for token acquisition.| | ||||
| |FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**TODO** – Provision MSRC SUG app registration, document client credential flow, rotation cadence, and secure storage expectations for Offline Kit deployments.| | ||||
| |FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**DONE (2025-10-15)** – Coordinated Ops handoff; drafted AAD onboarding brief (`docs/ops/feedser-msrc-operations.md`) with app registration requirements, secret rotation policy, sample configuration, and CVRF mirroring guidance for Offline Kit.| | ||||
|   | ||||
		Reference in New Issue
	
	Block a user