This commit is contained in:
Vladimir Moushkov
2025-10-15 10:03:56 +03:00
parent ea8226120c
commit ea1106ce7c
276 changed files with 21674 additions and 934 deletions

View File

@@ -1,29 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
public sealed class VndrMsrcConnectorPlugin : IConnectorPlugin
{
public string Name => "vndr-msrc";
public bool IsAvailable(IServiceProvider services) => true;
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
private sealed class StubConnector : IFeedConnector
{
public StubConnector(string sourceName) => SourceName = sourceName;
public string SourceName { get; }
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
}
}

View File

@@ -0,0 +1,132 @@
using System.Globalization;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
public sealed class MsrcOptions
{
public const string HttpClientName = "feedser.source.vndr.msrc";
public const string TokenClientName = "feedser.source.vndr.msrc.token";
public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute);
public string Locale { get; set; } = "en-US";
public string ApiVersion { get; set; } = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier used for client credential flow.
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD client secret used for token acquisition.
/// </summary>
public string ClientSecret { get; set; } = string.Empty;
/// <summary>
/// Scope requested during client-credential token acquisition.
/// </summary>
public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default";
/// <summary>
/// Maximum advisories to fetch per cycle.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Page size used when iterating the MSRC API.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Overlap window added when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// When enabled the connector downloads the CVRF artefact referenced by each advisory.
/// </summary>
public bool DownloadCvrf { get; set; } = false;
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Optional lower bound for the initial sync if the cursor is empty.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("MSRC base URI must be absolute.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("API version must be provided.");
}
if (!Guid.TryParse(TenantId, out _))
{
throw new InvalidOperationException("TenantId must be a valid GUID.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId must be provided.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret must be provided.");
}
if (string.IsNullOrWhiteSpace(Scope))
{
throw new InvalidOperationException("Scope must be provided.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}

View File

@@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed record MsrcAdvisoryDto
{
public string AdvisoryId { get; init; } = string.Empty;
public string Title { get; init; } = string.Empty;
public string? Description { get; init; }
public string? Severity { get; init; }
public DateTimeOffset? ReleaseDate { get; init; }
public DateTimeOffset? LastModifiedDate { get; init; }
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> KbIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<MsrcAdvisoryThreat> Threats { get; init; } = Array.Empty<MsrcAdvisoryThreat>();
public IReadOnlyList<MsrcAdvisoryRemediation> Remediations { get; init; } = Array.Empty<MsrcAdvisoryRemediation>();
public IReadOnlyList<MsrcAdvisoryProduct> Products { get; init; } = Array.Empty<MsrcAdvisoryProduct>();
public double? CvssBaseScore { get; init; }
public string? CvssVector { get; init; }
public string? ReleaseNoteUrl { get; init; }
public string? CvrfUrl { get; init; }
}
public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity);
public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb);
public sealed record MsrcAdvisoryProduct(
string Identifier,
string? ProductName,
string? Platform,
string? Architecture,
string? BuildNumber,
string? Cpe);

View File

@@ -0,0 +1,138 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed class MsrcApiClient
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
WriteIndented = false,
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly MsrcOptions _options;
private readonly ILogger<MsrcApiClient> _logger;
public MsrcApiClient(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IOptions<MsrcOptions> options,
ILogger<MsrcApiClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<MsrcVulnerabilitySummary>> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var results = new List<MsrcVulnerabilitySummary>();
var requestUri = BuildSummaryUri(fromInclusive, toExclusive);
while (requestUri is not null)
{
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}");
}
var payload = await response.Content.ReadFromJsonAsync<MsrcSummaryResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
results.AddRange(payload.Value);
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
requestUri = new Uri(payload.NextLink, UriKind.Absolute);
}
return results;
}
public Uri BuildDetailUri(string vulnerabilityId)
{
var uri = CreateDetailUriInternal(vulnerabilityId);
return uri;
}
public async Task<byte[]> FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var uri = CreateDetailUriInternal(vulnerabilityId);
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}");
}
return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale);
return client;
}
private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive)
{
var builder = new StringBuilder();
builder.Append(_options.BaseUri.ToString().TrimEnd('/'));
builder.Append("/vulnerabilities?");
builder.Append("$top=").Append(_options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O")));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O")));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private Uri CreateDetailUriInternal(string vulnerabilityId)
{
if (string.IsNullOrWhiteSpace(vulnerabilityId))
{
throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId));
}
var baseUri = _options.BaseUri.ToString().TrimEnd('/');
var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}";
return new Uri(path, UriKind.Absolute);
}
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
internal sealed record MsrcCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
DateTimeOffset? LastModifiedCursor)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidSet = Array.Empty<Guid>();
public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null);
public MsrcCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public MsrcCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp)
=> this with { LastModifiedCursor = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModifiedCursor.HasValue)
{
document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime;
}
return document;
}
public static MsrcCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var lastModified = document.TryGetValue("lastModifiedCursor", out var value)
? ParseDate(value)
: null;
return new MsrcCursor(pendingDocuments, pendingMappings, lastModified);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuidSet;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidSet;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,113 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed record MsrcVulnerabilityDetailDto
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string VulnerabilityId { get; init; } = string.Empty;
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("threats")]
public IReadOnlyList<MsrcThreatDto> Threats { get; init; } = Array.Empty<MsrcThreatDto>();
[JsonPropertyName("remediations")]
public IReadOnlyList<MsrcRemediationDto> Remediations { get; init; } = Array.Empty<MsrcRemediationDto>();
[JsonPropertyName("affectedProducts")]
public IReadOnlyList<MsrcAffectedProductDto> AffectedProducts { get; init; } = Array.Empty<MsrcAffectedProductDto>();
[JsonPropertyName("cvssV3")]
public MsrcCvssDto? Cvss { get; init; }
[JsonPropertyName("releaseNoteUrl")]
public string? ReleaseNoteUrl { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}
public sealed record MsrcThreatDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
}
public sealed record MsrcRemediationDto
{
[JsonPropertyName("id")]
public string? Id { get; init; }
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("kbNumber")]
public string? KbNumber { get; init; }
}
public sealed record MsrcAffectedProductDto
{
[JsonPropertyName("productId")]
public string? ProductId { get; init; }
[JsonPropertyName("productName")]
public string? ProductName { get; init; }
[JsonPropertyName("cpe")]
public string? Cpe { get; init; }
[JsonPropertyName("platform")]
public string? Platform { get; init; }
[JsonPropertyName("architecture")]
public string? Architecture { get; init; }
[JsonPropertyName("buildNumber")]
public string? BuildNumber { get; init; }
}
public sealed record MsrcCvssDto
{
[JsonPropertyName("baseScore")]
public double? BaseScore { get; init; }
[JsonPropertyName("vectorString")]
public string? VectorString { get; init; }
}

View File

@@ -0,0 +1,71 @@
using System;
using System.Linq;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed class MsrcDetailParser
{
public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail)
{
ArgumentNullException.ThrowIfNull(detail);
var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId;
var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray()
?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty<string>() : new[] { detail.CveNumber! });
var kbIds = detail.Remediations?
.Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber))
.Select(static remediation => remediation.KbNumber!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return new MsrcAdvisoryDto
{
AdvisoryId = advisoryId,
Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(),
Description = detail.Description,
Severity = detail.Severity,
ReleaseDate = detail.ReleaseDate,
LastModifiedDate = detail.LastModifiedDate,
CveIds = cveIds,
KbIds = kbIds,
Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat(
threat.Type ?? "unspecified",
threat.Description,
threat.Severity)).ToArray() ?? Array.Empty<MsrcAdvisoryThreat>(),
Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation(
remediation.Type ?? "unspecified",
remediation.Description,
remediation.Url,
remediation.KbNumber)).ToArray() ?? Array.Empty<MsrcAdvisoryRemediation>(),
Products = detail.AffectedProducts?.Select(product =>
new MsrcAdvisoryProduct(
BuildProductIdentifier(product),
product.ProductName,
product.Platform,
product.Architecture,
product.BuildNumber,
product.Cpe)).ToArray() ?? Array.Empty<MsrcAdvisoryProduct>(),
CvssBaseScore = detail.Cvss?.BaseScore,
CvssVector = detail.Cvss?.VectorString,
ReleaseNoteUrl = detail.ReleaseNoteUrl,
CvrfUrl = detail.CvrfUrl,
};
}
private static string BuildProductIdentifier(MsrcAffectedProductDto product)
{
var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName;
if (string.IsNullOrWhiteSpace(name))
{
name = "Unknown Product";
}
if (!string.IsNullOrWhiteSpace(product.BuildNumber))
{
return $"{name} build {product.BuildNumber}";
}
return name;
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed class MsrcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Feedser.Source.Vndr.Msrc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchFailures;
private readonly Histogram<long> _summaryItemCount;
private readonly Histogram<double> _summaryWindowHours;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Histogram<long> _detailEnqueued;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseKbCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAliasCount;
private readonly Histogram<long> _mapAffectedCount;
public MsrcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_summaryFetchAttempts = _meter.CreateCounter<long>("msrc.summary.fetch.attempts", "operations");
_summaryFetchSuccess = _meter.CreateCounter<long>("msrc.summary.fetch.success", "operations");
_summaryFetchFailures = _meter.CreateCounter<long>("msrc.summary.fetch.failures", "operations");
_summaryItemCount = _meter.CreateHistogram<long>("msrc.summary.items.count", "items");
_summaryWindowHours = _meter.CreateHistogram<double>("msrc.summary.window.hours", "hours");
_detailFetchAttempts = _meter.CreateCounter<long>("msrc.detail.fetch.attempts", "operations");
_detailFetchSuccess = _meter.CreateCounter<long>("msrc.detail.fetch.success", "operations");
_detailFetchNotModified = _meter.CreateCounter<long>("msrc.detail.fetch.not_modified", "operations");
_detailFetchFailures = _meter.CreateCounter<long>("msrc.detail.fetch.failures", "operations");
_detailEnqueued = _meter.CreateHistogram<long>("msrc.detail.enqueued.count", "documents");
_parseSuccess = _meter.CreateCounter<long>("msrc.parse.success", "documents");
_parseFailures = _meter.CreateCounter<long>("msrc.parse.failures", "documents");
_parseProductCount = _meter.CreateHistogram<long>("msrc.parse.products.count", "products");
_parseKbCount = _meter.CreateHistogram<long>("msrc.parse.kb.count", "kb");
_mapSuccess = _meter.CreateCounter<long>("msrc.map.success", "advisories");
_mapFailures = _meter.CreateCounter<long>("msrc.map.failures", "advisories");
_mapAliasCount = _meter.CreateHistogram<long>("msrc.map.aliases.count", "aliases");
_mapAffectedCount = _meter.CreateHistogram<long>("msrc.map.affected.count", "packages");
}
public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1);
public void SummaryFetchSuccess(int count, double? windowHours)
{
_summaryFetchSuccess.Add(1);
if (count >= 0)
{
_summaryItemCount.Record(count);
}
if (windowHours is { } value && value >= 0)
{
_summaryWindowHours.Record(value);
}
}
public void SummaryFetchFailure(string reason)
=> _summaryFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason)
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailEnqueued(int count)
{
if (count >= 0)
{
_detailEnqueued.Record(count);
}
}
public void ParseSuccess(int productCount, int kbCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (kbCount >= 0)
{
_parseKbCount.Record(kbCount);
}
}
public void ParseFailure(string reason)
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int aliasCount, int packageCount)
{
_mapSuccess.Add(1);
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
if (packageCount >= 0)
{
_mapAffectedCount.Record(packageCount);
}
}
public void MapFailure(string reason)
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
internal static class MsrcDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
["msrc.id"] = summary.Id,
};
if (summary.LastModifiedDate.HasValue)
{
metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O");
}
if (summary.ReleaseDate.HasValue)
{
metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O");
}
if (!string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
metadata["msrc.cvrfUrl"] = summary.CvrfUrl!;
}
if (summary.CveNumbers.Count > 0)
{
metadata["msrc.cves"] = string.Join(",", summary.CveNumbers);
}
return metadata;
}
public static Dictionary<string, string> CreateCvrfMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = CreateMetadata(summary);
metadata["msrc.cvrf"] = "true";
return metadata;
}
}

View File

@@ -0,0 +1,239 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Storage.Mongo.Documents;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
internal static class MsrcMapper
{
public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var advisoryKey = dto.AdvisoryId;
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affectedPackages = BuildPackages(dto, recordedAt);
var cvssMetrics = BuildCvss(dto, recordedAt);
var provenance = new AdvisoryProvenance(
source: MsrcConnectorPlugin.SourceName,
kind: "advisory",
value: advisoryKey,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: advisoryKey,
title: dto.Title,
summary: dto.Description,
language: "en",
published: dto.ReleaseDate,
modified: dto.LastModifiedDate,
severity: NormalizeSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: cvssMetrics,
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(MsrcAdvisoryDto dto)
{
var aliases = new List<string> { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
foreach (var kb in dto.KbIds)
{
if (!string.IsNullOrWhiteSpace(kb))
{
aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}");
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl))
{
references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt));
}
if (!string.IsNullOrWhiteSpace(dto.CvrfUrl))
{
references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt));
}
foreach (var remediation in dto.Remediations)
{
if (!string.IsNullOrWhiteSpace(remediation.Url))
{
references.Add(CreateReference(
remediation.Url!,
string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference",
recordedAt,
remediation.Description));
}
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null)
=> new(
url,
kind: kind.ToLowerInvariant(),
sourceTag: "msrc",
summary: summary,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"reference",
url,
recordedAt,
new[] { ProvenanceFieldMasks.References }));
private static IReadOnlyList<AffectedPackage> BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier;
var provenance = new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var notes = new List<string>();
if (!string.IsNullOrWhiteSpace(product.Platform))
{
notes.Add($"platform:{product.Platform}");
}
if (!string.IsNullOrWhiteSpace(product.Architecture))
{
notes.Add($"arch:{product.Architecture}");
}
if (!string.IsNullOrWhiteSpace(product.Cpe))
{
notes.Add($"cpe:{product.Cpe}");
}
var range = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new AffectedVersionRange(
rangeKind: "custom",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: $"build:{product.BuildNumber}",
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package-range",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges })),
}
: Array.Empty<AffectedVersionRange>();
var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new NormalizedVersionRule(
scheme: "msrc.build",
type: NormalizedVersionRuleTypes.Exact,
value: product.BuildNumber,
notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n))))
}
: Array.Empty<NormalizedVersionRule>();
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: identifier,
platform: product.Platform,
versionRanges: range,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedRules));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CvssMetric> BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector))
{
return Array.Empty<CvssMetric>();
}
var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value);
return new[]
{
new CvssMetric(
version: "3.1",
vector: dto.CvssVector!,
baseScore: dto.CvssBaseScore.Value,
baseSeverity: severity,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"cvss",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.CvssMetrics })),
};
}
private static string CvssSeverityFromScore(double score)
=> score switch
{
< 0 => "none",
< 4 => "low",
< 7 => "medium",
< 9 => "high",
_ => "critical",
};
private static string? NormalizeSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.Trim().ToLowerInvariant();
}
}

View File

@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
public sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal;
public interface IMsrcTokenProvider
{
Task<string> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly MsrcOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
private AccessToken? _currentToken;
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IOptions<MsrcOptions> options,
TimeProvider? timeProvider,
ILogger<MsrcTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
{
var token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
_logger.LogInformation("Requesting new MSRC access token");
var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName);
var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret,
["grant_type"] = "client_credentials",
["scope"] = _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("AAD token response was null.");
var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60);
_currentToken = new AccessToken(payload.AccessToken, expiresAt);
return payload.AccessToken;
}
finally
{
_refreshLock.Release();
}
}
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string AccessToken { get; init; } = string.Empty;
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}

View File

@@ -0,0 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Feedser.Core.Jobs;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
internal static class MsrcJobKinds
{
public const string Fetch = "source:vndr.msrc:fetch";
}
internal sealed class MsrcFetchJob : IJob
{
private readonly MsrcConnector _connector;
public MsrcFetchJob(MsrcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

View File

@@ -0,0 +1,447 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
using StellaOps.Feedser.Source.Vndr.Msrc.Internal;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
public sealed class MsrcConnector : IFeedConnector
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
WriteIndented = false,
};
private readonly MsrcApiClient _apiClient;
private readonly MsrcDetailParser _detailParser;
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
private readonly IDtoStore _dtoStore;
private readonly IAdvisoryStore _advisoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly MsrcOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<MsrcConnector> _logger;
private readonly MsrcDiagnostics _diagnostics;
public MsrcConnector(
MsrcApiClient apiClient,
MsrcDetailParser detailParser,
SourceFetchService fetchService,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<MsrcOptions> options,
TimeProvider? timeProvider,
MsrcDiagnostics diagnostics,
ILogger<MsrcConnector> logger)
{
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
_detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser));
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => MsrcConnectorPlugin.SourceName;
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var from = cursor.LastModifiedCursor ?? _options.InitialLastModified ?? now.AddDays(-30);
from = from.Add(-_options.CursorOverlap);
var to = now;
_diagnostics.SummaryFetchAttempt();
IReadOnlyList<MsrcVulnerabilitySummary> summaries;
try
{
summaries = await _apiClient.FetchSummariesAsync(from, to, cancellationToken).ConfigureAwait(false);
var windowHours = (to - from).TotalHours;
_diagnostics.SummaryFetchSuccess(summaries.Count, windowHours);
}
catch (Exception ex)
{
_diagnostics.SummaryFetchFailure("exception");
_logger.LogError(ex, "MSRC summary fetch failed");
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
if (summaries.Count == 0)
{
await UpdateCursorAsync(cursor.WithLastModifiedCursor(to), cancellationToken).ConfigureAwait(false);
return;
}
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
var processed = 0;
var failures = 0;
foreach (var summary in summaries.OrderBy(static s => s.LastModifiedDate ?? DateTimeOffset.MinValue))
{
cancellationToken.ThrowIfCancellationRequested();
if (processed >= _options.MaxAdvisoriesPerFetch)
{
break;
}
var vulnerabilityId = string.IsNullOrWhiteSpace(summary.VulnerabilityId) ? summary.Id : summary.VulnerabilityId!;
var detailUri = _apiClient.BuildDetailUri(vulnerabilityId).ToString();
try
{
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri, cancellationToken).ConfigureAwait(false);
if (existing is not null && !ShouldRefresh(summary, existing))
{
_diagnostics.DetailFetchNotModified();
continue;
}
_diagnostics.DetailFetchAttempt();
if (existing?.GridFsId is { } oldGridId)
{
await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false);
}
var bytes = await _apiClient.FetchDetailAsync(vulnerabilityId, cancellationToken).ConfigureAwait(false);
var sha = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
var gridId = await _rawDocumentStorage.UploadAsync(SourceName, detailUri, bytes, "application/json", cancellationToken).ConfigureAwait(false);
var metadata = MsrcDocumentMetadata.CreateMetadata(summary);
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["content-type"] = "application/json",
};
var documentId = existing?.Id ?? Guid.NewGuid();
var record = new DocumentRecord(
documentId,
SourceName,
detailUri,
now,
sha,
DocumentStatuses.PendingParse,
ContentType: "application/json",
Headers: headers,
metadata,
existing?.Etag,
summary.LastModifiedDate,
gridId);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
pendingDocuments.Add(upserted.Id);
pendingMappings.Remove(upserted.Id);
_diagnostics.DetailFetchSuccess();
processed++;
if (_options.DownloadCvrf)
{
await FetchCvrfAsync(summary, now, cancellationToken).ConfigureAwait(false);
}
if (_options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_diagnostics.DetailFetchFailure("exception");
failures++;
_logger.LogError(ex, "MSRC detail fetch failed for {VulnerabilityId}", vulnerabilityId);
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
}
_diagnostics.DetailEnqueued(processed);
if (processed > 0 || failures > 0)
{
_logger.LogInformation("MSRC fetch cycle enqueued {Processed} advisories (failures={Failures}, pendingDocuments={PendingDocuments}, pendingMappings={PendingMappings})", processed, failures, pendingDocuments.Count, pendingMappings.Count);
}
var latestCursor = summaries
.Where(static s => s.LastModifiedDate.HasValue)
.Select(static s => s.LastModifiedDate!.Value)
.DefaultIfEmpty(to)
.Max();
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(pendingMappings)
.WithLastModifiedCursor(latestCursor);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingDocuments.Count == 0)
{
return;
}
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
var pendingMappings = cursor.PendingMappings.ToHashSet();
var now = _timeProvider.GetUtcNow();
foreach (var documentId in cursor.PendingDocuments)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
if (!document.GridFsId.HasValue)
{
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
_diagnostics.ParseFailure("missing_payload");
continue;
}
byte[] payload;
try
{
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.ParseFailure("download_failed");
_logger.LogError(ex, "MSRC unable to download document {DocumentId}", document.Id);
throw;
}
MsrcVulnerabilityDetailDto? detail;
try
{
detail = JsonSerializer.Deserialize<MsrcVulnerabilityDetailDto>(payload, SerializerOptions);
}
catch (Exception ex)
{
_diagnostics.ParseFailure("deserialize_failed");
_logger.LogError(ex, "MSRC failed to deserialize detail payload for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
if (detail is null)
{
_diagnostics.ParseFailure("empty_payload");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
continue;
}
var dto = _detailParser.Parse(detail);
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "msrc.detail.v1", bson, now);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
pendingMappings.Add(document.Id);
_diagnostics.ParseSuccess(dto.Products.Count, dto.KbIds.Count);
}
var updatedCursor = cursor
.WithPendingDocuments(remainingDocuments)
.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(services);
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingMappings.Count == 0)
{
return;
}
var pendingMappings = cursor.PendingMappings.ToHashSet();
foreach (var documentId in cursor.PendingMappings)
{
cancellationToken.ThrowIfCancellationRequested();
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
pendingMappings.Remove(documentId);
continue;
}
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
if (dtoRecord is null)
{
_diagnostics.MapFailure("missing_dto");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
continue;
}
MsrcAdvisoryDto? dto;
try
{
dto = JsonSerializer.Deserialize<MsrcAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions);
}
catch (Exception ex)
{
_diagnostics.MapFailure("deserialize_dto_failed");
_logger.LogError(ex, "MSRC failed to deserialize DTO for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
continue;
}
if (dto is null)
{
_diagnostics.MapFailure("null_dto");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
continue;
}
try
{
var advisory = MsrcMapper.Map(dto, document, dtoRecord.ValidatedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapSuccess(advisory.Aliases.Length, advisory.AffectedPackages.Length);
}
catch (Exception ex)
{
_diagnostics.MapFailure("exception");
_logger.LogError(ex, "MSRC mapping failed for document {DocumentId}", document.Id);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
}
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
private bool ShouldRefresh(MsrcVulnerabilitySummary summary, DocumentRecord existing)
{
if (existing.Status == DocumentStatuses.Failed)
{
return true;
}
if (summary.LastModifiedDate is null)
{
return true;
}
if (existing.Metadata is null || !existing.Metadata.TryGetValue("msrc.lastModified", out var stored))
{
return true;
}
return !string.Equals(stored, summary.LastModifiedDate.Value.ToString("O"), StringComparison.OrdinalIgnoreCase);
}
private async Task<MsrcCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return state is null ? MsrcCursor.Empty : MsrcCursor.FromBson(state.Cursor);
}
private async Task FetchCvrfAsync(MsrcVulnerabilitySummary summary, DateTimeOffset fetchedAt, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
return;
}
try
{
var uri = new Uri(summary.CvrfUrl);
var metadata = MsrcDocumentMetadata.CreateCvrfMetadata(summary);
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false);
var request = new SourceFetchRequest(
MsrcOptions.HttpClientName,
SourceName,
HttpMethod.Get,
uri,
metadata,
existing?.Etag,
existing?.LastModified,
AcceptHeaders: new[] { "application/zip", "application/xml", "application/json" });
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
if (result.IsNotModified || !result.IsSuccess || result.Document is null)
{
return;
}
await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("MSRC CVRF artefact captured for {AdvisoryId} ({Uri})", summary.VulnerabilityId ?? summary.Id, uri);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "MSRC CVRF download failed for {CvrfUrl}", summary.CvrfUrl);
}
}
private Task UpdateCursorAsync(MsrcCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var completedAt = _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}
}

View File

@@ -0,0 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
public sealed class MsrcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "vndr.msrc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<MsrcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<MsrcConnector>();
}
}

View File

@@ -0,0 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Feedser.Core.Jobs;
using StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "feedser:sources:vndr:msrc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddMsrcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<MsrcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, MsrcJobKinds.Fetch, typeof(MsrcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Feedser.Source.Common.Http;
using StellaOps.Feedser.Source.Vndr.Msrc.Configuration;
using StellaOps.Feedser.Source.Vndr.Msrc.Internal;
namespace StellaOps.Feedser.Source.Vndr.Msrc;
public static class MsrcServiceCollectionExtensions
{
public static IServiceCollection AddMsrcConnector(this IServiceCollection services, Action<MsrcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<MsrcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(MsrcOptions.HttpClientName, static (provider, clientOptions) =>
{
var options = provider.GetRequiredService<IOptions<MsrcOptions>>().Value;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseUri.Host);
clientOptions.AllowedHosts.Add("download.microsoft.com");
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.All;
};
});
services.AddSourceHttpClient(MsrcOptions.TokenClientName, static (_, clientOptions) =>
{
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add("login.microsoftonline.com");
clientOptions.ConfigureHandler = handler =>
{
handler.AutomaticDecompression = DecompressionMethods.All;
};
});
services.TryAddSingleton<IMsrcTokenProvider, MsrcTokenProvider>();
services.TryAddSingleton<MsrcApiClient>();
services.TryAddSingleton<MsrcDetailParser>();
services.TryAddSingleton<MsrcDiagnostics>();
services.AddTransient<MsrcConnector>();
return services;
}
}

View File

@@ -0,0 +1,19 @@
# MSRC Security Updates Connector Notes
## API endpoints
- **Vulnerability summaries** `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerabilities` (requires `api-version=2024-08-01`, client credential bearer token).
- **Vulnerability detail** `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerability/{id}` (same headers/scopes).
- **CVRF package** the detail payload contains `cvrfUrl` pointing to a ZIP/JSON asset that is stable per revision. We surface the URL as a reference and capture it in metadata for future offline bundling.
## Cursor behaviour
- Connector keeps a `lastModifiedCursor` and replays the previous 10 minutes on every fetch to cover late revisions.
- MSRC limits requests to ~60/minute; `requestDelay` defaults to 250ms and is configurable.
## Authentication
- Uses Azure AD client credential flow against `https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token` with scope `api://api.msrc.microsoft.com/.default`.
- Token refresh happens lazily and is cached until 60 seconds before expiry.
- Configuration values (`tenantId`, `clientId`, `clientSecret`) must be supplied via `feedser:sources:vndr:msrc`.
## CVRF handling
- Detail payload is persisted with the `cvrfUrl` in metadata (`msrc.cvrfUrl`).
- Mapping stage emits the CVRF link as a reference so offline runs can fetch it later. When `DownloadCvrf` is enabled the connector also saves the ZIP artefact to the documents store (marked as `msrc.cvrf=true`) for Offline Kit staging.

View File

@@ -2,10 +2,10 @@
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DONE (2025-10-11)** Confirmed REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) + CVRF ZIP download flow, required Azure AD client-credentials scope (`api://api.msrc.microsoft.com/.default`), mandatory `api-version=2024-08-01` header, and delta params (`lastModifiedStartDateTime`, `lastModifiedEndDateTime`). Findings recorded in `docs/feedser-connector-research-20251011.md`.|
|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**TODO** Implement fetch job that loops over `lastModifiedStartDateTime` cursor, handles `Retry-After` on throttling (default quota 60 req/min), and persists both REST JSON + optional CVRF attachments. Maintain source_state cursor at minute precision with overlap to cover delayed revisions.|
|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**TODO** Extract `vulnerabilityId`, `cveNumber`, `title`, `description`, `threats[]`, `remediations[]`, KB list, CVSS data, and `affectedProducts`. Map products into package identifiers (Windows build numbers, Office version) and capture `releaseNotes` URLs as references.|
|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**TODO** Map advisories to canonical records with aliases, references, range primitives for product/build coverage. Coordinate scheme naming and normalized outputs with `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: normalized array exemplar `[{"scheme":"semver","type":"range","min":"<build-start>","minInclusive":true,"max":"<build-end>","maxInclusive":false,"notes":"msrc:KB<id>"}]`; if monthly rollups require `msrc.patch` scheme, gather samples and align with Models before emitting.|
|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**TODO** Add regression tests with fixtures; support `UPDATE_MSRC_FIXTURES=1`.|
|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**TODO** Add logging/metrics and documentation; update backlog once connector is production-ready.|
|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**DONE (2025-10-15)** Added `MsrcApiClient` + token provider, cursor overlap handling, and detail persistence via GridFS (metadata carries CVRF URL + timestamps). State tracks `lastModifiedCursor` with configurable overlap/backoff. **Next:** coordinate with Tools on shared state-seeding helper once CVRF download flag stabilises.|
|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**DONE (2025-10-15)** Implemented `MsrcDetailParser`/DTOs capturing threats, remediations, KB IDs, CVEs, CVSS, and affected products (build/platform metadata preserved).|
|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**DONE (2025-10-15)** `MsrcMapper` emits aliases (MSRC ID/CVE/KB), references (release notes + CVRF), vendor packages with `msrc.build` normalized rules, and CVSS provenance.|
|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-15)** Added `StellaOps.Feedser.Source.Vndr.Msrc.Tests` with canned token/summary/detail responses and snapshot assertions via Mongo2Go. Fixtures regenerate via `UPDATE_MSRC_FIXTURES`.|
|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** Introduced `MsrcDiagnostics` meter (summary/detail/parse/map metrics), structured fetch logs, README updates, and Ops brief `docs/ops/feedser-msrc-operations.md` covering AAD onboarding + CVRF handling.|
|FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**DONE (2025-10-11)** Completed memo outline recommending dual-path (REST for incremental, CVRF for offline); implementation hinges on `FEEDCONN-MSRC-02-008` AAD onboarding for token acquisition.|
|FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**TODO** Provision MSRC SUG app registration, document client credential flow, rotation cadence, and secure storage expectations for Offline Kit deployments.|
|FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**DONE (2025-10-15)** Coordinated Ops handoff; drafted AAD onboarding brief (`docs/ops/feedser-msrc-operations.md`) with app registration requirements, secret rotation policy, sample configuration, and CVRF mirroring guidance for Offline Kit.|