Restructure solution layout by module
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Implement the Microsoft Security Response Center (MSRC) connector to ingest Microsoft security updates (Security Updates API / CVRF).
|
||||
|
||||
## Scope
|
||||
- Identify MSRC data sources (Security Update Guide API, CVRF downloads) and incremental update strategy.
|
||||
- Implement fetch/cursor pipeline with retry/backoff, handling API keys if required.
|
||||
- Parse advisories to extract summary, affected products, KBs, CVEs, severities, mitigations.
|
||||
- Map entries into canonical `Advisory` objects with aliases, references, affected packages, and range primitives (e.g., Windows build numbers, SemVer).
|
||||
- Provide deterministic fixtures and regression tests.
|
||||
|
||||
## Participants
|
||||
- `Source.Common`, `Storage.Mongo`, `Concelier.Models`, `Concelier.Testing`.
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `msrc:fetch`, `msrc:parse`, `msrc:map`.
|
||||
- Persist upstream metadata (e.g., `lastModified`, `releaseDate`).
|
||||
- Alias set should include MSRC ID, CVEs, and KB identifiers.
|
||||
|
||||
## In/Out of scope
|
||||
In scope: Microsoft Security Update Guide advisories.
|
||||
Out of scope: Non-security Microsoft release notes.
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch/mapping stats, respect API rate limits, handle authentication securely.
|
||||
- Sanitize payloads; validate JSON/CVRF before persistence.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Vndr.Msrc.Tests` with fixtures covering fetch/parse/map.
|
||||
- Snapshot canonical advisories; support fixture regeneration.
|
||||
@@ -0,0 +1,132 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
|
||||
public sealed class MsrcOptions
|
||||
{
|
||||
public const string HttpClientName = "concelier.source.vndr.msrc";
|
||||
public const string TokenClientName = "concelier.source.vndr.msrc.token";
|
||||
|
||||
public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute);
|
||||
|
||||
public string Locale { get; set; } = "en-US";
|
||||
|
||||
public string ApiVersion { get; set; } = "2024-08-01";
|
||||
|
||||
/// <summary>
|
||||
/// Azure AD tenant identifier used for client credential flow.
|
||||
/// </summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Azure AD application (client) identifier.
|
||||
/// </summary>
|
||||
public string ClientId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Azure AD client secret used for token acquisition.
|
||||
/// </summary>
|
||||
public string ClientSecret { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Scope requested during client-credential token acquisition.
|
||||
/// </summary>
|
||||
public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum advisories to fetch per cycle.
|
||||
/// </summary>
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 200;
|
||||
|
||||
/// <summary>
|
||||
/// Page size used when iterating the MSRC API.
|
||||
/// </summary>
|
||||
public int PageSize { get; set; } = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Overlap window added when resuming from the last modified cursor.
|
||||
/// </summary>
|
||||
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// When enabled the connector downloads the CVRF artefact referenced by each advisory.
|
||||
/// </summary>
|
||||
public bool DownloadCvrf { get; set; } = false;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Optional lower bound for the initial sync if the cursor is empty.
|
||||
/// </summary>
|
||||
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("MSRC base URI must be absolute.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Locale))
|
||||
{
|
||||
throw new InvalidOperationException("Locale must be provided.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ApiVersion))
|
||||
{
|
||||
throw new InvalidOperationException("API version must be provided.");
|
||||
}
|
||||
|
||||
if (!Guid.TryParse(TenantId, out _))
|
||||
{
|
||||
throw new InvalidOperationException("TenantId must be a valid GUID.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ClientId))
|
||||
{
|
||||
throw new InvalidOperationException("ClientId must be provided.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ClientSecret))
|
||||
{
|
||||
throw new InvalidOperationException("ClientSecret must be provided.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Scope))
|
||||
{
|
||||
throw new InvalidOperationException("Scope must be provided.");
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (PageSize <= 0 || PageSize > 500)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
|
||||
}
|
||||
|
||||
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed record MsrcAdvisoryDto
|
||||
{
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
public string? Description { get; init; }
|
||||
|
||||
public string? Severity { get; init; }
|
||||
|
||||
public DateTimeOffset? ReleaseDate { get; init; }
|
||||
|
||||
public DateTimeOffset? LastModifiedDate { get; init; }
|
||||
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> KbIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<MsrcAdvisoryThreat> Threats { get; init; } = Array.Empty<MsrcAdvisoryThreat>();
|
||||
|
||||
public IReadOnlyList<MsrcAdvisoryRemediation> Remediations { get; init; } = Array.Empty<MsrcAdvisoryRemediation>();
|
||||
|
||||
public IReadOnlyList<MsrcAdvisoryProduct> Products { get; init; } = Array.Empty<MsrcAdvisoryProduct>();
|
||||
|
||||
public double? CvssBaseScore { get; init; }
|
||||
|
||||
public string? CvssVector { get; init; }
|
||||
|
||||
public string? ReleaseNoteUrl { get; init; }
|
||||
|
||||
public string? CvrfUrl { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity);
|
||||
|
||||
public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb);
|
||||
|
||||
public sealed record MsrcAdvisoryProduct(
|
||||
string Identifier,
|
||||
string? ProductName,
|
||||
string? Platform,
|
||||
string? Architecture,
|
||||
string? BuildNumber,
|
||||
string? Cpe);
|
||||
@@ -0,0 +1,138 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed class MsrcApiClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IMsrcTokenProvider _tokenProvider;
|
||||
private readonly MsrcOptions _options;
|
||||
private readonly ILogger<MsrcApiClient> _logger;
|
||||
|
||||
public MsrcApiClient(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IMsrcTokenProvider tokenProvider,
|
||||
IOptions<MsrcOptions> options,
|
||||
ILogger<MsrcApiClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<MsrcVulnerabilitySummary>> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<MsrcVulnerabilitySummary>();
|
||||
var requestUri = BuildSummaryUri(fromInclusive, toExclusive);
|
||||
|
||||
while (requestUri is not null)
|
||||
{
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
|
||||
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}");
|
||||
}
|
||||
|
||||
var payload = await response.Content.ReadFromJsonAsync<MsrcSummaryResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false)
|
||||
?? new MsrcSummaryResponse();
|
||||
|
||||
results.AddRange(payload.Value);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(payload.NextLink))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
requestUri = new Uri(payload.NextLink, UriKind.Absolute);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public Uri BuildDetailUri(string vulnerabilityId)
|
||||
{
|
||||
var uri = CreateDetailUriInternal(vulnerabilityId);
|
||||
return uri;
|
||||
}
|
||||
|
||||
public async Task<byte[]> FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
|
||||
var uri = CreateDetailUriInternal(vulnerabilityId);
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
|
||||
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}");
|
||||
}
|
||||
|
||||
return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<HttpClient> CreateAuthenticatedClientAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
|
||||
var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName);
|
||||
client.DefaultRequestHeaders.Remove("Authorization");
|
||||
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
|
||||
client.DefaultRequestHeaders.Remove("Accept");
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/json");
|
||||
client.DefaultRequestHeaders.Remove("api-version");
|
||||
client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion);
|
||||
client.DefaultRequestHeaders.Remove("Accept-Language");
|
||||
client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale);
|
||||
return client;
|
||||
}
|
||||
|
||||
private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
builder.Append(_options.BaseUri.ToString().TrimEnd('/'));
|
||||
builder.Append("/vulnerabilities?");
|
||||
builder.Append("$top=").Append(_options.PageSize);
|
||||
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O")));
|
||||
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O")));
|
||||
builder.Append("&$orderby=lastModifiedDate");
|
||||
builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale));
|
||||
builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion));
|
||||
|
||||
return new Uri(builder.ToString(), UriKind.Absolute);
|
||||
}
|
||||
|
||||
private Uri CreateDetailUriInternal(string vulnerabilityId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId))
|
||||
{
|
||||
throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId));
|
||||
}
|
||||
|
||||
var baseUri = _options.BaseUri.ToString().TrimEnd('/');
|
||||
var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}";
|
||||
return new Uri(path, UriKind.Absolute);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
internal sealed record MsrcCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
DateTimeOffset? LastModifiedCursor)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidSet = Array.Empty<Guid>();
|
||||
|
||||
public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null);
|
||||
|
||||
public MsrcCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = Distinct(documents) };
|
||||
|
||||
public MsrcCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = Distinct(mappings) };
|
||||
|
||||
public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp)
|
||||
=> this with { LastModifiedCursor = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastModifiedCursor.HasValue)
|
||||
{
|
||||
document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static MsrcCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var lastModified = document.TryGetValue("lastModifiedCursor", out var value)
|
||||
? ParseDate(value)
|
||||
: null;
|
||||
|
||||
return new MsrcCursor(pendingDocuments, pendingMappings, lastModified);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
|
||||
=> values?.Distinct().ToArray() ?? EmptyGuidSet;
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidSet;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var id))
|
||||
{
|
||||
items.Add(id);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed record MsrcVulnerabilityDetailDto
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("vulnerabilityId")]
|
||||
public string VulnerabilityId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("cveNumber")]
|
||||
public string? CveNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("cveNumbers")]
|
||||
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("releaseDate")]
|
||||
public DateTimeOffset? ReleaseDate { get; init; }
|
||||
|
||||
[JsonPropertyName("lastModifiedDate")]
|
||||
public DateTimeOffset? LastModifiedDate { get; init; }
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("threats")]
|
||||
public IReadOnlyList<MsrcThreatDto> Threats { get; init; } = Array.Empty<MsrcThreatDto>();
|
||||
|
||||
[JsonPropertyName("remediations")]
|
||||
public IReadOnlyList<MsrcRemediationDto> Remediations { get; init; } = Array.Empty<MsrcRemediationDto>();
|
||||
|
||||
[JsonPropertyName("affectedProducts")]
|
||||
public IReadOnlyList<MsrcAffectedProductDto> AffectedProducts { get; init; } = Array.Empty<MsrcAffectedProductDto>();
|
||||
|
||||
[JsonPropertyName("cvssV3")]
|
||||
public MsrcCvssDto? Cvss { get; init; }
|
||||
|
||||
[JsonPropertyName("releaseNoteUrl")]
|
||||
public string? ReleaseNoteUrl { get; init; }
|
||||
|
||||
[JsonPropertyName("cvrfUrl")]
|
||||
public string? CvrfUrl { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcThreatDto
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string? Type { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcRemediationDto
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string? Id { get; init; }
|
||||
|
||||
[JsonPropertyName("type")]
|
||||
public string? Type { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("kbNumber")]
|
||||
public string? KbNumber { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcAffectedProductDto
|
||||
{
|
||||
[JsonPropertyName("productId")]
|
||||
public string? ProductId { get; init; }
|
||||
|
||||
[JsonPropertyName("productName")]
|
||||
public string? ProductName { get; init; }
|
||||
|
||||
[JsonPropertyName("cpe")]
|
||||
public string? Cpe { get; init; }
|
||||
|
||||
[JsonPropertyName("platform")]
|
||||
public string? Platform { get; init; }
|
||||
|
||||
[JsonPropertyName("architecture")]
|
||||
public string? Architecture { get; init; }
|
||||
|
||||
[JsonPropertyName("buildNumber")]
|
||||
public string? BuildNumber { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcCvssDto
|
||||
{
|
||||
[JsonPropertyName("baseScore")]
|
||||
public double? BaseScore { get; init; }
|
||||
|
||||
[JsonPropertyName("vectorString")]
|
||||
public string? VectorString { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed class MsrcDetailParser
|
||||
{
|
||||
public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(detail);
|
||||
|
||||
var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId;
|
||||
var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray()
|
||||
?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty<string>() : new[] { detail.CveNumber! });
|
||||
|
||||
var kbIds = detail.Remediations?
|
||||
.Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber))
|
||||
.Select(static remediation => remediation.KbNumber!.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? Array.Empty<string>();
|
||||
|
||||
return new MsrcAdvisoryDto
|
||||
{
|
||||
AdvisoryId = advisoryId,
|
||||
Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(),
|
||||
Description = detail.Description,
|
||||
Severity = detail.Severity,
|
||||
ReleaseDate = detail.ReleaseDate,
|
||||
LastModifiedDate = detail.LastModifiedDate,
|
||||
CveIds = cveIds,
|
||||
KbIds = kbIds,
|
||||
Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat(
|
||||
threat.Type ?? "unspecified",
|
||||
threat.Description,
|
||||
threat.Severity)).ToArray() ?? Array.Empty<MsrcAdvisoryThreat>(),
|
||||
Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation(
|
||||
remediation.Type ?? "unspecified",
|
||||
remediation.Description,
|
||||
remediation.Url,
|
||||
remediation.KbNumber)).ToArray() ?? Array.Empty<MsrcAdvisoryRemediation>(),
|
||||
Products = detail.AffectedProducts?.Select(product =>
|
||||
new MsrcAdvisoryProduct(
|
||||
BuildProductIdentifier(product),
|
||||
product.ProductName,
|
||||
product.Platform,
|
||||
product.Architecture,
|
||||
product.BuildNumber,
|
||||
product.Cpe)).ToArray() ?? Array.Empty<MsrcAdvisoryProduct>(),
|
||||
CvssBaseScore = detail.Cvss?.BaseScore,
|
||||
CvssVector = detail.Cvss?.VectorString,
|
||||
ReleaseNoteUrl = detail.ReleaseNoteUrl,
|
||||
CvrfUrl = detail.CvrfUrl,
|
||||
};
|
||||
}
|
||||
|
||||
private static string BuildProductIdentifier(MsrcAffectedProductDto product)
|
||||
{
|
||||
var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName;
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
name = "Unknown Product";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.BuildNumber))
|
||||
{
|
||||
return $"{name} build {product.BuildNumber}";
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed class MsrcDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Vndr.Msrc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _summaryFetchAttempts;
|
||||
private readonly Counter<long> _summaryFetchSuccess;
|
||||
private readonly Counter<long> _summaryFetchFailures;
|
||||
private readonly Histogram<long> _summaryItemCount;
|
||||
private readonly Histogram<double> _summaryWindowHours;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchNotModified;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Histogram<long> _detailEnqueued;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseProductCount;
|
||||
private readonly Histogram<long> _parseKbCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapAliasCount;
|
||||
private readonly Histogram<long> _mapAffectedCount;
|
||||
|
||||
public MsrcDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_summaryFetchAttempts = _meter.CreateCounter<long>("msrc.summary.fetch.attempts", "operations");
|
||||
_summaryFetchSuccess = _meter.CreateCounter<long>("msrc.summary.fetch.success", "operations");
|
||||
_summaryFetchFailures = _meter.CreateCounter<long>("msrc.summary.fetch.failures", "operations");
|
||||
_summaryItemCount = _meter.CreateHistogram<long>("msrc.summary.items.count", "items");
|
||||
_summaryWindowHours = _meter.CreateHistogram<double>("msrc.summary.window.hours", "hours");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>("msrc.detail.fetch.attempts", "operations");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>("msrc.detail.fetch.success", "operations");
|
||||
_detailFetchNotModified = _meter.CreateCounter<long>("msrc.detail.fetch.not_modified", "operations");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>("msrc.detail.fetch.failures", "operations");
|
||||
_detailEnqueued = _meter.CreateHistogram<long>("msrc.detail.enqueued.count", "documents");
|
||||
_parseSuccess = _meter.CreateCounter<long>("msrc.parse.success", "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("msrc.parse.failures", "documents");
|
||||
_parseProductCount = _meter.CreateHistogram<long>("msrc.parse.products.count", "products");
|
||||
_parseKbCount = _meter.CreateHistogram<long>("msrc.parse.kb.count", "kb");
|
||||
_mapSuccess = _meter.CreateCounter<long>("msrc.map.success", "advisories");
|
||||
_mapFailures = _meter.CreateCounter<long>("msrc.map.failures", "advisories");
|
||||
_mapAliasCount = _meter.CreateHistogram<long>("msrc.map.aliases.count", "aliases");
|
||||
_mapAffectedCount = _meter.CreateHistogram<long>("msrc.map.affected.count", "packages");
|
||||
}
|
||||
|
||||
public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1);
|
||||
|
||||
public void SummaryFetchSuccess(int count, double? windowHours)
|
||||
{
|
||||
_summaryFetchSuccess.Add(1);
|
||||
if (count >= 0)
|
||||
{
|
||||
_summaryItemCount.Record(count);
|
||||
}
|
||||
|
||||
if (windowHours is { } value && value >= 0)
|
||||
{
|
||||
_summaryWindowHours.Record(value);
|
||||
}
|
||||
}
|
||||
|
||||
public void SummaryFetchFailure(string reason)
|
||||
=> _summaryFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
|
||||
|
||||
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
|
||||
|
||||
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
|
||||
|
||||
public void DetailFetchFailure(string reason)
|
||||
=> _detailFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void DetailEnqueued(int count)
|
||||
{
|
||||
if (count >= 0)
|
||||
{
|
||||
_detailEnqueued.Record(count);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseSuccess(int productCount, int kbCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
if (productCount >= 0)
|
||||
{
|
||||
_parseProductCount.Record(productCount);
|
||||
}
|
||||
|
||||
if (kbCount >= 0)
|
||||
{
|
||||
_parseKbCount.Record(kbCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure(string reason)
|
||||
=> _parseFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void MapSuccess(int aliasCount, int packageCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
if (aliasCount >= 0)
|
||||
{
|
||||
_mapAliasCount.Record(aliasCount);
|
||||
}
|
||||
|
||||
if (packageCount >= 0)
|
||||
{
|
||||
_mapAffectedCount.Record(packageCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure(string reason)
|
||||
=> _mapFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
private static KeyValuePair<string, object?> ReasonTag(string reason)
|
||||
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
internal static class MsrcDocumentMetadata
|
||||
{
|
||||
public static Dictionary<string, string> CreateMetadata(MsrcVulnerabilitySummary summary)
|
||||
{
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
|
||||
["msrc.id"] = summary.Id,
|
||||
};
|
||||
|
||||
if (summary.LastModifiedDate.HasValue)
|
||||
{
|
||||
metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (summary.ReleaseDate.HasValue)
|
||||
{
|
||||
metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(summary.CvrfUrl))
|
||||
{
|
||||
metadata["msrc.cvrfUrl"] = summary.CvrfUrl!;
|
||||
}
|
||||
|
||||
if (summary.CveNumbers.Count > 0)
|
||||
{
|
||||
metadata["msrc.cves"] = string.Join(",", summary.CveNumbers);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public static Dictionary<string, string> CreateCvrfMetadata(MsrcVulnerabilitySummary summary)
|
||||
{
|
||||
var metadata = CreateMetadata(summary);
|
||||
metadata["msrc.cvrf"] = "true";
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
internal static class MsrcMapper
|
||||
{
|
||||
public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var advisoryKey = dto.AdvisoryId;
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var affectedPackages = BuildPackages(dto, recordedAt);
|
||||
var cvssMetrics = BuildCvss(dto, recordedAt);
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
source: MsrcConnectorPlugin.SourceName,
|
||||
kind: "advisory",
|
||||
value: advisoryKey,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: advisoryKey,
|
||||
title: dto.Title,
|
||||
summary: dto.Description,
|
||||
language: "en",
|
||||
published: dto.ReleaseDate,
|
||||
modified: dto.LastModifiedDate,
|
||||
severity: NormalizeSeverity(dto.Severity),
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: cvssMetrics,
|
||||
provenance: new[] { provenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(MsrcAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string> { dto.AdvisoryId };
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var kb in dto.KbIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(kb))
|
||||
{
|
||||
aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}");
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl))
|
||||
{
|
||||
references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.CvrfUrl))
|
||||
{
|
||||
references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt));
|
||||
}
|
||||
|
||||
foreach (var remediation in dto.Remediations)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(remediation.Url))
|
||||
{
|
||||
references.Add(CreateReference(
|
||||
remediation.Url!,
|
||||
string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference",
|
||||
recordedAt,
|
||||
remediation.Description));
|
||||
}
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null)
|
||||
=> new(
|
||||
url,
|
||||
kind: kind.ToLowerInvariant(),
|
||||
sourceTag: "msrc",
|
||||
summary: summary,
|
||||
provenance: new AdvisoryProvenance(
|
||||
MsrcConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }));
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
foreach (var product in dto.Products)
|
||||
{
|
||||
var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier;
|
||||
var provenance = new AdvisoryProvenance(
|
||||
MsrcConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var notes = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(product.Platform))
|
||||
{
|
||||
notes.Add($"platform:{product.Platform}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Architecture))
|
||||
{
|
||||
notes.Add($"arch:{product.Architecture}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(product.Cpe))
|
||||
{
|
||||
notes.Add($"cpe:{product.Cpe}");
|
||||
}
|
||||
|
||||
var range = !string.IsNullOrWhiteSpace(product.BuildNumber)
|
||||
? new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: "custom",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: $"build:{product.BuildNumber}",
|
||||
provenance: new AdvisoryProvenance(
|
||||
MsrcConnectorPlugin.SourceName,
|
||||
"package-range",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges })),
|
||||
}
|
||||
: Array.Empty<AffectedVersionRange>();
|
||||
|
||||
var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber)
|
||||
? new[]
|
||||
{
|
||||
new NormalizedVersionRule(
|
||||
scheme: "msrc.build",
|
||||
type: NormalizedVersionRuleTypes.Exact,
|
||||
value: product.BuildNumber,
|
||||
notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n))))
|
||||
}
|
||||
: Array.Empty<NormalizedVersionRule>();
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
type: AffectedPackageTypes.Vendor,
|
||||
identifier: identifier,
|
||||
platform: product.Platform,
|
||||
versionRanges: range,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: normalizedRules));
|
||||
}
|
||||
|
||||
return packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector))
|
||||
{
|
||||
return Array.Empty<CvssMetric>();
|
||||
}
|
||||
|
||||
var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value);
|
||||
|
||||
return new[]
|
||||
{
|
||||
new CvssMetric(
|
||||
version: "3.1",
|
||||
vector: dto.CvssVector!,
|
||||
baseScore: dto.CvssBaseScore.Value,
|
||||
baseSeverity: severity,
|
||||
provenance: new AdvisoryProvenance(
|
||||
MsrcConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
dto.AdvisoryId,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics })),
|
||||
};
|
||||
}
|
||||
|
||||
private static string CvssSeverityFromScore(double score)
|
||||
=> score switch
|
||||
{
|
||||
< 0 => "none",
|
||||
< 4 => "low",
|
||||
< 7 => "medium",
|
||||
< 9 => "high",
|
||||
_ => "critical",
|
||||
};
|
||||
|
||||
private static string? NormalizeSeverity(string? severity)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(severity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return severity.Trim().ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public sealed record MsrcSummaryResponse
|
||||
{
|
||||
[JsonPropertyName("value")]
|
||||
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("@odata.nextLink")]
|
||||
public string? NextLink { get; init; }
|
||||
}
|
||||
|
||||
public sealed record MsrcVulnerabilitySummary
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("vulnerabilityId")]
|
||||
public string? VulnerabilityId { get; init; }
|
||||
|
||||
[JsonPropertyName("cveNumber")]
|
||||
public string? CveNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("cveNumbers")]
|
||||
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("releaseDate")]
|
||||
public DateTimeOffset? ReleaseDate { get; init; }
|
||||
|
||||
[JsonPropertyName("lastModifiedDate")]
|
||||
public DateTimeOffset? LastModifiedDate { get; init; }
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("cvrfUrl")]
|
||||
public string? CvrfUrl { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
public interface IMsrcTokenProvider
|
||||
{
|
||||
Task<string> GetAccessTokenAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly MsrcOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<MsrcTokenProvider> _logger;
|
||||
private readonly SemaphoreSlim _refreshLock = new(1, 1);
|
||||
|
||||
private AccessToken? _currentToken;
|
||||
|
||||
public MsrcTokenProvider(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<MsrcOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<MsrcTokenProvider> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var token = _currentToken;
|
||||
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
|
||||
{
|
||||
return token.Token;
|
||||
}
|
||||
|
||||
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
token = _currentToken;
|
||||
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
|
||||
{
|
||||
return token.Token;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Requesting new MSRC access token");
|
||||
var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName);
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
|
||||
{
|
||||
Content = new FormUrlEncodedContent(new Dictionary<string, string>
|
||||
{
|
||||
["client_id"] = _options.ClientId,
|
||||
["client_secret"] = _options.ClientSecret,
|
||||
["grant_type"] = "client_credentials",
|
||||
["scope"] = _options.Scope,
|
||||
}),
|
||||
};
|
||||
|
||||
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var payload = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException("AAD token response was null.");
|
||||
|
||||
var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60);
|
||||
_currentToken = new AccessToken(payload.AccessToken, expiresAt);
|
||||
return payload.AccessToken;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_refreshLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private Uri BuildTokenUri()
|
||||
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
|
||||
|
||||
public void Dispose() => _refreshLock.Dispose();
|
||||
|
||||
private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt)
|
||||
{
|
||||
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
|
||||
}
|
||||
|
||||
private sealed record TokenResponse
|
||||
{
|
||||
[JsonPropertyName("access_token")]
|
||||
public string AccessToken { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("expires_in")]
|
||||
public int ExpiresIn { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc;
|
||||
|
||||
internal static class MsrcJobKinds
|
||||
{
|
||||
public const string Fetch = "source:vndr.msrc:fetch";
|
||||
}
|
||||
|
||||
internal sealed class MsrcFetchJob : IJob
|
||||
{
|
||||
private readonly MsrcConnector _connector;
|
||||
|
||||
public MsrcFetchJob(MsrcConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc;
|
||||
|
||||
public sealed class MsrcConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private readonly MsrcApiClient _apiClient;
|
||||
private readonly MsrcDetailParser _detailParser;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly MsrcOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<MsrcConnector> _logger;
|
||||
private readonly MsrcDiagnostics _diagnostics;
|
||||
|
||||
public MsrcConnector(
|
||||
MsrcApiClient apiClient,
|
||||
MsrcDetailParser detailParser,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<MsrcOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
MsrcDiagnostics diagnostics,
|
||||
ILogger<MsrcConnector> logger)
|
||||
{
|
||||
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
|
||||
_detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => MsrcConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var from = cursor.LastModifiedCursor ?? _options.InitialLastModified ?? now.AddDays(-30);
|
||||
from = from.Add(-_options.CursorOverlap);
|
||||
var to = now;
|
||||
|
||||
_diagnostics.SummaryFetchAttempt();
|
||||
IReadOnlyList<MsrcVulnerabilitySummary> summaries;
|
||||
|
||||
try
|
||||
{
|
||||
summaries = await _apiClient.FetchSummariesAsync(from, to, cancellationToken).ConfigureAwait(false);
|
||||
var windowHours = (to - from).TotalHours;
|
||||
_diagnostics.SummaryFetchSuccess(summaries.Count, windowHours);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.SummaryFetchFailure("exception");
|
||||
_logger.LogError(ex, "MSRC summary fetch failed");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (summaries.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor.WithLastModifiedCursor(to), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var processed = 0;
|
||||
var failures = 0;
|
||||
|
||||
foreach (var summary in summaries.OrderBy(static s => s.LastModifiedDate ?? DateTimeOffset.MinValue))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (processed >= _options.MaxAdvisoriesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var vulnerabilityId = string.IsNullOrWhiteSpace(summary.VulnerabilityId) ? summary.Id : summary.VulnerabilityId!;
|
||||
var detailUri = _apiClient.BuildDetailUri(vulnerabilityId).ToString();
|
||||
|
||||
try
|
||||
{
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && !ShouldRefresh(summary, existing))
|
||||
{
|
||||
_diagnostics.DetailFetchNotModified();
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.DetailFetchAttempt();
|
||||
if (existing?.GridFsId is { } oldGridId)
|
||||
{
|
||||
await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var bytes = await _apiClient.FetchDetailAsync(vulnerabilityId, cancellationToken).ConfigureAwait(false);
|
||||
var sha = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
|
||||
var gridId = await _rawDocumentStorage.UploadAsync(SourceName, detailUri, bytes, "application/json", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = MsrcDocumentMetadata.CreateMetadata(summary);
|
||||
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["content-type"] = "application/json",
|
||||
};
|
||||
|
||||
var documentId = existing?.Id ?? Guid.NewGuid();
|
||||
var record = new DocumentRecord(
|
||||
documentId,
|
||||
SourceName,
|
||||
detailUri,
|
||||
now,
|
||||
sha,
|
||||
DocumentStatuses.PendingParse,
|
||||
ContentType: "application/json",
|
||||
Headers: headers,
|
||||
metadata,
|
||||
existing?.Etag,
|
||||
summary.LastModifiedDate,
|
||||
gridId);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
pendingMappings.Remove(upserted.Id);
|
||||
_diagnostics.DetailFetchSuccess();
|
||||
processed++;
|
||||
|
||||
if (_options.DownloadCvrf)
|
||||
{
|
||||
await FetchCvrfAsync(summary, now, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.DetailFetchFailure("exception");
|
||||
failures++;
|
||||
_logger.LogError(ex, "MSRC detail fetch failed for {VulnerabilityId}", vulnerabilityId);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
_diagnostics.DetailEnqueued(processed);
|
||||
|
||||
if (processed > 0 || failures > 0)
|
||||
{
|
||||
_logger.LogInformation("MSRC fetch cycle enqueued {Processed} advisories (failures={Failures}, pendingDocuments={PendingDocuments}, pendingMappings={PendingMappings})", processed, failures, pendingDocuments.Count, pendingMappings.Count);
|
||||
}
|
||||
|
||||
var latestCursor = summaries
|
||||
.Where(static s => s.LastModifiedDate.HasValue)
|
||||
.Select(static s => s.LastModifiedDate!.Value)
|
||||
.DefaultIfEmpty(to)
|
||||
.Max();
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithLastModifiedCursor(latestCursor);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure("missing_payload");
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("download_failed");
|
||||
_logger.LogError(ex, "MSRC unable to download document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
MsrcVulnerabilityDetailDto? detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<MsrcVulnerabilityDetailDto>(payload, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("deserialize_failed");
|
||||
_logger.LogError(ex, "MSRC failed to deserialize detail payload for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detail is null)
|
||||
{
|
||||
_diagnostics.ParseFailure("empty_payload");
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dto = _detailParser.Parse(detail);
|
||||
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "msrc.detail.v1", bson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
_diagnostics.ParseSuccess(dto.Products.Count, dto.KbIds.Count);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_diagnostics.MapFailure("missing_dto");
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
MsrcAdvisoryDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<MsrcAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.MapFailure("deserialize_dto_failed");
|
||||
_logger.LogError(ex, "MSRC failed to deserialize DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_diagnostics.MapFailure("null_dto");
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = MsrcMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapSuccess(advisory.Aliases.Length, advisory.AffectedPackages.Length);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.MapFailure("exception");
|
||||
_logger.LogError(ex, "MSRC mapping failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private bool ShouldRefresh(MsrcVulnerabilitySummary summary, DocumentRecord existing)
|
||||
{
|
||||
if (existing.Status == DocumentStatuses.Failed)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (summary.LastModifiedDate is null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (existing.Metadata is null || !existing.Metadata.TryGetValue("msrc.lastModified", out var stored))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return !string.Equals(stored, summary.LastModifiedDate.Value.ToString("O"), StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private async Task<MsrcCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? MsrcCursor.Empty : MsrcCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task FetchCvrfAsync(MsrcVulnerabilitySummary summary, DateTimeOffset fetchedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var uri = new Uri(summary.CvrfUrl);
|
||||
var metadata = MsrcDocumentMetadata.CreateCvrfMetadata(summary);
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var request = new SourceFetchRequest(
|
||||
MsrcOptions.HttpClientName,
|
||||
SourceName,
|
||||
HttpMethod.Get,
|
||||
uri,
|
||||
metadata,
|
||||
existing?.Etag,
|
||||
existing?.LastModified,
|
||||
AcceptHeaders: new[] { "application/zip", "application/xml", "application/json" });
|
||||
|
||||
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (result.IsNotModified || !result.IsSuccess || result.Document is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("MSRC CVRF artefact captured for {AdvisoryId} ({Uri})", summary.VulnerabilityId ?? summary.Id, uri);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "MSRC CVRF download failed for {CvrfUrl}", summary.CvrfUrl);
|
||||
}
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(MsrcCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc;
|
||||
|
||||
public sealed class MsrcConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "vndr.msrc";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<MsrcConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<MsrcConnector>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc;
|
||||
|
||||
public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:vndr:msrc";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddMsrcConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<MsrcFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, MsrcJobKinds.Fetch, typeof(MsrcFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Vndr.Msrc;
|
||||
|
||||
public static class MsrcServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddMsrcConnector(this IServiceCollection services, Action<MsrcOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<MsrcOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(MsrcOptions.HttpClientName, static (provider, clientOptions) =>
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<MsrcOptions>>().Value;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseUri.Host);
|
||||
clientOptions.AllowedHosts.Add("download.microsoft.com");
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.All;
|
||||
};
|
||||
});
|
||||
|
||||
services.AddSourceHttpClient(MsrcOptions.TokenClientName, static (_, clientOptions) =>
|
||||
{
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add("login.microsoftonline.com");
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.All;
|
||||
};
|
||||
});
|
||||
|
||||
services.TryAddSingleton<IMsrcTokenProvider, MsrcTokenProvider>();
|
||||
services.TryAddSingleton<MsrcApiClient>();
|
||||
services.TryAddSingleton<MsrcDetailParser>();
|
||||
services.TryAddSingleton<MsrcDiagnostics>();
|
||||
services.AddTransient<MsrcConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# MSRC Security Updates – Connector Notes
|
||||
|
||||
## API endpoints
|
||||
- **Vulnerability summaries** – `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerabilities` (requires `api-version=2024-08-01`, client credential bearer token).
|
||||
- **Vulnerability detail** – `GET https://api.msrc.microsoft.com/sug/v2.0/<locale>/vulnerability/{id}` (same headers/scopes).
|
||||
- **CVRF package** – the detail payload contains `cvrfUrl` pointing to a ZIP/JSON asset that is stable per revision. We surface the URL as a reference and capture it in metadata for future offline bundling.
|
||||
|
||||
## Cursor behaviour
|
||||
- Connector keeps a `lastModifiedCursor` and replays the previous 10 minutes on every fetch to cover late revisions.
|
||||
- MSRC limits requests to ~60/minute; `requestDelay` defaults to 250 ms and is configurable.
|
||||
|
||||
## Authentication
|
||||
- Uses Azure AD client credential flow against `https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token` with scope `api://api.msrc.microsoft.com/.default`.
|
||||
- Token refresh happens lazily and is cached until 60 seconds before expiry.
|
||||
- Configuration values (`tenantId`, `clientId`, `clientSecret`) must be supplied via `concelier:sources:vndr:msrc`.
|
||||
|
||||
## CVRF handling
|
||||
- Detail payload is persisted with the `cvrfUrl` in metadata (`msrc.cvrfUrl`).
|
||||
- Mapping stage emits the CVRF link as a reference so offline runs can fetch it later. When `DownloadCvrf` is enabled the connector also saves the ZIP artefact to the documents store (marked as `msrc.cvrf=true`) for Offline Kit staging.
|
||||
@@ -0,0 +1,16 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,11 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Confirmed REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) + CVRF ZIP download flow, required Azure AD client-credentials scope (`api://api.msrc.microsoft.com/.default`), mandatory `api-version=2024-08-01` header, and delta params (`lastModifiedStartDateTime`, `lastModifiedEndDateTime`). Findings recorded in `docs/concelier-connector-research-20251011.md`.|
|
||||
|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**DONE (2025-10-15)** – Added `MsrcApiClient` + token provider, cursor overlap handling, and detail persistence via GridFS (metadata carries CVRF URL + timestamps). State tracks `lastModifiedCursor` with configurable overlap/backoff. **Next:** coordinate with Tools on shared state-seeding helper once CVRF download flag stabilises.|
|
||||
|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**DONE (2025-10-15)** – Implemented `MsrcDetailParser`/DTOs capturing threats, remediations, KB IDs, CVEs, CVSS, and affected products (build/platform metadata preserved).|
|
||||
|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**DONE (2025-10-15)** – `MsrcMapper` emits aliases (MSRC ID/CVE/KB), references (release notes + CVRF), vendor packages with `msrc.build` normalized rules, and CVSS provenance.|
|
||||
|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-15)** – Added `StellaOps.Concelier.Connector.Vndr.Msrc.Tests` with canned token/summary/detail responses and snapshot assertions via Mongo2Go. Fixtures regenerate via `UPDATE_MSRC_FIXTURES`.|
|
||||
|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Introduced `MsrcDiagnostics` meter (summary/detail/parse/map metrics), structured fetch logs, README updates, and Ops brief `docs/ops/concelier-msrc-operations.md` covering AAD onboarding + CVRF handling.|
|
||||
|FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Completed memo outline recommending dual-path (REST for incremental, CVRF for offline); implementation hinges on `FEEDCONN-MSRC-02-008` AAD onboarding for token acquisition.|
|
||||
|FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**DONE (2025-10-15)** – Coordinated Ops handoff; drafted AAD onboarding brief (`docs/ops/concelier-msrc-operations.md`) with app registration requirements, secret rotation policy, sample configuration, and CVRF mirroring guidance for Offline Kit.|
|
||||
Reference in New Issue
Block a user