Resolve Concelier/Excititor merge conflicts

This commit is contained in:
master
2025-10-20 14:19:25 +03:00
2687 changed files with 212646 additions and 85913 deletions

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Connector for Microsoft Security Response Center (MSRC) CSAF advisories, handling authenticated downloads, throttling, and raw document persistence.
## Scope
- MSRC API onboarding (AAD client credentials), metadata discovery, and CSAF listing retrieval.
- Download pipeline with retry/backoff, checksum validation, and document deduplication.
- Mapping MSRC-specific identifiers (CVE, ADV, KB) and remediation guidance into connector metadata.
- Emitting trust metadata (AAD issuer, signing certificates) for policy weighting.
## Participants
- Worker schedules MSRC pulls honoring rate limits; WebService may trigger manual runs for urgent updates.
- CSAF normalizer processes retrieved documents into claims.
- Policy subsystem references connector trust hints for consensus scoring.
## Interfaces & contracts
- Implements `IVexConnector`, requires configuration options for tenant/client/secret or managed identity.
- Uses shared HTTP helpers, resume markers, and telemetry from Abstractions module.
## In/Out of scope
In: authenticated fetching, raw document storage, metadata mapping, retry logic.
Out: normalization/export, attestation, storage implementations (handled elsewhere).
## Observability & security expectations
- Log request batches, rate-limit responses, and token refresh events without leaking secrets.
- Track metrics for documents fetched, retries, and failure categories.
## Tests
- Connector tests with mocked MSRC endpoints and AAD token flow will live in `../StellaOps.Excititor.Connectors.MSRC.CSAF.Tests`.

View File

@@ -0,0 +1,185 @@
using System;
using System.Collections.Generic;
using System.IO.Abstractions;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
public interface IMsrcTokenProvider
{
ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private const string CachePrefix = "StellaOps.Excititor.Connectors.MSRC.CSAF.Token";
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
private readonly IFileSystem _fileSystem;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly TimeProvider _timeProvider;
private readonly MsrcConnectorOptions _options;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IMemoryCache cache,
IFileSystem fileSystem,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcTokenProvider> logger,
TimeProvider? timeProvider = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate(_fileSystem);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async ValueTask<MsrcAccessToken> GetAccessTokenAsync(CancellationToken cancellationToken)
{
if (_options.PreferOfflineToken)
{
return LoadOfflineToken();
}
var cacheKey = CreateCacheKey();
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out var cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_cache.TryGetValue<MsrcAccessToken>(cacheKey, out cachedToken) &&
cachedToken is not null &&
!cachedToken.IsExpired(_timeProvider.GetUtcNow()))
{
return cachedToken;
}
var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false);
var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue
? (DateTimeOffset?)null
: token.ExpiresAt;
var options = new MemoryCacheEntryOptions();
if (absoluteExpiration.HasValue)
{
options.AbsoluteExpiration = absoluteExpiration.Value;
}
_cache.Set(cacheKey, token, options);
return token;
}
finally
{
_refreshLock.Release();
}
}
private MsrcAccessToken LoadOfflineToken()
{
if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken))
{
return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue);
}
if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath))
{
throw new InvalidOperationException("Offline token mode is enabled but no token was provided.");
}
if (!_fileSystem.File.Exists(_options.OfflineTokenPath))
{
throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist.");
}
var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim();
if (string.IsNullOrEmpty(token))
{
throw new InvalidOperationException("Offline token file was empty.");
}
return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue);
}
private async Task<MsrcAccessToken> RequestTokenAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName);
using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret!,
["grant_type"] = "client_credentials",
["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}");
}
var tokenResponse = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("Token endpoint returned an empty payload.");
if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken))
{
throw new InvalidOperationException("Token endpoint response did not include an access_token.");
}
var now = _timeProvider.GetUtcNow();
var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds
? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds)
: now.AddMinutes(5);
return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt);
}
private string CreateCacheKey()
=> $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}";
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string? AccessToken { get; init; }
[JsonPropertyName("token_type")]
public string? TokenType { get; init; }
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}
public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}

View File

@@ -0,0 +1,211 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
public sealed class MsrcConnectorOptions
{
public const string TokenClientName = "excititor.connector.msrc.token";
public const string DefaultScope = "https://api.msrc.microsoft.com/.default";
public const string ApiClientName = "excititor.connector.msrc.api";
public const string DefaultBaseUri = "https://api.msrc.microsoft.com/sug/v2.0/";
public const string DefaultLocale = "en-US";
public const string DefaultApiVersion = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier (GUID or domain).
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application secret for client credential flow.
/// </summary>
public string? ClientSecret { get; set; }
/// <summary>
/// OAuth scope requested for MSRC API access.
/// </summary>
public string Scope { get; set; } = DefaultScope;
/// <summary>
/// When true, token acquisition is skipped and the connector expects offline handling.
/// </summary>
public bool PreferOfflineToken { get; set; }
/// <summary>
/// Optional path to a pre-provisioned bearer token used when <see cref="PreferOfflineToken"/> is enabled.
/// </summary>
public string? OfflineTokenPath { get; set; }
/// <summary>
/// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles).
/// </summary>
public string? StaticAccessToken { get; set; }
/// <summary>
/// Minimum buffer (seconds) subtracted from token expiry before refresh.
/// </summary>
public int ExpiryLeewaySeconds { get; set; } = 60;
/// <summary>
/// Base URI for MSRC Security Update Guide API.
/// </summary>
public Uri BaseUri { get; set; } = new(DefaultBaseUri, UriKind.Absolute);
/// <summary>
/// Locale requested when fetching summaries.
/// </summary>
public string Locale { get; set; } = DefaultLocale;
/// <summary>
/// API version appended to MSRC requests.
/// </summary>
public string ApiVersion { get; set; } = DefaultApiVersion;
/// <summary>
/// Page size used while enumerating summaries.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Maximum CSAF advisories fetched per connector run.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Overlap window applied when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// Delay between CSAF downloads to respect rate limits.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Maximum retry attempts for summary/detail fetch operations.
/// </summary>
public int MaxRetryAttempts { get; set; } = 3;
/// <summary>
/// Base delay applied between retries (jitter handled by connector).
/// </summary>
public TimeSpan RetryBaseDelay { get; set; } = TimeSpan.FromSeconds(2);
/// <summary>
/// Optional lower bound for initial synchronisation when no cursor is stored.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
/// <summary>
/// Maximum number of document digests persisted for deduplication.
/// </summary>
public int MaxTrackedDigests { get; set; } = 2048;
public void Validate(IFileSystem? fileSystem = null)
{
if (PreferOfflineToken)
{
if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken))
{
throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled.");
}
}
else
{
if (string.IsNullOrWhiteSpace(TenantId))
{
throw new InvalidOperationException("TenantId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId is required when not operating in offline token mode.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode.");
}
}
if (string.IsNullOrWhiteSpace(Scope))
{
Scope = DefaultScope;
}
if (ExpiryLeewaySeconds < 10)
{
ExpiryLeewaySeconds = 10;
}
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("BaseUri must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("ApiVersion must be provided.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10))
{
throw new InvalidOperationException($"{nameof(RequestDelay)} must be between 0 and 10 seconds.");
}
if (MaxRetryAttempts <= 0 || MaxRetryAttempts > 10)
{
throw new InvalidOperationException($"{nameof(MaxRetryAttempts)} must be between 1 and 10.");
}
if (RetryBaseDelay < TimeSpan.Zero || RetryBaseDelay > TimeSpan.FromMinutes(5))
{
throw new InvalidOperationException($"{nameof(RetryBaseDelay)} must be between 0 and 5 minutes.");
}
if (MaxTrackedDigests <= 0 || MaxTrackedDigests > 10000)
{
throw new InvalidOperationException($"{nameof(MaxTrackedDigests)} must be between 1 and 10000.");
}
if (!string.IsNullOrWhiteSpace(OfflineTokenPath))
{
var fs = fileSystem ?? new FileSystem();
var directory = Path.GetDirectoryName(OfflineTokenPath);
if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory))
{
fs.Directory.CreateDirectory(directory);
}
}
}
}

View File

@@ -0,0 +1,58 @@
using System;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using System.IO.Abstractions;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection;
public static class MsrcConnectorServiceCollectionExtensions
{
public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action<MsrcConnectorOptions>? configure = null)
{
ArgumentNullException.ThrowIfNull(services);
services.TryAddSingleton<IMemoryCache, MemoryCache>();
services.TryAddSingleton<IFileSystem, FileSystem>();
services.AddOptions<MsrcConnectorOptions>()
.Configure(options => configure?.Invoke(options));
services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client =>
{
client.Timeout = TimeSpan.FromSeconds(30);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddHttpClient(MsrcConnectorOptions.ApiClientName)
.ConfigureHttpClient((provider, client) =>
{
var options = provider.GetRequiredService<IOptions<MsrcConnectorOptions>>().Value;
client.BaseAddress = options.BaseUri;
client.Timeout = TimeSpan.FromSeconds(60);
client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0");
client.DefaultRequestHeaders.Accept.ParseAdd("application/json");
})
.ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.All,
});
services.AddSingleton<IMsrcTokenProvider, MsrcTokenProvider>();
services.AddSingleton<IVexConnector, MsrcCsafConnector>();
return services;
}
}

View File

@@ -0,0 +1,581 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF;
public sealed class MsrcCsafConnector : VexConnectorBase
{
private const string QuarantineMetadataKey = "excititor.quarantine.reason";
private const string FormatMetadataKey = "msrc.csaf.format";
private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId";
private const string AdvisoryIdMetadataKey = "msrc.advisoryId";
private const string LastModifiedMetadataKey = "msrc.lastModified";
private const string ReleaseDateMetadataKey = "msrc.releaseDate";
private const string CvssSeverityMetadataKey = "msrc.severity";
private const string CvrfUrlMetadataKey = "msrc.cvrfUrl";
private static readonly VexConnectorDescriptor DescriptorInstance = new(
id: "excititor:msrc",
kind: VexProviderKind.Vendor,
displayName: "Microsoft MSRC CSAF")
{
Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.",
SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf),
Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"),
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IOptions<MsrcConnectorOptions> _options;
private readonly ILogger<MsrcCsafConnector> _logger;
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
};
private MsrcConnectorOptions? _validatedOptions;
public MsrcCsafConnector(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IVexConnectorStateRepository stateRepository,
IOptions<MsrcConnectorOptions> options,
ILogger<MsrcCsafConnector> logger,
TimeProvider timeProvider)
: base(DescriptorInstance, logger, timeProvider)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
LogConnectorEvent(
LogLevel.Information,
"validate",
"Validated MSRC CSAF connector options.",
new Dictionary<string, object?>
{
["baseUri"] = options.BaseUri.ToString(),
["locale"] = options.Locale,
["apiVersion"] = options.ApiVersion,
["pageSize"] = options.PageSize,
["maxAdvisories"] = options.MaxAdvisoriesPerFetch,
});
return ValueTask.CompletedTask;
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(
VexConnectorContext context,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var options = EnsureOptionsValidated();
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
var (from, to) = CalculateWindow(context.Since, state, options);
LogConnectorEvent(
LogLevel.Information,
"fetch.window",
$"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.",
new Dictionary<string, object?>
{
["from"] = from,
["to"] = to,
["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds,
});
var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false);
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
var digestList = new List<string>(knownDigests);
var latest = state?.LastUpdated ?? from;
var fetched = 0;
var stateChanged = false;
await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
if (fetched >= options.MaxAdvisoriesPerFetch)
{
break;
}
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided.");
continue;
}
var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl);
VexRawDocument? rawDocument = null;
try
{
rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary<string, object?>
{
["advisoryId"] = summary.Id,
["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
}, ex);
await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false);
continue;
}
if (!digestSet.Add(rawDocument.Digest))
{
LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed.");
continue;
}
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
digestList.Add(rawDocument.Digest);
stateChanged = true;
fetched++;
latest = DetermineLatest(summary, latest) ?? latest;
var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null;
if (quarantineReason is not null)
{
LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason}).");
continue;
}
yield return rawDocument;
if (options.RequestDelay > TimeSpan.Zero)
{
await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false);
}
}
if (stateChanged)
{
if (digestList.Count > options.MaxTrackedDigests)
{
var trimmed = digestList.Count - options.MaxTrackedDigests;
digestList.RemoveRange(0, trimmed);
}
var baseState = state ?? new VexConnectorState(
Descriptor.Id,
null,
ImmutableArray<string>.Empty,
ImmutableDictionary<string, string>.Empty,
null,
0,
null,
null);
var newState = baseState with
{
LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest,
DocumentDigests = digestList.ToImmutableArray(),
};
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
}
LogConnectorEvent(
LogLevel.Information,
"fetch.completed",
$"MSRC CSAF fetch completed with {fetched} new documents.",
new Dictionary<string, object?>
{
["fetched"] = fetched,
["stateChanged"] = stateChanged,
["lastUpdated"] = latest,
});
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing.");
private async Task<VexRawDocument> DownloadCsafAsync(
HttpClient client,
MsrcVulnerabilitySummary summary,
Uri documentUri,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, documentUri),
options,
cancellationToken).ConfigureAwait(false);
var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var validation = ValidateCsafPayload(payload);
var metadata = BuildMetadata(builder =>
{
builder.Add(AdvisoryIdMetadataKey, summary.Id);
builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id);
builder.Add(CvrfUrlMetadataKey, documentUri.ToString());
builder.Add(FormatMetadataKey, validation.Format);
if (!string.IsNullOrWhiteSpace(summary.Severity))
{
builder.Add(CvssSeverityMetadataKey, summary.Severity);
}
if (summary.LastModifiedDate is not null)
{
builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O"));
}
if (summary.ReleaseDate is not null)
{
builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O"));
}
if (!string.IsNullOrWhiteSpace(validation.QuarantineReason))
{
builder.Add(QuarantineMetadataKey, validation.QuarantineReason);
}
if (response.Headers.ETag is not null)
{
builder.Add("http.etag", response.Headers.ETag.Tag);
}
if (response.Content.Headers.LastModified is { } lastModified)
{
builder.Add("http.lastModified", lastModified.ToString("O"));
}
});
return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}");
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", options.Locale);
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
return client;
}
private async Task<HttpResponseMessage> SendWithRetryAsync(
HttpClient client,
Func<HttpRequestMessage> requestFactory,
MsrcConnectorOptions options,
CancellationToken cancellationToken)
{
Exception? lastError = null;
HttpResponseMessage? response = null;
for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++)
{
response?.Dispose();
using var request = requestFactory();
try
{
response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return response;
}
if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts)
{
response.EnsureSuccessStatusCode();
}
}
catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts)
{
lastError = ex;
LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex);
}
catch (Exception ex)
{
response?.Dispose();
throw;
}
await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false);
}
response?.Dispose();
throw lastError ?? new InvalidOperationException("MSRC request retries exhausted.");
}
private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt)
{
var baseDelay = options.RetryBaseDelay.TotalMilliseconds;
var multiplier = Math.Pow(2, Math.Max(0, attempt - 1));
var jitter = Random.Shared.NextDouble() * baseDelay * 0.25;
var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds);
return TimeSpan.FromMilliseconds(delayMs);
}
private async IAsyncEnumerable<MsrcVulnerabilitySummary> EnumerateSummariesAsync(
HttpClient client,
MsrcConnectorOptions options,
DateTimeOffset from,
DateTimeOffset to,
[EnumeratorCancellation] CancellationToken cancellationToken)
{
var fetched = 0;
var requestUri = BuildSummaryUri(options, from, to);
while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch)
{
using var response = await SendWithRetryAsync(
client,
() => new HttpRequestMessage(HttpMethod.Get, requestUri),
options,
cancellationToken).ConfigureAwait(false);
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var payload = await JsonSerializer.DeserializeAsync<MsrcSummaryResponse>(stream, _serializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
foreach (var summary in payload.Value)
{
if (string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
continue;
}
yield return summary;
fetched++;
if (fetched >= options.MaxAdvisoriesPerFetch)
{
yield break;
}
}
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri))
{
LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'.");
break;
}
}
}
private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to)
{
var baseText = options.BaseUri.ToString().TrimEnd('/');
var builder = new StringBuilder(baseText.Length + 128);
builder.Append(baseText);
if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase))
{
builder.Append("/vulnerabilities");
}
builder.Append("?");
builder.Append("$top=").Append(options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O")));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O")));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private (DateTimeOffset From, DateTimeOffset To) CalculateWindow(
DateTimeOffset? contextSince,
VexConnectorState? state,
MsrcConnectorOptions options)
{
var now = UtcNow();
var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30);
if (state?.LastUpdated is { } persisted && persisted > since)
{
since = persisted;
}
if (options.CursorOverlap > TimeSpan.Zero)
{
since = since.Add(-options.CursorOverlap);
}
if (since < now.AddYears(-20))
{
since = now.AddYears(-20);
}
return (since, now);
}
private static bool ShouldRetry(HttpStatusCode statusCode)
=> statusCode == HttpStatusCode.TooManyRequests ||
(int)statusCode >= 500;
private static bool IsTransient(Exception exception)
=> exception is HttpRequestException or IOException or TaskCanceledException;
private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl)
=> Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute)
? absolute
: new Uri(baseUri, cvrfUrl);
private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory<byte> payload)
{
try
{
if (IsZip(payload.Span))
{
using var zipStream = new MemoryStream(payload.ToArray(), writable: false);
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true);
var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
?? archive.Entries.FirstOrDefault();
if (entry is null)
{
return new CsafValidationResult("zip", "Zip archive did not contain any entries.");
}
using var entryStream = entry.Open();
using var reader = new StreamReader(entryStream, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("zip");
}
if (IsGzip(payload.Span))
{
using var input = new MemoryStream(payload.ToArray(), writable: false);
using var gzip = new GZipStream(input, CompressionMode.Decompress);
using var reader = new StreamReader(gzip, Encoding.UTF8);
using var json = JsonDocument.Parse(reader.ReadToEnd());
return CsafValidationResult.Valid("gzip");
}
using var jsonDocument = JsonDocument.Parse(payload.Span);
return CsafValidationResult.Valid("json");
}
catch (JsonException ex)
{
return new CsafValidationResult("json", $"JSON parse failed: {ex.Message}");
}
catch (InvalidDataException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
catch (EndOfStreamException ex)
{
return new CsafValidationResult("invalid", ex.Message);
}
}
private static bool IsZip(ReadOnlySpan<byte> content)
=> content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B;
private static bool IsGzip(ReadOnlySpan<byte> content)
=> content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B;
private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current)
{
var candidate = summary.LastModifiedDate ?? summary.ReleaseDate;
if (candidate is null)
{
return current;
}
if (current is null || candidate > current)
{
return candidate;
}
return current;
}
private MsrcConnectorOptions EnsureOptionsValidated()
{
if (_validatedOptions is not null)
{
return _validatedOptions;
}
var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered.");
options.Validate();
_validatedOptions = options;
return options;
}
private sealed record CsafValidationResult(string Format, string? QuarantineReason)
{
public static CsafValidationResult Valid(string format) => new(format, null);
}
}
internal sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
internal sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.0" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-CONN-MS-01-001 AAD onboarding & token cache|Team Excititor Connectors MSRC|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.|
|EXCITITOR-CONN-MS-01-002 CSAF download pipeline|Team Excititor Connectors MSRC|EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** Prereqs verified (EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003); drafting fetch/retry plan and storage wiring before implementation of CSAF package download, checksum validation, and quarantine flows.|
|EXCITITOR-CONN-MS-01-003 Trust metadata & provenance hints|Team Excititor Connectors MSRC|EXCITITOR-CONN-MS-01-002, EXCITITOR-POLICY-01-001|TODO Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration.|