Restore vendor connector internals and configure offline packages
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
root
2025-10-20 14:50:58 +03:00
parent 09b6a28172
commit 44ad31591c
59 changed files with 7512 additions and 3797 deletions

17
NuGet.config Normal file
View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear />
<add key="local" value="local-nuget" />
<add key="mirror" value="https://mirrors.ablera.dev/nuget/nuget-mirror/v3/index.json" />
</packageSources>
<packageSourceMapping>
<packageSource key="local">
<package pattern="Mongo2Go" />
<package pattern="Microsoft.Extensions.Http.Polly" />
</packageSource>
<packageSource key="mirror">
<package pattern="*" />
</packageSource>
</packageSourceMapping>
</configuration>

Binary file not shown.

View File

@@ -64,12 +64,13 @@ public class StandardClientProvisioningStoreTests
var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None);
Assert.True(result.Succeeded);
var document = Assert.Contains("signer", store.Documents);
Assert.Equal("attestor signer", document.Value.Properties[AuthorityClientMetadataKeys.Audiences]);
Assert.True(store.Documents.TryGetValue("signer", out var document));
Assert.NotNull(document);
Assert.Equal("attestor signer", document!.Properties[AuthorityClientMetadataKeys.Audiences]);
var descriptor = await provisioning.FindByClientIdAsync("signer", CancellationToken.None);
Assert.NotNull(descriptor);
Assert.Equal(new[] { "attestor", "signer" }, descriptor!.Audiences.OrderBy(value => value, StringComparer.Ordinal));
Assert.Equal(new[] { "attestor", "signer" }, descriptor!.AllowedAudiences.OrderBy(value => value, StringComparer.Ordinal));
}
[Fact]
@@ -101,8 +102,9 @@ public class StandardClientProvisioningStoreTests
await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None);
var document = Assert.Contains("mtls-client", store.Documents).Value;
var binding = Assert.Single(document.CertificateBindings);
Assert.True(store.Documents.TryGetValue("mtls-client", out var document));
Assert.NotNull(document);
var binding = Assert.Single(document!.CertificateBindings);
Assert.Equal("AABBCCDD", binding.Thumbprint);
Assert.Equal("01ff", binding.SerialNumber);
Assert.Equal("CN=mtls-client", binding.Subject);

View File

@@ -96,26 +96,3 @@ internal static class BootstrapInviteTypes
public const string User = "user";
public const string Client = "client";
}
internal sealed record BootstrapInviteRequest
{
public string Type { get; init; } = BootstrapInviteTypes.User;
public string? Token { get; init; }
public string? Provider { get; init; }
public string? Target { get; init; }
public DateTimeOffset? ExpiresAt { get; init; }
public string? IssuedBy { get; init; }
public IReadOnlyDictionary<string, string?>? Metadata { get; init; }
}
internal static class BootstrapInviteTypes
{
public const string User = "user";
public const string Client = "client";
}

View File

@@ -0,0 +1,104 @@
using System.Net;
namespace StellaOps.Concelier.Connector.CertBund.Configuration;
public sealed class CertBundOptions
{
public const string HttpClientName = "concelier.source.certbund";
/// <summary>
/// RSS feed providing the latest CERT-Bund advisories.
/// </summary>
public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss");
/// <summary>
/// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API).
/// </summary>
public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/");
/// <summary>
/// Detail API endpoint template; advisory identifier is appended as the <c>name</c> query parameter.
/// </summary>
public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory");
/// <summary>
/// Optional timeout override for feed/detail requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay applied between successive detail fetches to respect upstream politeness.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when a fetch attempt fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum number of advisories to enqueue per fetch iteration.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 50;
/// <summary>
/// Maximum number of advisory identifiers remembered to prevent re-processing.
/// </summary>
public int MaxKnownAdvisories { get; set; } = 512;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI.");
}
if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI.");
}
if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (MaxKnownAdvisories <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero.");
}
}
public Uri BuildDetailUri(string advisoryId)
{
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId));
}
var builder = new UriBuilder(DetailApiUri);
var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&";
builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}";
return builder.Uri;
}
}

View File

@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed record CertBundAdvisoryDto
{
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "de";
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("portalUri")]
public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("detailUri")]
public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
[JsonPropertyName("products")]
public IReadOnlyList<CertBundProductDto> Products { get; init; } = Array.Empty<CertBundProductDto>();
[JsonPropertyName("references")]
public IReadOnlyList<CertBundReferenceDto> References { get; init; } = Array.Empty<CertBundReferenceDto>();
}
public sealed record CertBundProductDto
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}
public sealed record CertBundReferenceDto
{
[JsonPropertyName("url")]
public string Url { get; init; } = string.Empty;
[JsonPropertyName("label")]
public string? Label { get; init; }
}

View File

@@ -0,0 +1,118 @@
using System;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal sealed record CertBundCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyCollection<string> KnownAdvisories,
DateTimeOffset? LastPublished,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>();
public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null);
public CertBundCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public CertBundCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public CertBundCursor WithKnownAdvisories(IEnumerable<string> advisories)
=> this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings };
public CertBundCursor WithLastPublished(DateTimeOffset? published)
=> this with { LastPublished = published };
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new BsonArray(KnownAdvisories),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
if (LastFetchAt.HasValue)
{
document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return document;
}
public static CertBundCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var knownAdvisories = ReadStringArray(document, "knownAdvisories");
var lastPublished = document.TryGetValue("lastPublished", out var publishedValue)
? ParseDate(publishedValue)
: null;
var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue)
? ParseDate(fetchValue)
: null;
return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuids;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyStrings;
}
return array.Select(element => element?.ToString() ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed class CertBundDetailParser
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly HtmlContentSanitizer _sanitizer;
public CertBundDetailParser(HtmlContentSanitizer sanitizer)
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload)
{
var detail = JsonSerializer.Deserialize<CertBundDetailResponse>(payload, SerializerOptions)
?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null.");
var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name.");
var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri);
return new CertBundAdvisoryDto
{
AdvisoryId = advisoryId,
Title = detail.Title ?? advisoryId,
Summary = detail.Summary,
ContentHtml = contentHtml,
Severity = detail.Severity,
Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!,
Published = detail.Published,
Modified = detail.Updated ?? detail.Published,
PortalUri = portalUri,
DetailUri = detailUri,
CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>(),
References = MapReferences(detail.References),
Products = MapProducts(detail.Products),
};
}
private static IReadOnlyList<CertBundReferenceDto> MapReferences(CertBundDetailReference[]? references)
{
if (references is null || references.Length == 0)
{
return Array.Empty<CertBundReferenceDto>();
}
return references
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
.Select(reference => new CertBundReferenceDto
{
Url = reference.Url!,
Label = reference.Label,
})
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CertBundProductDto> MapProducts(CertBundDetailProduct[]? products)
{
if (products is null || products.Length == 0)
{
return Array.Empty<CertBundProductDto>();
}
return products
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
.Select(product => new CertBundProductDto
{
Vendor = product.Vendor,
Name = product.Name,
Versions = product.Versions,
})
.ToArray();
}
}

View File

@@ -0,0 +1,60 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal sealed record CertBundDetailResponse
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string? Language { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("updated")]
public DateTimeOffset? Updated { get; init; }
[JsonPropertyName("cveIds")]
public string[]? CveIds { get; init; }
[JsonPropertyName("references")]
public CertBundDetailReference[]? References { get; init; }
[JsonPropertyName("products")]
public CertBundDetailProduct[]? Products { get; init; }
}
internal sealed record CertBundDetailReference
{
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("label")]
public string? Label { get; init; }
}
internal sealed record CertBundDetailProduct
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}

View File

@@ -0,0 +1,191 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
/// <summary>
/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector.
/// </summary>
public sealed class CertBundDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertBund";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _feedFetchAttempts;
private readonly Counter<long> _feedFetchSuccess;
private readonly Counter<long> _feedFetchFailures;
private readonly Histogram<long> _feedItemCount;
private readonly Histogram<long> _feedEnqueuedCount;
private readonly Histogram<double> _feedCoverageDays;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseCveCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapPackageCount;
private readonly Histogram<long> _mapAliasCount;
public CertBundDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_feedFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.attempts",
unit: "operations",
description: "Number of RSS feed load attempts.");
_feedFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.success",
unit: "operations",
description: "Number of successful RSS feed loads.");
_feedFetchFailures = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.failures",
unit: "operations",
description: "Number of RSS feed load failures.");
_feedItemCount = _meter.CreateHistogram<long>(
name: "certbund.feed.items.count",
unit: "items",
description: "Distribution of RSS item counts per fetch.");
_feedEnqueuedCount = _meter.CreateHistogram<long>(
name: "certbund.feed.enqueued.count",
unit: "documents",
description: "Distribution of advisory documents enqueued per fetch.");
_feedCoverageDays = _meter.CreateHistogram<double>(
name: "certbund.feed.coverage.days",
unit: "days",
description: "Coverage window in days between fetch time and the oldest published advisory in the feed.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.attempts",
unit: "operations",
description: "Number of detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.success",
unit: "operations",
description: "Number of detail fetches that persisted a document.");
_detailFetchNotModified = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.not_modified",
unit: "operations",
description: "Number of detail fetches returning HTTP 304.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.failures",
unit: "operations",
description: "Number of detail fetches that failed.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certbund.parse.success",
unit: "documents",
description: "Number of documents parsed into CERT-Bund DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certbund.parse.failures",
unit: "documents",
description: "Number of documents that failed to parse.");
_parseProductCount = _meter.CreateHistogram<long>(
name: "certbund.parse.products.count",
unit: "products",
description: "Distribution of product entries captured per advisory.");
_parseCveCount = _meter.CreateHistogram<long>(
name: "certbund.parse.cve.count",
unit: "aliases",
description: "Distribution of CVE identifiers captured per advisory.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certbund.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certbund.map.failures",
unit: "advisories",
description: "Number of mapping failures.");
_mapPackageCount = _meter.CreateHistogram<long>(
name: "certbund.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per advisory.");
_mapAliasCount = _meter.CreateHistogram<long>(
name: "certbund.map.aliases.count",
unit: "aliases",
description: "Distribution of alias counts per advisory.");
}
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
public void FeedFetchSuccess(int itemCount)
{
_feedFetchSuccess.Add(1);
if (itemCount >= 0)
{
_feedItemCount.Record(itemCount);
}
}
public void FeedFetchFailure(string reason = "error")
=> _feedFetchFailures.Add(1, ReasonTag(reason));
public void RecordFeedCoverage(double? coverageDays)
{
if (coverageDays is { } days && days >= 0)
{
_feedCoverageDays.Record(days);
}
}
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason = "error")
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchEnqueued(int count)
{
if (count >= 0)
{
_feedEnqueuedCount.Record(count);
}
}
public void ParseSuccess(int productCount, int cveCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (cveCount >= 0)
{
_parseCveCount.Record(cveCount);
}
}
public void ParseFailure(string reason = "error")
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int affectedPackages, int aliasCount)
{
_mapSuccess.Add(1);
if (affectedPackages >= 0)
{
_mapPackageCount.Record(affectedPackages);
}
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
}
public void MapFailure(string reason = "error")
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal static class CertBundDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(CertBundFeedItem item)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["certbund.advisoryId"] = item.AdvisoryId,
["certbund.portalUri"] = item.PortalUri.ToString(),
["certbund.published"] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Category))
{
metadata["certbund.category"] = item.Category!;
}
if (!string.IsNullOrWhiteSpace(item.Title))
{
metadata["certbund.title"] = item.Title!;
}
return metadata;
}
}

View File

@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertBund.Configuration;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed class CertBundFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertBundOptions _options;
private readonly ILogger<CertBundFeedClient> _logger;
private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1);
private volatile bool _bootstrapped;
public CertBundFeedClient(
IHttpClientFactory httpClientFactory,
IOptions<CertBundOptions> options,
ILogger<CertBundFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertBundFeedItem>> LoadAsync(CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName);
await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri);
request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertBundFeedItem>();
foreach (var element in document.Descendants("item"))
{
cancellationToken.ThrowIfCancellationRequested();
var linkValue = element.Element("link")?.Value?.Trim();
if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri))
{
continue;
}
var advisoryId = TryExtractNameParameter(portalUri);
if (string.IsNullOrWhiteSpace(advisoryId))
{
continue;
}
var detailUri = _options.BuildDetailUri(advisoryId);
var pubDateText = element.Element("pubDate")?.Value;
var published = ParseDate(pubDateText);
var title = element.Element("title")?.Value?.Trim();
var category = element.Element("category")?.Value?.Trim();
items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category));
}
return items;
}
private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken)
{
if (_bootstrapped)
{
return;
}
await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_bootstrapped)
{
return;
}
using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri);
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_bootstrapped = true;
}
finally
{
_bootstrapSemaphore.Release();
}
}
private static string? TryExtractNameParameter(Uri portalUri)
{
if (portalUri is null)
{
return null;
}
var query = portalUri.Query;
if (string.IsNullOrEmpty(query))
{
return null;
}
var trimmed = query.TrimStart('?');
foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex <= 0)
{
continue;
}
var key = pair[..separatorIndex].Trim();
if (!key.Equals("name", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var value = pair[(separatorIndex + 1)..];
return Uri.UnescapeDataString(value);
}
return null;
}
private static DateTimeOffset ParseDate(string? value)
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: DateTimeOffset.UtcNow;
}

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Concelier.Connector.CertBund.Internal;
using System;
public sealed record CertBundFeedItem(
string AdvisoryId,
Uri DetailUri,
Uri PortalUri,
DateTimeOffset Published,
string? Title,
string? Category);

View File

@@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal static class CertBundMapper
{
public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"advisory",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: dto.Language?.ToLowerInvariant() ?? "de",
published: dto.Published,
modified: dto.Modified,
severity: MapSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(CertBundAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4) { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
dto.DetailUri.ToString(),
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
kind: "reference",
sourceTag: "cert-bund",
summary: reference.Label,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified";
var name = Validation.TrimToNull(product.Name);
var identifier = name is null ? vendor : $"{vendor} {name}";
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var ranges = string.IsNullOrWhiteSpace(product.Versions)
? Array.Empty<AffectedVersionRange>()
: new[]
{
new AffectedVersionRange(
rangeKind: "string",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.Versions,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package-range",
product.Versions,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }))
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? MapSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.ToLowerInvariant() switch
{
"hoch" or "high" => "high",
"mittel" or "medium" => "medium",
"gering" or "low" => "low",
_ => severity.ToLowerInvariant(),
};
}
}

View File

@@ -103,15 +103,9 @@ internal sealed record CertCcCursor(
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is BsonString bsonString && Guid.TryParse(bsonString.AsString, out var parsed))
if (TryReadGuid(element, out var parsed))
{
results.Add(parsed);
continue;
}
if (element is BsonBinaryData binary && binary.GuidRepresentation != GuidRepresentation.Unspecified)
{
results.Add(binary.ToGuid());
}
}
@@ -148,6 +142,37 @@ internal sealed record CertCcCursor(
.ToArray();
}
private static bool TryReadGuid(BsonValue value, out Guid guid)
{
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
{
return true;
}
if (value is BsonBinaryData binary)
{
try
{
guid = binary.ToGuid();
return true;
}
catch (FormatException)
{
// ignore and fall back to byte array parsing
}
var bytes = binary.AsByteArray;
if (bytes.Length == 16)
{
guid = new Guid(bytes);
return true;
}
}
guid = default;
return false;
}
private static Guid[] NormalizeGuidSet(IEnumerable<Guid>? ids)
=> ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray;

View File

@@ -0,0 +1,124 @@
using System.Globalization;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
public sealed class CiscoOptions
{
public const string HttpClientName = "concelier.source.vndr.cisco";
public const string AuthHttpClientName = "concelier.source.vndr.cisco.auth";
public Uri BaseUri { get; set; } = new("https://api.cisco.com/security/advisories/v2/", UriKind.Absolute);
public Uri TokenEndpoint { get; set; } = new("https://id.cisco.com/oauth2/default/v1/token", UriKind.Absolute);
public string ClientId { get; set; } = string.Empty;
public string ClientSecret { get; set; } = string.Empty;
public int PageSize { get; set; } = 100;
public int MaxPagesPerFetch { get; set; } = 5;
public int MaxAdvisoriesPerFetch { get; set; } = 200;
public TimeSpan InitialBackfillWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan TokenRefreshSkew { get; set; } = TimeSpan.FromMinutes(1);
public string LastModifiedPathTemplate { get; set; } = "advisories/lastmodified/{0}";
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cisco BaseUri must be an absolute URI.");
}
if (TokenEndpoint is null || !TokenEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("Cisco TokenEndpoint must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("Cisco clientId must be configured.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("Cisco clientSecret must be configured.");
}
if (PageSize is < 1 or > 100)
{
throw new InvalidOperationException("Cisco PageSize must be between 1 and 100.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("Cisco MaxPagesPerFetch must be greater than zero.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Cisco MaxAdvisoriesPerFetch must be greater than zero.");
}
if (InitialBackfillWindow <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco InitialBackfillWindow must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco RequestDelay cannot be negative.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco RequestTimeout must be positive.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco FailureBackoff must be positive.");
}
if (TokenRefreshSkew < TimeSpan.FromSeconds(5))
{
throw new InvalidOperationException("Cisco TokenRefreshSkew must be at least 5 seconds.");
}
if (string.IsNullOrWhiteSpace(LastModifiedPathTemplate))
{
throw new InvalidOperationException("Cisco LastModifiedPathTemplate must be configured.");
}
}
public Uri BuildLastModifiedUri(DateOnly date, int pageIndex, int pageSize)
{
if (pageIndex < 1)
{
throw new ArgumentOutOfRangeException(nameof(pageIndex), pageIndex, "Page index must be >= 1.");
}
if (pageSize is < 1 or > 100)
{
throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, "Page size must be between 1 and 100.");
}
var path = string.Format(CultureInfo.InvariantCulture, LastModifiedPathTemplate, date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
var builder = new UriBuilder(BaseUri);
var basePath = builder.Path.TrimEnd('/');
builder.Path = $"{basePath}/{path}".Replace("//", "/", StringComparison.Ordinal);
var query = $"pageIndex={pageIndex.ToString(CultureInfo.InvariantCulture)}&pageSize={pageSize.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(builder.Query) ? query : builder.Query.TrimStart('?') + "&" + query;
return builder.Uri;
}
}

View File

@@ -0,0 +1,145 @@
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed class CiscoAccessTokenProvider : IDisposable
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IOptionsMonitor<CiscoOptions> _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<CiscoAccessTokenProvider> _logger;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
private volatile AccessToken? _cached;
private bool _disposed;
public CiscoAccessTokenProvider(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<CiscoOptions> options,
TimeProvider? timeProvider,
ILogger<CiscoAccessTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<string> GetTokenAsync(CancellationToken cancellationToken)
=> await GetTokenInternalAsync(forceRefresh: false, cancellationToken).ConfigureAwait(false);
public void Invalidate()
=> _cached = null;
private async Task<string> GetTokenInternalAsync(bool forceRefresh, CancellationToken cancellationToken)
{
ThrowIfDisposed();
var options = _options.CurrentValue;
var now = _timeProvider.GetUtcNow();
var cached = _cached;
if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew)
{
return cached.Value;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
cached = _cached;
now = _timeProvider.GetUtcNow();
if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew)
{
return cached.Value;
}
var fresh = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false);
_cached = fresh;
return fresh.Value;
}
finally
{
_refreshLock.Release();
}
}
private async Task<AccessToken> RequestTokenAsync(CiscoOptions options, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CiscoOptions.AuthHttpClientName);
client.Timeout = options.RequestTimeout;
using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint);
request.Headers.Accept.Clear();
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["grant_type"] = "client_credentials",
["client_id"] = options.ClientId,
["client_secret"] = options.ClientSecret,
});
request.Content = content;
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var message = $"Cisco OAuth token request failed with status {(int)response.StatusCode} {response.StatusCode}.";
_logger.LogError("Cisco openVuln token request failed: {Message}; response={Preview}", message, preview);
throw new HttpRequestException(message);
}
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var payload = await JsonSerializer.DeserializeAsync<TokenResponse>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
if (payload is null || string.IsNullOrWhiteSpace(payload.AccessToken))
{
throw new InvalidOperationException("Cisco OAuth token response did not include an access token.");
}
var expiresIn = payload.ExpiresIn > 0 ? TimeSpan.FromSeconds(payload.ExpiresIn) : TimeSpan.FromHours(1);
var now = _timeProvider.GetUtcNow();
var expiresAt = now + expiresIn;
_logger.LogInformation("Cisco openVuln token issued; expires in {ExpiresIn}", expiresIn);
return new AccessToken(payload.AccessToken, expiresAt);
}
public async Task<string> RefreshAsync(CancellationToken cancellationToken)
=> await GetTokenInternalAsync(forceRefresh: true, cancellationToken).ConfigureAwait(false);
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CiscoAccessTokenProvider));
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
_refreshLock.Dispose();
_disposed = true;
}
private sealed record AccessToken(string Value, DateTimeOffset ExpiresAt);
private sealed record TokenResponse(
[property: JsonPropertyName("access_token")] string AccessToken,
[property: JsonPropertyName("expires_in")] int ExpiresIn,
[property: JsonPropertyName("token_type")] string? TokenType);
}

View File

@@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed record CiscoAdvisoryDto(
string AdvisoryId,
string Title,
string? Summary,
string? Severity,
DateTimeOffset? Published,
DateTimeOffset? Updated,
string? PublicationUrl,
string? CsafUrl,
string? CvrfUrl,
double? CvssBaseScore,
IReadOnlyList<string> Cves,
IReadOnlyList<string> BugIds,
IReadOnlyList<CiscoAffectedProductDto> Products);
public sealed record CiscoAffectedProductDto(
string Name,
string? ProductId,
string? Version,
IReadOnlyCollection<string> Statuses);

View File

@@ -0,0 +1,64 @@
using System;
using System.IO;
using System.Net.Http;
using System.Text;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public interface ICiscoCsafClient
{
Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken);
}
public class CiscoCsafClient : ICiscoCsafClient
{
private static readonly string[] AcceptHeaders = { "application/json", "application/csaf+json", "application/vnd.cisco.csaf+json" };
private readonly SourceFetchService _fetchService;
private readonly ILogger<CiscoCsafClient> _logger;
public CiscoCsafClient(SourceFetchService fetchService, ILogger<CiscoCsafClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public virtual async Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(url))
{
return null;
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
_logger.LogWarning("Cisco CSAF URL '{Url}' is not a valid absolute URI.", url);
return null;
}
try
{
var request = new SourceFetchRequest(CiscoOptions.HttpClientName, VndrCiscoConnectorPlugin.SourceName, uri)
{
AcceptHeaders = AcceptHeaders,
};
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("Cisco CSAF download returned status {Status} for {Url}", result.StatusCode, url);
return null;
}
return System.Text.Encoding.UTF8.GetString(result.Content);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException or InvalidOperationException)
{
_logger.LogWarning(ex, "Cisco CSAF download failed for {Url}", url);
return null;
}
}
}

View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed record CiscoCsafData(
IReadOnlyDictionary<string, CiscoCsafProduct> Products,
IReadOnlyDictionary<string, IReadOnlyCollection<string>> ProductStatuses);
internal sealed record CiscoCsafProduct(string ProductId, string Name);

View File

@@ -0,0 +1,123 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal static class CiscoCsafParser
{
public static CiscoCsafData Parse(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return new CiscoCsafData(
Products: new Dictionary<string, CiscoCsafProduct>(0, StringComparer.OrdinalIgnoreCase),
ProductStatuses: new Dictionary<string, IReadOnlyCollection<string>>(0, StringComparer.OrdinalIgnoreCase));
}
using var document = JsonDocument.Parse(content);
var root = document.RootElement;
var products = ParseProducts(root);
var statuses = ParseStatuses(root);
return new CiscoCsafData(products, statuses);
}
private static IReadOnlyDictionary<string, CiscoCsafProduct> ParseProducts(JsonElement root)
{
var dictionary = new Dictionary<string, CiscoCsafProduct>(StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("product_tree", out var productTree))
{
return dictionary;
}
if (productTree.TryGetProperty("full_product_names", out var fullProductNames)
&& fullProductNames.ValueKind == JsonValueKind.Array)
{
foreach (var entry in fullProductNames.EnumerateArray())
{
var productId = entry.TryGetProperty("product_id", out var idElement) && idElement.ValueKind == JsonValueKind.String
? idElement.GetString()
: null;
if (string.IsNullOrWhiteSpace(productId))
{
continue;
}
var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String
? nameElement.GetString()
: null;
if (string.IsNullOrWhiteSpace(name))
{
name = productId;
}
dictionary[productId] = new CiscoCsafProduct(productId, name);
}
}
return dictionary;
}
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ParseStatuses(JsonElement root)
{
var map = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities)
|| vulnerabilities.ValueKind != JsonValueKind.Array)
{
return map.ToDictionary(
static kvp => kvp.Key,
static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(),
StringComparer.OrdinalIgnoreCase);
}
foreach (var vulnerability in vulnerabilities.EnumerateArray())
{
if (!vulnerability.TryGetProperty("product_status", out var productStatus)
|| productStatus.ValueKind != JsonValueKind.Object)
{
continue;
}
foreach (var property in productStatus.EnumerateObject())
{
var statusLabel = property.Name;
if (property.Value.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var productIdElement in property.Value.EnumerateArray())
{
if (productIdElement.ValueKind != JsonValueKind.String)
{
continue;
}
var productId = productIdElement.GetString();
if (string.IsNullOrWhiteSpace(productId))
{
continue;
}
if (!map.TryGetValue(productId, out var set))
{
set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
map[productId] = set;
}
set.Add(statusLabel);
}
}
}
return map.ToDictionary(
static kvp => kvp.Key,
static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(),
StringComparer.OrdinalIgnoreCase);
}
}

View File

@@ -0,0 +1,101 @@
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed record CiscoCursor(
DateTimeOffset? LastModified,
string? LastAdvisoryId,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
public static CiscoCursor Empty { get; } = new(null, null, EmptyGuidCollection, EmptyGuidCollection);
public BsonDocument ToBson()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModified.HasValue)
{
document["lastModified"] = LastModified.Value.UtcDateTime;
}
if (!string.IsNullOrWhiteSpace(LastAdvisoryId))
{
document["lastAdvisoryId"] = LastAdvisoryId;
}
return document;
}
public static CiscoCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
DateTimeOffset? lastModified = null;
if (document.TryGetValue("lastModified", out var lastModifiedValue))
{
lastModified = lastModifiedValue.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
string? lastAdvisoryId = null;
if (document.TryGetValue("lastAdvisoryId", out var idValue) && idValue.BsonType == BsonType.String)
{
var value = idValue.AsString.Trim();
if (value.Length > 0)
{
lastAdvisoryId = value;
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new CiscoCursor(lastModified, lastAdvisoryId, pendingDocuments, pendingMappings);
}
public CiscoCursor WithCheckpoint(DateTimeOffset lastModified, string advisoryId)
=> this with
{
LastModified = lastModified.ToUniversalTime(),
LastAdvisoryId = string.IsNullOrWhiteSpace(advisoryId) ? null : advisoryId.Trim(),
};
public CiscoCursor WithPendingDocuments(IEnumerable<Guid>? documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidCollection };
public CiscoCursor WithPendingMappings(IEnumerable<Guid>? mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidCollection };
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key)
{
if (!document.TryGetValue(key, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

View File

@@ -0,0 +1,82 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed class CiscoDiagnostics : IDisposable
{
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Cisco";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffected;
public CiscoDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchDocuments = _meter.CreateCounter<long>(
name: "cisco.fetch.documents",
unit: "documents",
description: "Number of Cisco advisories fetched.");
_fetchFailures = _meter.CreateCounter<long>(
name: "cisco.fetch.failures",
unit: "operations",
description: "Number of Cisco fetch failures.");
_fetchUnchanged = _meter.CreateCounter<long>(
name: "cisco.fetch.unchanged",
unit: "documents",
description: "Number of Cisco advisories skipped because they were unchanged.");
_parseSuccess = _meter.CreateCounter<long>(
name: "cisco.parse.success",
unit: "documents",
description: "Number of Cisco documents parsed successfully.");
_parseFailures = _meter.CreateCounter<long>(
name: "cisco.parse.failures",
unit: "documents",
description: "Number of Cisco documents that failed to parse.");
_mapSuccess = _meter.CreateCounter<long>(
name: "cisco.map.success",
unit: "documents",
description: "Number of Cisco advisories mapped successfully.");
_mapFailures = _meter.CreateCounter<long>(
name: "cisco.map.failures",
unit: "documents",
description: "Number of Cisco advisories that failed to map to canonical form.");
_mapAffected = _meter.CreateHistogram<long>(
name: "cisco.map.affected.packages",
unit: "packages",
description: "Distribution of affected package counts emitted per Cisco advisory.");
}
public Meter Meter => _meter;
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void MapAffected(int count)
{
if (count >= 0)
{
_mapAffected.Record(count);
}
}
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,190 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public class CiscoDtoFactory
{
private readonly ICiscoCsafClient _csafClient;
private readonly ILogger<CiscoDtoFactory> _logger;
public CiscoDtoFactory(ICiscoCsafClient csafClient, ILogger<CiscoDtoFactory> logger)
{
_csafClient = csafClient ?? throw new ArgumentNullException(nameof(csafClient));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<CiscoAdvisoryDto> CreateAsync(CiscoRawAdvisory raw, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(raw);
var advisoryId = raw.AdvisoryId?.Trim();
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cisco advisory is missing advisoryId.");
}
var title = string.IsNullOrWhiteSpace(raw.AdvisoryTitle) ? advisoryId : raw.AdvisoryTitle!.Trim();
var severity = SeverityNormalization.Normalize(raw.Sir);
var published = ParseDate(raw.FirstPublished);
var updated = ParseDate(raw.LastUpdated);
CiscoCsafData? csafData = null;
if (!string.IsNullOrWhiteSpace(raw.CsafUrl))
{
var csafContent = await _csafClient.TryFetchAsync(raw.CsafUrl, cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(csafContent))
{
try
{
csafData = CiscoCsafParser.Parse(csafContent!);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Cisco CSAF payload parsing failed for {AdvisoryId}", advisoryId);
}
}
}
var products = BuildProducts(raw, csafData);
var cves = NormalizeList(raw.Cves);
var bugIds = NormalizeList(raw.BugIds);
var cvss = ParseDouble(raw.CvssBaseScore);
return new CiscoAdvisoryDto(
AdvisoryId: advisoryId,
Title: title,
Summary: string.IsNullOrWhiteSpace(raw.Summary) ? null : raw.Summary!.Trim(),
Severity: severity,
Published: published,
Updated: updated,
PublicationUrl: NormalizeUrl(raw.PublicationUrl),
CsafUrl: NormalizeUrl(raw.CsafUrl),
CvrfUrl: NormalizeUrl(raw.CvrfUrl),
CvssBaseScore: cvss,
Cves: cves,
BugIds: bugIds,
Products: products);
}
private static IReadOnlyList<CiscoAffectedProductDto> BuildProducts(CiscoRawAdvisory raw, CiscoCsafData? csafData)
{
var map = new Dictionary<string, CiscoAffectedProductDto>(StringComparer.OrdinalIgnoreCase);
if (csafData is not null)
{
foreach (var entry in csafData.ProductStatuses)
{
var productId = entry.Key;
var name = csafData.Products.TryGetValue(productId, out var product)
? product.Name
: productId;
var statuses = NormalizeStatuses(entry.Value);
map[name] = new CiscoAffectedProductDto(
Name: name,
ProductId: productId,
Version: raw.Version?.Trim(),
Statuses: statuses);
}
}
var rawProducts = NormalizeList(raw.ProductNames);
foreach (var productName in rawProducts)
{
if (map.ContainsKey(productName))
{
continue;
}
map[productName] = new CiscoAffectedProductDto(
Name: productName,
ProductId: null,
Version: raw.Version?.Trim(),
Statuses: new[] { AffectedPackageStatusCatalog.KnownAffected });
}
return map.Count == 0
? Array.Empty<CiscoAffectedProductDto>()
: map.Values
.OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase)
.ThenBy(static p => p.ProductId, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyCollection<string> NormalizeStatuses(IEnumerable<string> statuses)
{
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var status in statuses)
{
if (AffectedPackageStatusCatalog.TryNormalize(status, out var normalized))
{
set.Add(normalized);
}
else if (!string.IsNullOrWhiteSpace(status))
{
set.Add(status.Trim().ToLowerInvariant());
}
}
if (set.Count == 0)
{
set.Add(AffectedPackageStatusCatalog.KnownAffected);
}
return set;
}
private static IReadOnlyList<string> NormalizeList(IEnumerable<string>? items)
{
if (items is null)
{
return Array.Empty<string>();
}
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var item in items)
{
if (!string.IsNullOrWhiteSpace(item))
{
set.Add(item.Trim());
}
}
return set.Count == 0 ? Array.Empty<string>() : set.ToArray();
}
private static double? ParseDouble(string? value)
=> double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)
? parsed
: null;
private static DateTimeOffset? ParseDate(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed.ToUniversalTime();
}
return null;
}
private static string? NormalizeUrl(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value.Trim(), UriKind.Absolute, out var uri) ? uri.ToString() : null;
}
}

View File

@@ -0,0 +1,263 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common.Packages;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public static class CiscoMapper
{
public static Advisory Map(CiscoAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
var fetchProvenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime());
var mapProvenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"map",
dto.AdvisoryId,
recordedAt);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affected = BuildAffectedPackages(dto, recordedAt);
return new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: "en",
published: dto.Published,
modified: dto.Updated,
severity: dto.Severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affected,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { fetchProvenance, mapProvenance });
}
private static IReadOnlyList<string> BuildAliases(CiscoAdvisoryDto dto)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
dto.AdvisoryId,
};
foreach (var cve in dto.Cves)
{
if (!string.IsNullOrWhiteSpace(cve))
{
set.Add(cve.Trim());
}
}
foreach (var bugId in dto.BugIds)
{
if (!string.IsNullOrWhiteSpace(bugId))
{
set.Add(bugId.Trim());
}
}
if (dto.PublicationUrl is not null)
{
set.Add(dto.PublicationUrl);
}
return set.Count == 0
? Array.Empty<string>()
: set.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CiscoAdvisoryDto dto, DateTimeOffset recordedAt)
{
var list = new List<AdvisoryReference>(3);
AddReference(list, dto.PublicationUrl, "publication", recordedAt);
AddReference(list, dto.CvrfUrl, "cvrf", recordedAt);
AddReference(list, dto.CsafUrl, "csaf", recordedAt);
return list.Count == 0
? Array.Empty<AdvisoryReference>()
: list.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static void AddReference(ICollection<AdvisoryReference> references, string? url, string kind, DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
return;
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
$"reference:{kind}",
uri.ToString(),
recordedAt);
try
{
references.Add(new AdvisoryReference(
url: uri.ToString(),
kind: kind,
sourceTag: null,
summary: null,
provenance: provenance));
}
catch (ArgumentException)
{
// ignore invalid URLs
}
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(CiscoAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
if (string.IsNullOrWhiteSpace(product.Name))
{
continue;
}
var range = BuildVersionRange(product, recordedAt);
var statuses = BuildStatuses(product, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"affected",
product.ProductId ?? product.Name,
recordedAt),
};
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: product.Name,
platform: null,
versionRanges: range is null ? Array.Empty<AffectedVersionRange>() : new[] { range },
statuses: statuses,
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages.OrderBy(static p => p.Identifier, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static AffectedVersionRange? BuildVersionRange(CiscoAffectedProductDto product, DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(product.Version))
{
return null;
}
var version = product.Version.Trim();
RangePrimitives? primitives = null;
string rangeKind = "vendor";
string? rangeExpression = version;
if (PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized))
{
var semver = new SemVerPrimitive(
Introduced: null,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: normalized);
primitives = new RangePrimitives(semver, null, null, BuildVendorExtensions(product));
rangeKind = "semver";
rangeExpression = normalized;
}
else
{
primitives = new RangePrimitives(null, null, null, BuildVendorExtensions(product, includeVersion: true));
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"range",
product.ProductId ?? product.Name,
recordedAt);
return new AffectedVersionRange(
rangeKind: rangeKind,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: rangeExpression,
provenance: provenance,
primitives: primitives);
}
private static IReadOnlyDictionary<string, string>? BuildVendorExtensions(CiscoAffectedProductDto product, bool includeVersion = false)
{
var dictionary = new Dictionary<string, string>(StringComparer.Ordinal);
if (!string.IsNullOrWhiteSpace(product.ProductId))
{
dictionary["cisco.productId"] = product.ProductId!;
}
if (includeVersion && !string.IsNullOrWhiteSpace(product.Version))
{
dictionary["cisco.version.raw"] = product.Version!;
}
return dictionary.Count == 0 ? null : dictionary;
}
private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(CiscoAffectedProductDto product, DateTimeOffset recordedAt)
{
if (product.Statuses is null || product.Statuses.Count == 0)
{
return Array.Empty<AffectedPackageStatus>();
}
var list = new List<AffectedPackageStatus>(product.Statuses.Count);
foreach (var status in product.Statuses)
{
if (!AffectedPackageStatusCatalog.TryNormalize(status, out var normalized)
|| string.IsNullOrWhiteSpace(normalized))
{
continue;
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"status",
product.ProductId ?? product.Name,
recordedAt);
list.Add(new AffectedPackageStatus(normalized, provenance));
}
return list.Count == 0 ? Array.Empty<AffectedPackageStatus>() : list;
}
}

View File

@@ -0,0 +1,101 @@
using System.IO;
using System.Net;
using System.Net.Http.Headers;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed class CiscoOAuthMessageHandler : DelegatingHandler
{
private readonly CiscoAccessTokenProvider _tokenProvider;
private readonly ILogger<CiscoOAuthMessageHandler> _logger;
public CiscoOAuthMessageHandler(
CiscoAccessTokenProvider tokenProvider,
ILogger<CiscoOAuthMessageHandler> logger)
{
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
HttpRequestMessage? retryTemplate = null;
try
{
retryTemplate = await CloneRequestAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (IOException)
{
// Unable to buffer content; retry will fail if needed.
retryTemplate = null;
}
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false));
var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.Unauthorized)
{
return response;
}
response.Dispose();
_logger.LogWarning("Cisco openVuln request returned 401 Unauthorized; refreshing access token.");
await _tokenProvider.RefreshAsync(cancellationToken).ConfigureAwait(false);
if (retryTemplate is null)
{
_tokenProvider.Invalidate();
throw new HttpRequestException("Cisco openVuln request returned 401 Unauthorized and could not be retried.");
}
retryTemplate.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false));
try
{
var retryResponse = await base.SendAsync(retryTemplate, cancellationToken).ConfigureAwait(false);
if (retryResponse.StatusCode == HttpStatusCode.Unauthorized)
{
_tokenProvider.Invalidate();
}
return retryResponse;
}
finally
{
retryTemplate.Dispose();
}
}
private static async Task<HttpRequestMessage?> CloneRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var clone = new HttpRequestMessage(request.Method, request.RequestUri)
{
Version = request.Version,
VersionPolicy = request.VersionPolicy,
};
foreach (var header in request.Headers)
{
clone.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
if (request.Content is not null)
{
using var memory = new MemoryStream();
await request.Content.CopyToAsync(memory, cancellationToken).ConfigureAwait(false);
memory.Position = 0;
var buffer = memory.ToArray();
var contentClone = new ByteArrayContent(buffer);
foreach (var header in request.Content.Headers)
{
contentClone.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
clone.Content = contentClone;
}
return clone;
}
}

View File

@@ -0,0 +1,196 @@
using System.Diagnostics.CodeAnalysis;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed class CiscoOpenVulnClient
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString,
};
private readonly SourceFetchService _fetchService;
private readonly IOptionsMonitor<CiscoOptions> _options;
private readonly ILogger<CiscoOpenVulnClient> _logger;
private readonly string _sourceName;
public CiscoOpenVulnClient(
SourceFetchService fetchService,
IOptionsMonitor<CiscoOptions> options,
ILogger<CiscoOpenVulnClient> logger,
string sourceName)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_sourceName = sourceName ?? throw new ArgumentNullException(nameof(sourceName));
}
internal async Task<CiscoAdvisoryPage?> FetchAsync(DateOnly date, int pageIndex, CancellationToken cancellationToken)
{
var options = _options.CurrentValue;
var requestUri = options.BuildLastModifiedUri(date, pageIndex, options.PageSize);
var request = new SourceFetchRequest(CiscoOptions.HttpClientName, _sourceName, requestUri)
{
AcceptHeaders = new[] { "application/json" },
TimeoutOverride = options.RequestTimeout,
};
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("Cisco openVuln request returned empty payload for {Uri} (status {Status})", requestUri, result.StatusCode);
return null;
}
return CiscoAdvisoryPage.Parse(result.Content);
}
}
internal sealed record CiscoAdvisoryPage(
IReadOnlyList<CiscoAdvisoryItem> Advisories,
CiscoPagination Pagination)
{
public bool HasMore => Pagination.PageIndex < Pagination.TotalPages;
public static CiscoAdvisoryPage Parse(byte[] content)
{
using var document = JsonDocument.Parse(content);
var root = document.RootElement;
var advisories = new List<CiscoAdvisoryItem>();
if (root.TryGetProperty("advisories", out var advisoriesElement) && advisoriesElement.ValueKind == JsonValueKind.Array)
{
foreach (var advisory in advisoriesElement.EnumerateArray())
{
if (!TryCreateItem(advisory, out var item))
{
continue;
}
advisories.Add(item);
}
}
var pagination = CiscoPagination.FromJson(root.TryGetProperty("pagination", out var paginationElement) ? paginationElement : default);
return new CiscoAdvisoryPage(advisories, pagination);
}
private static bool TryCreateItem(JsonElement advisory, [NotNullWhen(true)] out CiscoAdvisoryItem? item)
{
var rawJson = advisory.GetRawText();
var advisoryId = GetString(advisory, "advisoryId");
if (string.IsNullOrWhiteSpace(advisoryId))
{
item = null;
return false;
}
var lastUpdated = ParseDate(GetString(advisory, "lastUpdated"));
var firstPublished = ParseDate(GetString(advisory, "firstPublished"));
var severity = GetString(advisory, "sir");
var publicationUrl = GetString(advisory, "publicationUrl");
var csafUrl = GetString(advisory, "csafUrl");
var cvrfUrl = GetString(advisory, "cvrfUrl");
var cvss = GetString(advisory, "cvssBaseScore");
var cves = ReadStringArray(advisory, "cves");
var bugIds = ReadStringArray(advisory, "bugIDs");
var productNames = ReadStringArray(advisory, "productNames");
item = new CiscoAdvisoryItem(
advisoryId,
lastUpdated,
firstPublished,
severity,
publicationUrl,
csafUrl,
cvrfUrl,
cvss,
cves,
bugIds,
productNames,
rawJson);
return true;
}
private static string? GetString(JsonElement element, string propertyName)
=> element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String
? value.GetString()
: null;
private static DateTimeOffset? ParseDate(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, out var parsed))
{
return parsed.ToUniversalTime();
}
return null;
}
private static IReadOnlyList<string> ReadStringArray(JsonElement element, string property)
{
if (!element.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.Array)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var child in value.EnumerateArray())
{
if (child.ValueKind == JsonValueKind.String)
{
var text = child.GetString();
if (!string.IsNullOrWhiteSpace(text))
{
results.Add(text.Trim());
}
}
}
return results;
}
}
internal sealed record CiscoAdvisoryItem(
string AdvisoryId,
DateTimeOffset? LastUpdated,
DateTimeOffset? FirstPublished,
string? Severity,
string? PublicationUrl,
string? CsafUrl,
string? CvrfUrl,
string? CvssBaseScore,
IReadOnlyList<string> Cves,
IReadOnlyList<string> BugIds,
IReadOnlyList<string> ProductNames,
string RawJson)
{
public byte[] GetRawBytes() => Encoding.UTF8.GetBytes(RawJson);
}
internal sealed record CiscoPagination(int PageIndex, int PageSize, int TotalPages, int TotalRecords)
{
public static CiscoPagination FromJson(JsonElement element)
{
var pageIndex = element.TryGetProperty("pageIndex", out var index) && index.TryGetInt32(out var parsedIndex) ? parsedIndex : 1;
var pageSize = element.TryGetProperty("pageSize", out var size) && size.TryGetInt32(out var parsedSize) ? parsedSize : 0;
var totalPages = element.TryGetProperty("totalPages", out var pages) && pages.TryGetInt32(out var parsedPages) ? parsedPages : pageIndex;
var totalRecords = element.TryGetProperty("totalRecords", out var records) && records.TryGetInt32(out var parsedRecords) ? parsedRecords : 0;
return new CiscoPagination(pageIndex, pageSize, totalPages, totalRecords);
}
}

View File

@@ -0,0 +1,64 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public class CiscoRawAdvisory
{
[JsonPropertyName("advisoryId")]
public string? AdvisoryId { get; set; }
[JsonPropertyName("advisoryTitle")]
public string? AdvisoryTitle { get; set; }
[JsonPropertyName("publicationUrl")]
public string? PublicationUrl { get; set; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; set; }
[JsonPropertyName("csafUrl")]
public string? CsafUrl { get; set; }
[JsonPropertyName("summary")]
public string? Summary { get; set; }
[JsonPropertyName("sir")]
public string? Sir { get; set; }
[JsonPropertyName("firstPublished")]
public string? FirstPublished { get; set; }
[JsonPropertyName("lastUpdated")]
public string? LastUpdated { get; set; }
[JsonPropertyName("productNames")]
public List<string>? ProductNames { get; set; }
[JsonPropertyName("version")]
public string? Version { get; set; }
[JsonPropertyName("iosRelease")]
public string? IosRelease { get; set; }
[JsonPropertyName("cves")]
public List<string>? Cves { get; set; }
[JsonPropertyName("bugIDs")]
public List<string>? BugIds { get; set; }
[JsonPropertyName("cvssBaseScore")]
public string? CvssBaseScore { get; set; }
[JsonPropertyName("cvssTemporalScore")]
public string? CvssTemporalScore { get; set; }
[JsonPropertyName("cvssEnvironmentalScore")]
public string? CvssEnvironmentalScore { get; set; }
[JsonPropertyName("cvssBaseScoreVersion2")]
public string? CvssBaseScoreV2 { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
}

View File

@@ -0,0 +1,132 @@
using System.Globalization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
public sealed class MsrcOptions
{
public const string HttpClientName = "concelier.source.vndr.msrc";
public const string TokenClientName = "concelier.source.vndr.msrc.token";
public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute);
public string Locale { get; set; } = "en-US";
public string ApiVersion { get; set; } = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier used for client credential flow.
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD client secret used for token acquisition.
/// </summary>
public string ClientSecret { get; set; } = string.Empty;
/// <summary>
/// Scope requested during client-credential token acquisition.
/// </summary>
public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default";
/// <summary>
/// Maximum advisories to fetch per cycle.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Page size used when iterating the MSRC API.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Overlap window added when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// When enabled the connector downloads the CVRF artefact referenced by each advisory.
/// </summary>
public bool DownloadCvrf { get; set; } = false;
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Optional lower bound for the initial sync if the cursor is empty.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("MSRC base URI must be absolute.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("API version must be provided.");
}
if (!Guid.TryParse(TenantId, out _))
{
throw new InvalidOperationException("TenantId must be a valid GUID.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId must be provided.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret must be provided.");
}
if (string.IsNullOrWhiteSpace(Scope))
{
throw new InvalidOperationException("Scope must be provided.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}

View File

@@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcAdvisoryDto
{
public string AdvisoryId { get; init; } = string.Empty;
public string Title { get; init; } = string.Empty;
public string? Description { get; init; }
public string? Severity { get; init; }
public DateTimeOffset? ReleaseDate { get; init; }
public DateTimeOffset? LastModifiedDate { get; init; }
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> KbIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<MsrcAdvisoryThreat> Threats { get; init; } = Array.Empty<MsrcAdvisoryThreat>();
public IReadOnlyList<MsrcAdvisoryRemediation> Remediations { get; init; } = Array.Empty<MsrcAdvisoryRemediation>();
public IReadOnlyList<MsrcAdvisoryProduct> Products { get; init; } = Array.Empty<MsrcAdvisoryProduct>();
public double? CvssBaseScore { get; init; }
public string? CvssVector { get; init; }
public string? ReleaseNoteUrl { get; init; }
public string? CvrfUrl { get; init; }
}
public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity);
public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb);
public sealed record MsrcAdvisoryProduct(
string Identifier,
string? ProductName,
string? Platform,
string? Architecture,
string? BuildNumber,
string? Cpe);

View File

@@ -0,0 +1,138 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcApiClient
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
WriteIndented = false,
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly MsrcOptions _options;
private readonly ILogger<MsrcApiClient> _logger;
public MsrcApiClient(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IOptions<MsrcOptions> options,
ILogger<MsrcApiClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<MsrcVulnerabilitySummary>> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var results = new List<MsrcVulnerabilitySummary>();
var requestUri = BuildSummaryUri(fromInclusive, toExclusive);
while (requestUri is not null)
{
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}");
}
var payload = await response.Content.ReadFromJsonAsync<MsrcSummaryResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
results.AddRange(payload.Value);
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
requestUri = new Uri(payload.NextLink, UriKind.Absolute);
}
return results;
}
public Uri BuildDetailUri(string vulnerabilityId)
{
var uri = CreateDetailUriInternal(vulnerabilityId);
return uri;
}
public async Task<byte[]> FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var uri = CreateDetailUriInternal(vulnerabilityId);
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}");
}
return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale);
return client;
}
private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive)
{
var builder = new StringBuilder();
builder.Append(_options.BaseUri.ToString().TrimEnd('/'));
builder.Append("/vulnerabilities?");
builder.Append("$top=").Append(_options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O")));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O")));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private Uri CreateDetailUriInternal(string vulnerabilityId)
{
if (string.IsNullOrWhiteSpace(vulnerabilityId))
{
throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId));
}
var baseUri = _options.BaseUri.ToString().TrimEnd('/');
var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}";
return new Uri(path, UriKind.Absolute);
}
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal sealed record MsrcCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
DateTimeOffset? LastModifiedCursor)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidSet = Array.Empty<Guid>();
public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null);
public MsrcCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public MsrcCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp)
=> this with { LastModifiedCursor = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModifiedCursor.HasValue)
{
document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime;
}
return document;
}
public static MsrcCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var lastModified = document.TryGetValue("lastModifiedCursor", out var value)
? ParseDate(value)
: null;
return new MsrcCursor(pendingDocuments, pendingMappings, lastModified);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuidSet;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidSet;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,113 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcVulnerabilityDetailDto
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string VulnerabilityId { get; init; } = string.Empty;
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("threats")]
public IReadOnlyList<MsrcThreatDto> Threats { get; init; } = Array.Empty<MsrcThreatDto>();
[JsonPropertyName("remediations")]
public IReadOnlyList<MsrcRemediationDto> Remediations { get; init; } = Array.Empty<MsrcRemediationDto>();
[JsonPropertyName("affectedProducts")]
public IReadOnlyList<MsrcAffectedProductDto> AffectedProducts { get; init; } = Array.Empty<MsrcAffectedProductDto>();
[JsonPropertyName("cvssV3")]
public MsrcCvssDto? Cvss { get; init; }
[JsonPropertyName("releaseNoteUrl")]
public string? ReleaseNoteUrl { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}
public sealed record MsrcThreatDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
}
public sealed record MsrcRemediationDto
{
[JsonPropertyName("id")]
public string? Id { get; init; }
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("kbNumber")]
public string? KbNumber { get; init; }
}
public sealed record MsrcAffectedProductDto
{
[JsonPropertyName("productId")]
public string? ProductId { get; init; }
[JsonPropertyName("productName")]
public string? ProductName { get; init; }
[JsonPropertyName("cpe")]
public string? Cpe { get; init; }
[JsonPropertyName("platform")]
public string? Platform { get; init; }
[JsonPropertyName("architecture")]
public string? Architecture { get; init; }
[JsonPropertyName("buildNumber")]
public string? BuildNumber { get; init; }
}
public sealed record MsrcCvssDto
{
[JsonPropertyName("baseScore")]
public double? BaseScore { get; init; }
[JsonPropertyName("vectorString")]
public string? VectorString { get; init; }
}

View File

@@ -0,0 +1,71 @@
using System;
using System.Linq;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcDetailParser
{
public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail)
{
ArgumentNullException.ThrowIfNull(detail);
var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId;
var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray()
?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty<string>() : new[] { detail.CveNumber! });
var kbIds = detail.Remediations?
.Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber))
.Select(static remediation => remediation.KbNumber!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return new MsrcAdvisoryDto
{
AdvisoryId = advisoryId,
Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(),
Description = detail.Description,
Severity = detail.Severity,
ReleaseDate = detail.ReleaseDate,
LastModifiedDate = detail.LastModifiedDate,
CveIds = cveIds,
KbIds = kbIds,
Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat(
threat.Type ?? "unspecified",
threat.Description,
threat.Severity)).ToArray() ?? Array.Empty<MsrcAdvisoryThreat>(),
Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation(
remediation.Type ?? "unspecified",
remediation.Description,
remediation.Url,
remediation.KbNumber)).ToArray() ?? Array.Empty<MsrcAdvisoryRemediation>(),
Products = detail.AffectedProducts?.Select(product =>
new MsrcAdvisoryProduct(
BuildProductIdentifier(product),
product.ProductName,
product.Platform,
product.Architecture,
product.BuildNumber,
product.Cpe)).ToArray() ?? Array.Empty<MsrcAdvisoryProduct>(),
CvssBaseScore = detail.Cvss?.BaseScore,
CvssVector = detail.Cvss?.VectorString,
ReleaseNoteUrl = detail.ReleaseNoteUrl,
CvrfUrl = detail.CvrfUrl,
};
}
private static string BuildProductIdentifier(MsrcAffectedProductDto product)
{
var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName;
if (string.IsNullOrWhiteSpace(name))
{
name = "Unknown Product";
}
if (!string.IsNullOrWhiteSpace(product.BuildNumber))
{
return $"{name} build {product.BuildNumber}";
}
return name;
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Vndr.Msrc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchFailures;
private readonly Histogram<long> _summaryItemCount;
private readonly Histogram<double> _summaryWindowHours;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Histogram<long> _detailEnqueued;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseKbCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAliasCount;
private readonly Histogram<long> _mapAffectedCount;
public MsrcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_summaryFetchAttempts = _meter.CreateCounter<long>("msrc.summary.fetch.attempts", "operations");
_summaryFetchSuccess = _meter.CreateCounter<long>("msrc.summary.fetch.success", "operations");
_summaryFetchFailures = _meter.CreateCounter<long>("msrc.summary.fetch.failures", "operations");
_summaryItemCount = _meter.CreateHistogram<long>("msrc.summary.items.count", "items");
_summaryWindowHours = _meter.CreateHistogram<double>("msrc.summary.window.hours", "hours");
_detailFetchAttempts = _meter.CreateCounter<long>("msrc.detail.fetch.attempts", "operations");
_detailFetchSuccess = _meter.CreateCounter<long>("msrc.detail.fetch.success", "operations");
_detailFetchNotModified = _meter.CreateCounter<long>("msrc.detail.fetch.not_modified", "operations");
_detailFetchFailures = _meter.CreateCounter<long>("msrc.detail.fetch.failures", "operations");
_detailEnqueued = _meter.CreateHistogram<long>("msrc.detail.enqueued.count", "documents");
_parseSuccess = _meter.CreateCounter<long>("msrc.parse.success", "documents");
_parseFailures = _meter.CreateCounter<long>("msrc.parse.failures", "documents");
_parseProductCount = _meter.CreateHistogram<long>("msrc.parse.products.count", "products");
_parseKbCount = _meter.CreateHistogram<long>("msrc.parse.kb.count", "kb");
_mapSuccess = _meter.CreateCounter<long>("msrc.map.success", "advisories");
_mapFailures = _meter.CreateCounter<long>("msrc.map.failures", "advisories");
_mapAliasCount = _meter.CreateHistogram<long>("msrc.map.aliases.count", "aliases");
_mapAffectedCount = _meter.CreateHistogram<long>("msrc.map.affected.count", "packages");
}
public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1);
public void SummaryFetchSuccess(int count, double? windowHours)
{
_summaryFetchSuccess.Add(1);
if (count >= 0)
{
_summaryItemCount.Record(count);
}
if (windowHours is { } value && value >= 0)
{
_summaryWindowHours.Record(value);
}
}
public void SummaryFetchFailure(string reason)
=> _summaryFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason)
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailEnqueued(int count)
{
if (count >= 0)
{
_detailEnqueued.Record(count);
}
}
public void ParseSuccess(int productCount, int kbCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (kbCount >= 0)
{
_parseKbCount.Record(kbCount);
}
}
public void ParseFailure(string reason)
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int aliasCount, int packageCount)
{
_mapSuccess.Add(1);
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
if (packageCount >= 0)
{
_mapAffectedCount.Record(packageCount);
}
}
public void MapFailure(string reason)
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal static class MsrcDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
["msrc.id"] = summary.Id,
};
if (summary.LastModifiedDate.HasValue)
{
metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O");
}
if (summary.ReleaseDate.HasValue)
{
metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O");
}
if (!string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
metadata["msrc.cvrfUrl"] = summary.CvrfUrl!;
}
if (summary.CveNumbers.Count > 0)
{
metadata["msrc.cves"] = string.Join(",", summary.CveNumbers);
}
return metadata;
}
public static Dictionary<string, string> CreateCvrfMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = CreateMetadata(summary);
metadata["msrc.cvrf"] = "true";
return metadata;
}
}

View File

@@ -0,0 +1,239 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal static class MsrcMapper
{
public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var advisoryKey = dto.AdvisoryId;
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affectedPackages = BuildPackages(dto, recordedAt);
var cvssMetrics = BuildCvss(dto, recordedAt);
var provenance = new AdvisoryProvenance(
source: MsrcConnectorPlugin.SourceName,
kind: "advisory",
value: advisoryKey,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: advisoryKey,
title: dto.Title,
summary: dto.Description,
language: "en",
published: dto.ReleaseDate,
modified: dto.LastModifiedDate,
severity: NormalizeSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: cvssMetrics,
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(MsrcAdvisoryDto dto)
{
var aliases = new List<string> { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
foreach (var kb in dto.KbIds)
{
if (!string.IsNullOrWhiteSpace(kb))
{
aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}");
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl))
{
references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt));
}
if (!string.IsNullOrWhiteSpace(dto.CvrfUrl))
{
references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt));
}
foreach (var remediation in dto.Remediations)
{
if (!string.IsNullOrWhiteSpace(remediation.Url))
{
references.Add(CreateReference(
remediation.Url!,
string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference",
recordedAt,
remediation.Description));
}
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null)
=> new(
url,
kind: kind.ToLowerInvariant(),
sourceTag: "msrc",
summary: summary,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"reference",
url,
recordedAt,
new[] { ProvenanceFieldMasks.References }));
private static IReadOnlyList<AffectedPackage> BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier;
var provenance = new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var notes = new List<string>();
if (!string.IsNullOrWhiteSpace(product.Platform))
{
notes.Add($"platform:{product.Platform}");
}
if (!string.IsNullOrWhiteSpace(product.Architecture))
{
notes.Add($"arch:{product.Architecture}");
}
if (!string.IsNullOrWhiteSpace(product.Cpe))
{
notes.Add($"cpe:{product.Cpe}");
}
var range = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new AffectedVersionRange(
rangeKind: "custom",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: $"build:{product.BuildNumber}",
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package-range",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges })),
}
: Array.Empty<AffectedVersionRange>();
var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new NormalizedVersionRule(
scheme: "msrc.build",
type: NormalizedVersionRuleTypes.Exact,
value: product.BuildNumber,
notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n))))
}
: Array.Empty<NormalizedVersionRule>();
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: identifier,
platform: product.Platform,
versionRanges: range,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedRules));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CvssMetric> BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector))
{
return Array.Empty<CvssMetric>();
}
var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value);
return new[]
{
new CvssMetric(
version: "3.1",
vector: dto.CvssVector!,
baseScore: dto.CvssBaseScore.Value,
baseSeverity: severity,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"cvss",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.CvssMetrics })),
};
}
private static string CvssSeverityFromScore(double score)
=> score switch
{
< 0 => "none",
< 4 => "low",
< 7 => "medium",
< 9 => "high",
_ => "critical",
};
private static string? NormalizeSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.Trim().ToLowerInvariant();
}
}

View File

@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
public sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public interface IMsrcTokenProvider
{
Task<string> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly MsrcOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
private AccessToken? _currentToken;
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IOptions<MsrcOptions> options,
TimeProvider? timeProvider,
ILogger<MsrcTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
{
var token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
_logger.LogInformation("Requesting new MSRC access token");
var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName);
var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret,
["grant_type"] = "client_credentials",
["scope"] = _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("AAD token response was null.");
var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60);
_currentToken = new AccessToken(payload.AccessToken, expiresAt);
return payload.AccessToken;
}
finally
{
_refreshLock.Release();
}
}
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string AccessToken { get; init; } = string.Empty;
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}

View File

@@ -17,6 +17,7 @@ using System.Collections.Immutable;
using System.IO.Abstractions.TestingHelpers;
using Xunit;
using System.Threading;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors;
@@ -159,10 +160,10 @@ public sealed class CiscoCsafConnectorTests
{
public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
CurrentState = state;
return ValueTask.CompletedTask;

View File

@@ -15,6 +15,7 @@ using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors;
@@ -316,10 +317,10 @@ public sealed class MsrcCsafConnectorTests
{
public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(State);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
State = state;
return ValueTask.CompletedTask;

View File

@@ -330,7 +330,7 @@ public sealed class MsrcCsafConnector : VexConnectorBase
lastError = ex;
LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex);
}
catch (Exception ex)
catch (Exception)
{
response?.Dispose();
throw;
@@ -492,7 +492,7 @@ public sealed class MsrcCsafConnector : VexConnectorBase
return CsafValidationResult.Valid("gzip");
}
using var jsonDocument = JsonDocument.Parse(payload.Span);
using var jsonDocument = JsonDocument.Parse(payload);
return CsafValidationResult.Valid("json");
}
catch (JsonException ex)

View File

@@ -20,6 +20,7 @@ using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers;
using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors;
@@ -254,10 +255,10 @@ public sealed class OracleCsafConnectorTests
{
public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(State);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
State = state;
return ValueTask.CompletedTask;

View File

@@ -13,6 +13,7 @@ using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors;
@@ -258,7 +259,7 @@ public sealed class RedHatCsafConnectorTests
{
public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase))
{
@@ -268,7 +269,7 @@ public sealed class RedHatCsafConnectorTests
return ValueTask.FromResult<VexConnectorState?>(null);
}
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
State = state;
return ValueTask.CompletedTask;

View File

@@ -16,7 +16,7 @@ using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
internal sealed class RancherHubEventClient
public sealed class RancherHubEventClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly RancherHubTokenProvider _tokenProvider;

View File

@@ -3,7 +3,7 @@ using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
internal sealed record RancherHubEventRecord(
public sealed record RancherHubEventRecord(
string RawJson,
string? Id,
string? Type,
@@ -13,7 +13,7 @@ internal sealed record RancherHubEventRecord(
string? DocumentDigest,
string? DocumentFormat);
internal sealed record RancherHubEventBatch(
public sealed record RancherHubEventBatch(
string? Cursor,
string? NextCursor,
ImmutableArray<RancherHubEventRecord> Events,

View File

@@ -8,6 +8,7 @@ using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;

View File

@@ -8,13 +8,13 @@ using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
internal sealed record RancherHubCheckpointState(
public sealed record RancherHubCheckpointState(
string? Cursor,
DateTimeOffset? LastPublishedAt,
DateTimeOffset? EffectiveSince,
ImmutableArray<string> Digests);
internal sealed class RancherHubCheckpointManager
public sealed class RancherHubCheckpointManager
{
private const string CheckpointPrefix = "checkpoint:";
private readonly IVexConnectorStateRepository _repository;

View File

@@ -19,6 +19,7 @@ using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers;
using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
@@ -146,7 +147,7 @@ public sealed class UbuntuCsafConnectorTests
private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp)
{
var indexJson = $$"""
var indexJson = """
{
"generated": "2025-10-18T00:00:00Z",
"channels": [
@@ -159,7 +160,7 @@ public sealed class UbuntuCsafConnectorTests
}
""";
var catalogJson = $$"""
var catalogJson = """
{
"resources": [
{
@@ -274,10 +275,10 @@ public sealed class UbuntuCsafConnectorTests
{
public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
CurrentState = state;
return ValueTask.CompletedTask;

View File

@@ -201,6 +201,8 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
{
var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName);
HttpResponseMessage? response = null;
List<UbuntuCatalogEntry>? entries = null;
try
{
response = await client.GetAsync(channel.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
@@ -219,6 +221,7 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
yield break;
}
entries = new List<UbuntuCatalogEntry>(resourcesElement.GetArrayLength());
foreach (var resource in resourcesElement.EnumerateArray())
{
var type = GetString(resource, "type");
@@ -247,7 +250,7 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
var version = GetString(resource, "version");
var advisoryId = GetString(resource, "id") ?? ExtractAdvisoryId(documentUri, title);
yield return new UbuntuCatalogEntry(
entries.Add(new UbuntuCatalogEntry(
channel.Name,
advisoryId,
documentUri,
@@ -255,7 +258,7 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
etag,
lastModified,
title,
version);
version));
}
}
catch (Exception ex) when (ex is not OperationCanceledException)
@@ -270,6 +273,17 @@ public sealed class UbuntuCsafConnector : VexConnectorBase
{
response?.Dispose();
}
if (entries is null)
{
yield break;
}
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
yield return entry;
}
}
private async Task<DownloadResult?> DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken)

View File

@@ -174,13 +174,13 @@ public sealed class ExportEngineTests
{
public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new();
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
RemoveCalls[(signature.Value, format)] = true;
return ValueTask.CompletedTask;

View File

@@ -53,13 +53,13 @@ public sealed class VexExportCacheServiceTests
public VexExportFormat LastFormat { get; private set; }
public int RemoveCalls { get; private set; }
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
LastSignature = signature;
LastFormat = format;

View File

@@ -1,4 +1,6 @@
using System;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
@@ -88,6 +90,10 @@ public sealed class VexExportEngine : IExportEngine
cached.SourceProviders,
fromCache: true,
cached.ConsensusRevision,
cached.PolicyRevisionId,
cached.PolicyDigest,
cached.ConsensusDigest,
cached.ScoreDigest,
cached.Attestation,
cached.SizeBytes);
}
@@ -100,6 +106,7 @@ public sealed class VexExportEngine : IExportEngine
var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false);
var exporter = ResolveExporter(context.Format);
var policySnapshot = _policyEvaluator.Snapshot;
var exportRequest = new VexExportRequest(
context.Query,
@@ -168,6 +175,9 @@ public sealed class VexExportEngine : IExportEngine
_logger.LogInformation("Attestation generated for export {ExportId}", exportId);
}
var consensusDigestAddress = TryGetContentAddress(result.Metadata, "consensusDigest");
var scoreDigestAddress = TryGetContentAddress(result.Metadata, "scoreDigest");
var manifest = new VexExportManifest(
exportId,
signature,
@@ -177,7 +187,11 @@ public sealed class VexExportEngine : IExportEngine
dataset.Claims.Length,
dataset.SourceProviders,
fromCache: false,
consensusRevision: _policyEvaluator.Version,
consensusRevision: policySnapshot.Version,
policyRevisionId: policySnapshot.RevisionId,
policyDigest: policySnapshot.Digest,
consensusDigest: consensusDigestAddress,
scoreDigest: scoreDigestAddress,
attestation: attestationMetadata,
sizeBytes: result.BytesWritten);
@@ -192,6 +206,27 @@ public sealed class VexExportEngine : IExportEngine
return manifest;
}
private static VexContentAddress? TryGetContentAddress(IReadOnlyDictionary<string, string> metadata, string key)
{
if (metadata is null)
{
return null;
}
if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
{
return null;
}
var parts = value.Split(':', 2, StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return null;
}
return new VexContentAddress(parts[0], parts[1]);
}
private IVexExporter ResolveExporter(VexExportFormat format)
=> _exporters.TryGetValue(format, out var exporter)
? exporter

View File

@@ -857,6 +857,10 @@ public sealed class CsafNormalizer : IVexNormalizer
ImmutableArray<string> UnsupportedJustifications,
ImmutableArray<string> ConflictingJustifications);
private sealed record CsafJustificationInfo(
string RawValue,
VexJustification? Normalized);
private sealed record CsafClaimEntry(
string VulnerabilityId,
CsafProductInfo Product,

View File

@@ -4,6 +4,7 @@ using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Plugin;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
@@ -28,41 +29,43 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public async ValueTask RunAsync(string providerId, CancellationToken cancellationToken)
public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(providerId);
ArgumentNullException.ThrowIfNull(schedule);
ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId);
using var scope = _serviceProvider.CreateScope();
var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
var matched = availablePlugins.FirstOrDefault(plugin =>
string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase));
string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (matched is not null)
{
_logger.LogInformation(
"Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.",
matched.Name,
providerId);
schedule.ProviderId);
}
else
{
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", providerId);
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId);
}
var connectors = scope.ServiceProvider.GetServices<IVexConnector>();
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, providerId, StringComparison.OrdinalIgnoreCase));
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
if (connector is null)
{
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", providerId);
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId);
return;
}
await ExecuteConnectorAsync(scope.ServiceProvider, connector, cancellationToken).ConfigureAwait(false);
await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false);
}
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, CancellationToken cancellationToken)
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken)
{
var effectiveSettings = settings ?? VexConnectorSettings.Empty;
var rawStore = scopeProvider.GetRequiredService<IVexRawStore>();
var claimStore = scopeProvider.GetRequiredService<IVexClaimStore>();
var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>();
@@ -82,11 +85,11 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
await connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false);
await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false);
var context = new VexConnectorContext(
Since: null,
Settings: VexConnectorSettings.Empty,
Settings: effectiveSettings,
RawSink: rawStore,
SignatureVerifier: signatureVerifier,
Normalizers: normalizerRouter,

View File

@@ -1,4 +1,5 @@
using System.Text;
using System.Security.Cryptography;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Hashing;
using StellaOps.Zastava.Core.Serialization;

View File

@@ -105,7 +105,7 @@ public static class ZastavaContractVersions
/// Canonical string representation (schema@vMajor.Minor).
/// </summary>
public override string ToString()
=> $"{Schema}@v{Version.ToString(2, CultureInfo.InvariantCulture)}";
=> $"{Schema}@v{Version.ToString(2)}";
/// <summary>
/// Determines whether a remote contract is compatible with the local definition.

View File

@@ -1,3 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Core.Serialization;
/// <summary>