up
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed record CertBundAdvisoryDto
|
||||
{
|
||||
[JsonPropertyName("advisoryId")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("contentHtml")]
|
||||
public string ContentHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "de";
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("portalUri")]
|
||||
public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/");
|
||||
|
||||
[JsonPropertyName("detailUri")]
|
||||
public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/");
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public IReadOnlyList<CertBundProductDto> Products { get; init; } = Array.Empty<CertBundProductDto>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CertBundReferenceDto> References { get; init; } = Array.Empty<CertBundReferenceDto>();
|
||||
}
|
||||
|
||||
public sealed record CertBundProductDto
|
||||
{
|
||||
[JsonPropertyName("vendor")]
|
||||
public string? Vendor { get; init; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("versions")]
|
||||
public string? Versions { get; init; }
|
||||
}
|
||||
|
||||
public sealed record CertBundReferenceDto
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string Url { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("label")]
|
||||
public string? Label { get; init; }
|
||||
}
|
||||
118
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs
Normal file
118
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs
Normal file
@@ -0,0 +1,118 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal sealed record CertBundCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownAdvisories,
|
||||
DateTimeOffset? LastPublished,
|
||||
DateTimeOffset? LastFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>();
|
||||
|
||||
public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null);
|
||||
|
||||
public CertBundCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = Distinct(documents) };
|
||||
|
||||
public CertBundCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = Distinct(mappings) };
|
||||
|
||||
public CertBundCursor WithKnownAdvisories(IEnumerable<string> advisories)
|
||||
=> this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings };
|
||||
|
||||
public CertBundCursor WithLastPublished(DateTimeOffset? published)
|
||||
=> this with { LastPublished = published };
|
||||
|
||||
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownAdvisories"] = new BsonArray(KnownAdvisories),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (LastFetchAt.HasValue)
|
||||
{
|
||||
document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertBundCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownAdvisories = ReadStringArray(document, "knownAdvisories");
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var publishedValue)
|
||||
? ParseDate(publishedValue)
|
||||
: null;
|
||||
var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue)
|
||||
? ParseDate(fetchValue)
|
||||
: null;
|
||||
|
||||
return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
|
||||
=> values?.Distinct().ToArray() ?? EmptyGuids;
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var id))
|
||||
{
|
||||
items.Add(id);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyStrings;
|
||||
}
|
||||
|
||||
return array.Select(element => element?.ToString() ?? string.Empty)
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed class CertBundDetailParser
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly HtmlContentSanitizer _sanitizer;
|
||||
|
||||
public CertBundDetailParser(HtmlContentSanitizer sanitizer)
|
||||
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
|
||||
|
||||
public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload)
|
||||
{
|
||||
var detail = JsonSerializer.Deserialize<CertBundDetailResponse>(payload, SerializerOptions)
|
||||
?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null.");
|
||||
|
||||
var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name.");
|
||||
var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri);
|
||||
|
||||
return new CertBundAdvisoryDto
|
||||
{
|
||||
AdvisoryId = advisoryId,
|
||||
Title = detail.Title ?? advisoryId,
|
||||
Summary = detail.Summary,
|
||||
ContentHtml = contentHtml,
|
||||
Severity = detail.Severity,
|
||||
Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!,
|
||||
Published = detail.Published,
|
||||
Modified = detail.Updated ?? detail.Published,
|
||||
PortalUri = portalUri,
|
||||
DetailUri = detailUri,
|
||||
CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id!.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? Array.Empty<string>(),
|
||||
References = MapReferences(detail.References),
|
||||
Products = MapProducts(detail.Products),
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertBundReferenceDto> MapReferences(CertBundDetailReference[]? references)
|
||||
{
|
||||
if (references is null || references.Length == 0)
|
||||
{
|
||||
return Array.Empty<CertBundReferenceDto>();
|
||||
}
|
||||
|
||||
return references
|
||||
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
|
||||
.Select(reference => new CertBundReferenceDto
|
||||
{
|
||||
Url = reference.Url!,
|
||||
Label = reference.Label,
|
||||
})
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertBundProductDto> MapProducts(CertBundDetailProduct[]? products)
|
||||
{
|
||||
if (products is null || products.Length == 0)
|
||||
{
|
||||
return Array.Empty<CertBundProductDto>();
|
||||
}
|
||||
|
||||
return products
|
||||
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
|
||||
.Select(product => new CertBundProductDto
|
||||
{
|
||||
Vendor = product.Vendor,
|
||||
Name = product.Name,
|
||||
Versions = product.Versions,
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal sealed record CertBundDetailResponse
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("updated")]
|
||||
public DateTimeOffset? Updated { get; init; }
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public string[]? CveIds { get; init; }
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public CertBundDetailReference[]? References { get; init; }
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public CertBundDetailProduct[]? Products { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CertBundDetailReference
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("label")]
|
||||
public string? Label { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CertBundDetailProduct
|
||||
{
|
||||
[JsonPropertyName("vendor")]
|
||||
public string? Vendor { get; init; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("versions")]
|
||||
public string? Versions { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector.
|
||||
/// </summary>
|
||||
public sealed class CertBundDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Feedser.Source.CertBund";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _feedFetchAttempts;
|
||||
private readonly Counter<long> _feedFetchSuccess;
|
||||
private readonly Counter<long> _feedFetchFailures;
|
||||
private readonly Histogram<long> _feedItemCount;
|
||||
private readonly Histogram<long> _feedEnqueuedCount;
|
||||
private readonly Histogram<double> _feedCoverageDays;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchNotModified;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseProductCount;
|
||||
private readonly Histogram<long> _parseCveCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapPackageCount;
|
||||
private readonly Histogram<long> _mapAliasCount;
|
||||
|
||||
public CertBundDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_feedFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of RSS feed load attempts.");
|
||||
_feedFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of successful RSS feed loads.");
|
||||
_feedFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of RSS feed load failures.");
|
||||
_feedItemCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.feed.items.count",
|
||||
unit: "items",
|
||||
description: "Distribution of RSS item counts per fetch.");
|
||||
_feedEnqueuedCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.feed.enqueued.count",
|
||||
unit: "documents",
|
||||
description: "Distribution of advisory documents enqueued per fetch.");
|
||||
_feedCoverageDays = _meter.CreateHistogram<double>(
|
||||
name: "certbund.feed.coverage.days",
|
||||
unit: "days",
|
||||
description: "Coverage window in days between fetch time and the oldest published advisory in the feed.");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetch attempts.");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches that persisted a document.");
|
||||
_detailFetchNotModified = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches returning HTTP 304.");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches that failed.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of documents parsed into CERT-Bund DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of documents that failed to parse.");
|
||||
_parseProductCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.parse.products.count",
|
||||
unit: "products",
|
||||
description: "Distribution of product entries captured per advisory.");
|
||||
_parseCveCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.parse.cve.count",
|
||||
unit: "aliases",
|
||||
description: "Distribution of CVE identifiers captured per advisory.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of mapping failures.");
|
||||
_mapPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages emitted per advisory.");
|
||||
_mapAliasCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.map.aliases.count",
|
||||
unit: "aliases",
|
||||
description: "Distribution of alias counts per advisory.");
|
||||
}
|
||||
|
||||
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
|
||||
|
||||
public void FeedFetchSuccess(int itemCount)
|
||||
{
|
||||
_feedFetchSuccess.Add(1);
|
||||
if (itemCount >= 0)
|
||||
{
|
||||
_feedItemCount.Record(itemCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void FeedFetchFailure(string reason = "error")
|
||||
=> _feedFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void RecordFeedCoverage(double? coverageDays)
|
||||
{
|
||||
if (coverageDays is { } days && days >= 0)
|
||||
{
|
||||
_feedCoverageDays.Record(days);
|
||||
}
|
||||
}
|
||||
|
||||
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
|
||||
|
||||
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
|
||||
|
||||
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
|
||||
|
||||
public void DetailFetchFailure(string reason = "error")
|
||||
=> _detailFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void DetailFetchEnqueued(int count)
|
||||
{
|
||||
if (count >= 0)
|
||||
{
|
||||
_feedEnqueuedCount.Record(count);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseSuccess(int productCount, int cveCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
|
||||
if (productCount >= 0)
|
||||
{
|
||||
_parseProductCount.Record(productCount);
|
||||
}
|
||||
|
||||
if (cveCount >= 0)
|
||||
{
|
||||
_parseCveCount.Record(cveCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure(string reason = "error")
|
||||
=> _parseFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void MapSuccess(int affectedPackages, int aliasCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
|
||||
if (affectedPackages >= 0)
|
||||
{
|
||||
_mapPackageCount.Record(affectedPackages);
|
||||
}
|
||||
|
||||
if (aliasCount >= 0)
|
||||
{
|
||||
_mapAliasCount.Record(aliasCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure(string reason = "error")
|
||||
=> _mapFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
private static KeyValuePair<string, object?> ReasonTag(string reason)
|
||||
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal static class CertBundDocumentMetadata
|
||||
{
|
||||
public static Dictionary<string, string> CreateMetadata(CertBundFeedItem item)
|
||||
{
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["certbund.advisoryId"] = item.AdvisoryId,
|
||||
["certbund.portalUri"] = item.PortalUri.ToString(),
|
||||
["certbund.published"] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Category))
|
||||
{
|
||||
metadata["certbund.category"] = item.Category!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Title))
|
||||
{
|
||||
metadata["certbund.title"] = item.Title!;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed class CertBundFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertBundOptions _options;
|
||||
private readonly ILogger<CertBundFeedClient> _logger;
|
||||
private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1);
|
||||
private volatile bool _bootstrapped;
|
||||
|
||||
public CertBundFeedClient(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<CertBundOptions> options,
|
||||
ILogger<CertBundFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertBundFeedItem>> LoadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName);
|
||||
await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri);
|
||||
request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8");
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = XDocument.Load(stream);
|
||||
|
||||
var items = new List<CertBundFeedItem>();
|
||||
foreach (var element in document.Descendants("item"))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var linkValue = element.Element("link")?.Value?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisoryId = TryExtractNameParameter(portalUri);
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var detailUri = _options.BuildDetailUri(advisoryId);
|
||||
var pubDateText = element.Element("pubDate")?.Value;
|
||||
var published = ParseDate(pubDateText);
|
||||
var title = element.Element("title")?.Value?.Trim();
|
||||
var category = element.Element("category")?.Value?.Trim();
|
||||
|
||||
items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_bootstrapped)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_bootstrapped)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri);
|
||||
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
_bootstrapped = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_bootstrapSemaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? TryExtractNameParameter(Uri portalUri)
|
||||
{
|
||||
if (portalUri is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var query = portalUri.Query;
|
||||
if (string.IsNullOrEmpty(query))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = query.TrimStart('?');
|
||||
foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separatorIndex = pair.IndexOf('=');
|
||||
if (separatorIndex <= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = pair[..separatorIndex].Trim();
|
||||
if (!key.Equals("name", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = pair[(separatorIndex + 1)..];
|
||||
return Uri.UnescapeDataString(value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset ParseDate(string? value)
|
||||
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
|
||||
? parsed
|
||||
: DateTimeOffset.UtcNow;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
using System;
|
||||
|
||||
public sealed record CertBundFeedItem(
|
||||
string AdvisoryId,
|
||||
Uri DetailUri,
|
||||
Uri PortalUri,
|
||||
DateTimeOffset Published,
|
||||
string? Title,
|
||||
string? Category);
|
||||
168
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs
Normal file
168
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs
Normal file
@@ -0,0 +1,168 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal static class CertBundMapper
|
||||
{
|
||||
public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AdvisoryId,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.AdvisoryId,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language?.ToLowerInvariant() ?? "de",
|
||||
published: dto.Published,
|
||||
modified: dto.Modified,
|
||||
severity: MapSeverity(dto.Severity),
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { provenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(CertBundAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string>(capacity: 4) { dto.AdvisoryId };
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
dto.DetailUri.ToString(),
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
kind: "reference",
|
||||
sourceTag: "cert-bund",
|
||||
summary: reference.Label,
|
||||
provenance: new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
foreach (var product in dto.Products)
|
||||
{
|
||||
var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified";
|
||||
var name = Validation.TrimToNull(product.Name);
|
||||
var identifier = name is null ? vendor : $"{vendor} {name}";
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var ranges = string.IsNullOrWhiteSpace(product.Versions)
|
||||
? Array.Empty<AffectedVersionRange>()
|
||||
: new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: "string",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.Versions,
|
||||
provenance: new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"package-range",
|
||||
product.Versions,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges }))
|
||||
};
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
|
||||
}
|
||||
|
||||
return packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? MapSeverity(string? severity)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(severity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return severity.ToLowerInvariant() switch
|
||||
{
|
||||
"hoch" or "high" => "high",
|
||||
"mittel" or "medium" => "medium",
|
||||
"gering" or "low" => "low",
|
||||
_ => severity.ToLowerInvariant(),
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user