Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,187 @@
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal sealed record CertCcCursor(
|
||||
TimeWindowCursorState SummaryState,
|
||||
IReadOnlyCollection<Guid> PendingSummaries,
|
||||
IReadOnlyCollection<string> PendingNotes,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
DateTimeOffset? LastRun)
|
||||
{
|
||||
private static readonly Guid[] EmptyGuidArray = Array.Empty<Guid>();
|
||||
private static readonly string[] EmptyStringArray = Array.Empty<string>();
|
||||
|
||||
public static CertCcCursor Empty { get; } = new(
|
||||
TimeWindowCursorState.Empty,
|
||||
EmptyGuidArray,
|
||||
EmptyStringArray,
|
||||
EmptyGuidArray,
|
||||
EmptyGuidArray,
|
||||
null);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument();
|
||||
|
||||
var summary = new BsonDocument();
|
||||
SummaryState.WriteTo(summary, "start", "end");
|
||||
document["summary"] = summary;
|
||||
|
||||
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
|
||||
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
|
||||
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
|
||||
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
|
||||
|
||||
if (LastRun.HasValue)
|
||||
{
|
||||
document["lastRun"] = LastRun.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertCcCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
|
||||
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
|
||||
{
|
||||
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
|
||||
}
|
||||
|
||||
var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
|
||||
var pendingNotes = ReadStringArray(document, "pendingNotes");
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
DateTimeOffset? lastRun = null;
|
||||
if (document.TryGetValue("lastRun", out var lastRunValue))
|
||||
{
|
||||
lastRun = lastRunValue.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun);
|
||||
}
|
||||
|
||||
public CertCcCursor WithSummaryState(TimeWindowCursorState state)
|
||||
=> this with { SummaryState = state ?? TimeWindowCursorState.Empty };
|
||||
|
||||
public CertCcCursor WithPendingSummaries(IEnumerable<Guid>? ids)
|
||||
=> this with { PendingSummaries = NormalizeGuidSet(ids) };
|
||||
|
||||
public CertCcCursor WithPendingNotes(IEnumerable<string>? notes)
|
||||
=> this with { PendingNotes = NormalizeStringSet(notes) };
|
||||
|
||||
public CertCcCursor WithPendingDocuments(IEnumerable<Guid>? ids)
|
||||
=> this with { PendingDocuments = NormalizeGuidSet(ids) };
|
||||
|
||||
public CertCcCursor WithPendingMappings(IEnumerable<Guid>? ids)
|
||||
=> this with { PendingMappings = NormalizeGuidSet(ids) };
|
||||
|
||||
public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
|
||||
=> this with { LastRun = timestamp };
|
||||
|
||||
private static Guid[] ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyGuidArray;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (TryReadGuid(element, out var parsed))
|
||||
{
|
||||
results.Add(parsed);
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
|
||||
}
|
||||
|
||||
private static string[] ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyStringArray;
|
||||
}
|
||||
|
||||
var results = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
switch (element)
|
||||
{
|
||||
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
|
||||
results.Add(bsonString.AsString.Trim());
|
||||
break;
|
||||
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
|
||||
results.Add(inner.AsString.Trim());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0
|
||||
? EmptyStringArray
|
||||
: results
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool TryReadGuid(BsonValue value, out Guid guid)
|
||||
{
|
||||
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (value is BsonBinaryData binary)
|
||||
{
|
||||
try
|
||||
{
|
||||
guid = binary.ToGuid();
|
||||
return true;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
// ignore and fall back to byte array parsing
|
||||
}
|
||||
|
||||
var bytes = binary.AsByteArray;
|
||||
if (bytes.Length == 16)
|
||||
{
|
||||
guid = new Guid(bytes);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
guid = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static Guid[] NormalizeGuidSet(IEnumerable<Guid>? ids)
|
||||
=> ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray;
|
||||
|
||||
private static string[] NormalizeStringSet(IEnumerable<string>? values)
|
||||
=> values is null
|
||||
? EmptyStringArray
|
||||
: values
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
|
||||
/// </summary>
|
||||
public sealed class CertCcDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _planWindows;
|
||||
private readonly Counter<long> _planRequests;
|
||||
private readonly Histogram<double> _planWindowDays;
|
||||
private readonly Counter<long> _summaryFetchAttempts;
|
||||
private readonly Counter<long> _summaryFetchSuccess;
|
||||
private readonly Counter<long> _summaryFetchUnchanged;
|
||||
private readonly Counter<long> _summaryFetchFailures;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchUnchanged;
|
||||
private readonly Counter<long> _detailFetchMissing;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseVendorCount;
|
||||
private readonly Histogram<long> _parseStatusCount;
|
||||
private readonly Histogram<long> _parseVulnerabilityCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapAffectedPackageCount;
|
||||
private readonly Histogram<long> _mapNormalizedVersionCount;
|
||||
|
||||
public CertCcDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_planWindows = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.windows",
|
||||
unit: "windows",
|
||||
description: "Number of summary planning windows evaluated.");
|
||||
_planRequests = _meter.CreateCounter<long>(
|
||||
name: "certcc.plan.requests",
|
||||
unit: "requests",
|
||||
description: "Total CERT/CC summary endpoints queued by the planner.");
|
||||
_planWindowDays = _meter.CreateHistogram<double>(
|
||||
name: "certcc.plan.window_days",
|
||||
unit: "day",
|
||||
description: "Duration of each planning window in days.");
|
||||
_summaryFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetch attempts.");
|
||||
_summaryFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches persisted to storage.");
|
||||
_summaryFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches returning HTTP 304.");
|
||||
_summaryFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.summary.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE summary fetches that failed after retries.");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetch attempts.");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that returned payloads.");
|
||||
_detailFetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.unchanged",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches returning HTTP 304.");
|
||||
_detailFetchMissing = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.missing",
|
||||
unit: "operations",
|
||||
description: "Number of optional VINCE detail endpoints missing but tolerated.");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.detail.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of VINCE detail fetches that failed after retries.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles parsed into DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of VINCE note bundles that failed to parse.");
|
||||
_parseVendorCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vendors.count",
|
||||
unit: "vendors",
|
||||
description: "Distribution of vendor statements per VINCE note.");
|
||||
_parseStatusCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.statuses.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vendor status entries per VINCE note.");
|
||||
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.parse.vulnerabilities.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vulnerability records per VINCE note.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "certcc.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of CERT/CC advisory mapping attempts that failed.");
|
||||
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages emitted per CERT/CC advisory.");
|
||||
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
|
||||
name: "certcc.map.normalized_versions.count",
|
||||
unit: "rules",
|
||||
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
|
||||
}
|
||||
|
||||
public void PlanEvaluated(TimeWindow window, int requestCount)
|
||||
{
|
||||
_planWindows.Add(1);
|
||||
|
||||
if (requestCount > 0)
|
||||
{
|
||||
_planRequests.Add(requestCount);
|
||||
}
|
||||
|
||||
var duration = window.Duration;
|
||||
if (duration > TimeSpan.Zero)
|
||||
{
|
||||
_planWindowDays.Record(duration.TotalDays);
|
||||
}
|
||||
}
|
||||
|
||||
public void SummaryFetchAttempt(CertCcSummaryScope scope)
|
||||
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchSuccess(CertCcSummaryScope scope)
|
||||
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
|
||||
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
|
||||
|
||||
public void SummaryFetchFailure(CertCcSummaryScope scope)
|
||||
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
|
||||
|
||||
public void DetailFetchAttempt(string endpoint)
|
||||
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchSuccess(string endpoint)
|
||||
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchUnchanged(string endpoint)
|
||||
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchMissing(string endpoint)
|
||||
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void DetailFetchFailure(string endpoint)
|
||||
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
|
||||
|
||||
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
if (vendorCount >= 0)
|
||||
{
|
||||
_parseVendorCount.Record(vendorCount);
|
||||
}
|
||||
if (statusCount >= 0)
|
||||
{
|
||||
_parseStatusCount.Record(statusCount);
|
||||
}
|
||||
if (vulnerabilityCount >= 0)
|
||||
{
|
||||
_parseVulnerabilityCount.Record(vulnerabilityCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure()
|
||||
=> _parseFailures.Add(1);
|
||||
|
||||
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
if (affectedPackageCount >= 0)
|
||||
{
|
||||
_mapAffectedPackageCount.Record(affectedPackageCount);
|
||||
}
|
||||
if (normalizedVersionCount >= 0)
|
||||
{
|
||||
_mapNormalizedVersionCount.Record(normalizedVersionCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure()
|
||||
=> _mapFailures.Add(1);
|
||||
|
||||
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
|
||||
=> new("scope", scope.ToString().ToLowerInvariant());
|
||||
|
||||
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
|
||||
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcMapper
|
||||
{
|
||||
private const string AdvisoryPrefix = "certcc";
|
||||
private const string VendorNormalizedVersionScheme = "certcc.vendor";
|
||||
|
||||
public static Advisory Map(
|
||||
CertCcNoteDto dto,
|
||||
DocumentRecord document,
|
||||
DtoRecord dtoRecord,
|
||||
string sourceName)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
ArgumentNullException.ThrowIfNull(dtoRecord);
|
||||
ArgumentException.ThrowIfNullOrEmpty(sourceName);
|
||||
|
||||
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
|
||||
var fetchedAt = document.FetchedAt.ToUniversalTime();
|
||||
|
||||
var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty;
|
||||
|
||||
var advisoryKey = BuildAdvisoryKey(metadata);
|
||||
var title = string.IsNullOrWhiteSpace(metadata.Title) ? advisoryKey : metadata.Title.Trim();
|
||||
var summary = ExtractSummary(metadata);
|
||||
|
||||
var aliases = BuildAliases(dto).ToArray();
|
||||
var references = BuildReferences(dto, metadata, sourceName, recordedAt).ToArray();
|
||||
var affectedPackages = BuildAffectedPackages(dto, metadata, sourceName, recordedAt).ToArray();
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(sourceName, "document", document.Uri, fetchedAt),
|
||||
new AdvisoryProvenance(sourceName, "map", metadata.VuId ?? metadata.IdNumber ?? advisoryKey, recordedAt),
|
||||
};
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey,
|
||||
title,
|
||||
summary,
|
||||
language: "en",
|
||||
metadata.Published?.ToUniversalTime(),
|
||||
metadata.Updated?.ToUniversalTime(),
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
references,
|
||||
affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance);
|
||||
}
|
||||
|
||||
private static string BuildAdvisoryKey(CertCcNoteMetadata metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return $"{AdvisoryPrefix}/{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
var vuKey = NormalizeVuId(metadata.VuId);
|
||||
if (vuKey.Length > 0)
|
||||
{
|
||||
return $"{AdvisoryPrefix}/{vuKey}";
|
||||
}
|
||||
|
||||
var id = SanitizeToken(metadata.IdNumber);
|
||||
if (id.Length > 0)
|
||||
{
|
||||
return $"{AdvisoryPrefix}/vu-{id}";
|
||||
}
|
||||
|
||||
return $"{AdvisoryPrefix}/{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
private static string NormalizeVuId(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var digits = new string(value.Where(char.IsDigit).ToArray());
|
||||
if (digits.Length > 0)
|
||||
{
|
||||
return $"vu-{digits}";
|
||||
}
|
||||
|
||||
var sanitized = value.Trim().ToLowerInvariant();
|
||||
sanitized = sanitized.Replace("vu#", "vu-", StringComparison.OrdinalIgnoreCase);
|
||||
sanitized = sanitized.Replace('#', '-');
|
||||
sanitized = sanitized.Replace(' ', '-');
|
||||
|
||||
return SanitizeToken(sanitized);
|
||||
}
|
||||
|
||||
private static string SanitizeToken(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
var filtered = new string(trimmed
|
||||
.Select(ch => char.IsLetterOrDigit(ch) || ch is '-' or '_' ? ch : '-')
|
||||
.ToArray());
|
||||
|
||||
return filtered.Trim('-').ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
private static readonly Regex WhitespaceRegex = new("[ \t\f\r]+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
private static readonly Regex ParagraphRegex = new("<\\s*/?\\s*p[^>]*>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static string? ExtractSummary(CertCcNoteMetadata metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var summary = string.IsNullOrWhiteSpace(metadata.Summary) ? metadata.Overview : metadata.Summary;
|
||||
if (string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return HtmlToPlainText(summary);
|
||||
}
|
||||
|
||||
private static string HtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var normalized = html
|
||||
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("<li>", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("</li>", "\n", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
normalized = ParagraphRegex.Replace(normalized, "\n");
|
||||
|
||||
var withoutTags = HtmlTagRegex.Replace(normalized, " ");
|
||||
var decoded = WebUtility.HtmlDecode(withoutTags) ?? string.Empty;
|
||||
var collapsedSpaces = WhitespaceRegex.Replace(decoded, " ");
|
||||
var collapsedNewlines = Regex.Replace(collapsedSpaces, "\n{2,}", "\n", RegexOptions.Compiled);
|
||||
return collapsedNewlines.Trim();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> BuildAliases(CertCcNoteDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metadata.VuId))
|
||||
{
|
||||
aliases.Add(metadata.VuId.Trim());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metadata.IdNumber))
|
||||
{
|
||||
aliases.Add($"VU#{metadata.IdNumber.Trim()}");
|
||||
}
|
||||
|
||||
foreach (var cve in metadata.CveIds ?? Array.Empty<string>())
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
aliases.Add(cve.Trim());
|
||||
}
|
||||
|
||||
foreach (var vulnerability in dto.Vulnerabilities ?? Array.Empty<CertCcVulnerabilityDto>())
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vulnerability.CveId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
aliases.Add(vulnerability.CveId.Trim());
|
||||
}
|
||||
|
||||
return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static IEnumerable<AdvisoryReference> BuildReferences(
|
||||
CertCcNoteDto dto,
|
||||
CertCcNoteMetadata metadata,
|
||||
string sourceName,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var canonicalUri = !string.IsNullOrWhiteSpace(metadata.PrimaryUrl)
|
||||
? metadata.PrimaryUrl!
|
||||
: (string.IsNullOrWhiteSpace(metadata.IdNumber)
|
||||
? "https://www.kb.cert.org/vuls/"
|
||||
: $"https://www.kb.cert.org/vuls/id/{metadata.IdNumber.Trim()}/");
|
||||
|
||||
var provenance = new AdvisoryProvenance(sourceName, "reference", canonicalUri, recordedAt);
|
||||
|
||||
TryAddReference(references, canonicalUri, "advisory", "certcc.note", null, provenance);
|
||||
|
||||
foreach (var url in metadata.PublicUrls ?? Array.Empty<string>())
|
||||
{
|
||||
TryAddReference(references, url, "reference", "certcc.public", null, provenance);
|
||||
}
|
||||
|
||||
foreach (var vendor in dto.Vendors ?? Array.Empty<CertCcVendorDto>())
|
||||
{
|
||||
foreach (var url in vendor.References ?? Array.Empty<string>())
|
||||
{
|
||||
TryAddReference(references, url, "reference", "certcc.vendor", vendor.Vendor, provenance);
|
||||
}
|
||||
|
||||
var statementText = vendor.Statement ?? string.Empty;
|
||||
var patches = CertCcVendorStatementParser.Parse(statementText);
|
||||
foreach (var patch in patches)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(patch.RawLine) && TryFindEmbeddedUrl(patch.RawLine!, out var rawUrl))
|
||||
{
|
||||
TryAddReference(references, rawUrl, "reference", "certcc.vendor.statement", vendor.Vendor, provenance);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var status in dto.VendorStatuses ?? Array.Empty<CertCcVendorStatusDto>())
|
||||
{
|
||||
foreach (var url in status.References ?? Array.Empty<string>())
|
||||
{
|
||||
TryAddReference(references, url, "reference", "certcc.vendor.status", status.Vendor, provenance);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(status.Statement) && TryFindEmbeddedUrl(status.Statement!, out var embedded))
|
||||
{
|
||||
TryAddReference(references, embedded, "reference", "certcc.vendor.status", status.Vendor, provenance);
|
||||
}
|
||||
}
|
||||
|
||||
return references
|
||||
.GroupBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(static group => group
|
||||
.OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal)
|
||||
.ThenBy(static reference => reference.SourceTag ?? string.Empty, StringComparer.Ordinal)
|
||||
.ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.First())
|
||||
.OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal)
|
||||
.ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static void TryAddReference(
|
||||
ICollection<AdvisoryReference> references,
|
||||
string? url,
|
||||
string kind,
|
||||
string? sourceTag,
|
||||
string? summary,
|
||||
AdvisoryProvenance provenance)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var candidate = url.Trim();
|
||||
if (!Uri.TryCreate(candidate, UriKind.Absolute, out var parsed))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var normalized = parsed.ToString();
|
||||
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(normalized, kind, sourceTag, summary, provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore invalid references
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryFindEmbeddedUrl(string text, out string? url)
|
||||
{
|
||||
url = null;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var tokens = text.Split(new[] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
var trimmed = token.Trim().TrimEnd('.', ',', ')', ';', ']', '}');
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var parsed))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
url = parsed.ToString();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static IEnumerable<AffectedPackage> BuildAffectedPackages(
|
||||
CertCcNoteDto dto,
|
||||
CertCcNoteMetadata metadata,
|
||||
string sourceName,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
var vendors = dto.Vendors ?? Array.Empty<CertCcVendorDto>();
|
||||
var statuses = dto.VendorStatuses ?? Array.Empty<CertCcVendorStatusDto>();
|
||||
|
||||
if (vendors.Count == 0 && statuses.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var statusLookup = statuses
|
||||
.GroupBy(static status => NormalizeVendorKey(status.Vendor))
|
||||
.ToDictionary(static group => group.Key, static group => group.ToArray(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var packages = new List<AffectedPackage>();
|
||||
|
||||
foreach (var vendor in vendors.OrderBy(static v => v.Vendor, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var key = NormalizeVendorKey(vendor.Vendor);
|
||||
var vendorStatuses = statusLookup.TryGetValue(key, out var value)
|
||||
? value
|
||||
: Array.Empty<CertCcVendorStatusDto>();
|
||||
|
||||
if (BuildVendorPackage(vendor, vendorStatuses, sourceName, recordedAt) is { } package)
|
||||
{
|
||||
packages.Add(package);
|
||||
}
|
||||
|
||||
statusLookup.Remove(key);
|
||||
}
|
||||
|
||||
foreach (var remaining in statusLookup.Values)
|
||||
{
|
||||
if (remaining.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vendorName = remaining[0].Vendor;
|
||||
var fallbackVendor = new CertCcVendorDto(
|
||||
vendorName,
|
||||
ContactDate: null,
|
||||
StatementDate: null,
|
||||
Updated: remaining
|
||||
.Select(static status => status.DateUpdated)
|
||||
.Where(static update => update.HasValue)
|
||||
.OrderByDescending(static update => update)
|
||||
.FirstOrDefault(),
|
||||
Statement: remaining
|
||||
.Select(static status => status.Statement)
|
||||
.FirstOrDefault(static statement => !string.IsNullOrWhiteSpace(statement)),
|
||||
Addendum: null,
|
||||
References: remaining
|
||||
.SelectMany(static status => status.References ?? Array.Empty<string>())
|
||||
.ToArray());
|
||||
|
||||
if (BuildVendorPackage(fallbackVendor, remaining, sourceName, recordedAt) is { } package)
|
||||
{
|
||||
packages.Add(package);
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static AffectedPackage? BuildVendorPackage(
|
||||
CertCcVendorDto vendor,
|
||||
IReadOnlyList<CertCcVendorStatusDto> statuses,
|
||||
string sourceName,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
var vendorName = string.IsNullOrWhiteSpace(vendor.Vendor)
|
||||
? (statuses.FirstOrDefault()?.Vendor?.Trim() ?? string.Empty)
|
||||
: vendor.Vendor.Trim();
|
||||
|
||||
if (vendorName.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var packageProvenance = new AdvisoryProvenance(sourceName, "vendor", vendorName, recordedAt);
|
||||
var rangeProvenance = new AdvisoryProvenance(sourceName, "vendor-range", vendorName, recordedAt);
|
||||
|
||||
var patches = CertCcVendorStatementParser.Parse(vendor.Statement ?? string.Empty);
|
||||
var normalizedVersions = BuildNormalizedVersions(vendorName, patches);
|
||||
var vendorStatuses = BuildStatuses(vendorName, statuses, sourceName, recordedAt);
|
||||
var primitives = BuildRangePrimitives(vendor, vendorStatuses, patches);
|
||||
|
||||
var range = new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: null,
|
||||
provenance: rangeProvenance,
|
||||
primitives: primitives);
|
||||
|
||||
return new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
vendorName,
|
||||
platform: null,
|
||||
versionRanges: new[] { range },
|
||||
normalizedVersions: normalizedVersions,
|
||||
statuses: vendorStatuses,
|
||||
provenance: new[] { packageProvenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
string vendorName,
|
||||
IReadOnlyList<CertCcVendorPatch> patches)
|
||||
{
|
||||
if (patches.Count == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var rules = new List<NormalizedVersionRule>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var patch in patches)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(patch.Version))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var version = patch.Version.Trim();
|
||||
if (!seen.Add($"{patch.Product}|{version}"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var notes = string.IsNullOrWhiteSpace(patch.Product)
|
||||
? vendorName
|
||||
: $"{vendorName}::{patch.Product.Trim()}";
|
||||
|
||||
rules.Add(new NormalizedVersionRule(
|
||||
VendorNormalizedVersionScheme,
|
||||
NormalizedVersionRuleTypes.Exact,
|
||||
value: version,
|
||||
notes: notes));
|
||||
}
|
||||
|
||||
return rules.Count == 0 ? Array.Empty<NormalizedVersionRule>() : rules;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(
|
||||
string vendorName,
|
||||
IReadOnlyList<CertCcVendorStatusDto> statuses,
|
||||
string sourceName,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
if (statuses.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackageStatus>();
|
||||
}
|
||||
|
||||
var result = new List<AffectedPackageStatus>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var status in statuses)
|
||||
{
|
||||
if (!AffectedPackageStatusCatalog.TryNormalize(status.Status, out var normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var cve = status.CveId?.Trim() ?? string.Empty;
|
||||
var key = string.IsNullOrWhiteSpace(cve)
|
||||
? normalized
|
||||
: $"{normalized}|{cve}";
|
||||
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"vendor-status",
|
||||
string.IsNullOrWhiteSpace(cve) ? vendorName : $"{vendorName}:{cve}",
|
||||
recordedAt);
|
||||
|
||||
result.Add(new AffectedPackageStatus(normalized, provenance));
|
||||
}
|
||||
|
||||
return result
|
||||
.OrderBy(static status => status.Status, StringComparer.Ordinal)
|
||||
.ThenBy(static status => status.Provenance.Value ?? string.Empty, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static RangePrimitives? BuildRangePrimitives(
|
||||
CertCcVendorDto vendor,
|
||||
IReadOnlyList<AffectedPackageStatus> statuses,
|
||||
IReadOnlyList<CertCcVendorPatch> patches)
|
||||
{
|
||||
var extensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
AddVendorExtension(extensions, "certcc.vendor.name", vendor.Vendor);
|
||||
AddVendorExtension(extensions, "certcc.vendor.statement.raw", HtmlToPlainText(vendor.Statement ?? string.Empty), 2048);
|
||||
AddVendorExtension(extensions, "certcc.vendor.addendum", HtmlToPlainText(vendor.Addendum ?? string.Empty), 1024);
|
||||
AddVendorExtension(extensions, "certcc.vendor.contactDate", FormatDate(vendor.ContactDate));
|
||||
AddVendorExtension(extensions, "certcc.vendor.statementDate", FormatDate(vendor.StatementDate));
|
||||
AddVendorExtension(extensions, "certcc.vendor.updated", FormatDate(vendor.Updated));
|
||||
|
||||
if (vendor.References is { Count: > 0 })
|
||||
{
|
||||
AddVendorExtension(extensions, "certcc.vendor.references", string.Join(" ", vendor.References));
|
||||
}
|
||||
|
||||
if (statuses.Count > 0)
|
||||
{
|
||||
var serialized = string.Join(";", statuses
|
||||
.Select(static status => status.Provenance.Value is { Length: > 0 }
|
||||
? $"{status.Provenance.Value.Split(':').Last()}={status.Status}"
|
||||
: status.Status));
|
||||
|
||||
AddVendorExtension(extensions, "certcc.vendor.statuses", serialized);
|
||||
}
|
||||
|
||||
if (patches.Count > 0)
|
||||
{
|
||||
var serialized = string.Join(";", patches.Select(static patch =>
|
||||
{
|
||||
var product = string.IsNullOrWhiteSpace(patch.Product) ? "unknown" : patch.Product.Trim();
|
||||
return $"{product}={patch.Version.Trim()}";
|
||||
}));
|
||||
|
||||
AddVendorExtension(extensions, "certcc.vendor.patches", serialized, 2048);
|
||||
}
|
||||
|
||||
return extensions.Count == 0
|
||||
? null
|
||||
: new RangePrimitives(null, null, null, extensions);
|
||||
}
|
||||
|
||||
private static void AddVendorExtension(IDictionary<string, string> extensions, string key, string? value, int maxLength = 512)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (trimmed.Length > maxLength)
|
||||
{
|
||||
trimmed = trimmed[..maxLength].Trim();
|
||||
}
|
||||
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
extensions[key] = trimmed;
|
||||
}
|
||||
|
||||
private static string? FormatDate(DateTimeOffset? value)
|
||||
=> value?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
|
||||
private static string NormalizeVendorKey(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal sealed record CertCcNoteDto(
|
||||
CertCcNoteMetadata Metadata,
|
||||
IReadOnlyList<CertCcVendorDto> Vendors,
|
||||
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
|
||||
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
|
||||
{
|
||||
public static CertCcNoteDto Empty { get; } = new(
|
||||
CertCcNoteMetadata.Empty,
|
||||
Array.Empty<CertCcVendorDto>(),
|
||||
Array.Empty<CertCcVendorStatusDto>(),
|
||||
Array.Empty<CertCcVulnerabilityDto>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcNoteMetadata(
|
||||
string? VuId,
|
||||
string IdNumber,
|
||||
string Title,
|
||||
string? Overview,
|
||||
string? Summary,
|
||||
DateTimeOffset? Published,
|
||||
DateTimeOffset? Updated,
|
||||
DateTimeOffset? Created,
|
||||
int? Revision,
|
||||
IReadOnlyList<string> CveIds,
|
||||
IReadOnlyList<string> PublicUrls,
|
||||
string? PrimaryUrl)
|
||||
{
|
||||
public static CertCcNoteMetadata Empty { get; } = new(
|
||||
VuId: null,
|
||||
IdNumber: string.Empty,
|
||||
Title: string.Empty,
|
||||
Overview: null,
|
||||
Summary: null,
|
||||
Published: null,
|
||||
Updated: null,
|
||||
Created: null,
|
||||
Revision: null,
|
||||
CveIds: Array.Empty<string>(),
|
||||
PublicUrls: Array.Empty<string>(),
|
||||
PrimaryUrl: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorDto(
|
||||
string Vendor,
|
||||
DateTimeOffset? ContactDate,
|
||||
DateTimeOffset? StatementDate,
|
||||
DateTimeOffset? Updated,
|
||||
string? Statement,
|
||||
string? Addendum,
|
||||
IReadOnlyList<string> References)
|
||||
{
|
||||
public static CertCcVendorDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
ContactDate: null,
|
||||
StatementDate: null,
|
||||
Updated: null,
|
||||
Statement: null,
|
||||
Addendum: null,
|
||||
References: Array.Empty<string>());
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorStatusDto(
|
||||
string Vendor,
|
||||
string CveId,
|
||||
string Status,
|
||||
string? Statement,
|
||||
IReadOnlyList<string> References,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVendorStatusDto Empty { get; } = new(
|
||||
Vendor: string.Empty,
|
||||
CveId: string.Empty,
|
||||
Status: string.Empty,
|
||||
Statement: null,
|
||||
References: Array.Empty<string>(),
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
|
||||
internal sealed record CertCcVulnerabilityDto(
|
||||
string CveId,
|
||||
string? Description,
|
||||
DateTimeOffset? DateAdded,
|
||||
DateTimeOffset? DateUpdated)
|
||||
{
|
||||
public static CertCcVulnerabilityDto Empty { get; } = new(
|
||||
CveId: string.Empty,
|
||||
Description: null,
|
||||
DateAdded: null,
|
||||
DateUpdated: null);
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using Markdig;
|
||||
using StellaOps.Concelier.Connector.Common.Html;
|
||||
using StellaOps.Concelier.Connector.Common.Url;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcNoteParser
|
||||
{
|
||||
private static readonly MarkdownPipeline MarkdownPipeline = new MarkdownPipelineBuilder()
|
||||
.UseAdvancedExtensions()
|
||||
.UseSoftlineBreakAsHardlineBreak()
|
||||
.DisableHtml()
|
||||
.Build();
|
||||
|
||||
private static readonly HtmlContentSanitizer HtmlSanitizer = new();
|
||||
private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
public static CertCcNoteDto Parse(
|
||||
ReadOnlySpan<byte> noteJson,
|
||||
ReadOnlySpan<byte> vendorsJson,
|
||||
ReadOnlySpan<byte> vulnerabilitiesJson,
|
||||
ReadOnlySpan<byte> vendorStatusesJson)
|
||||
{
|
||||
using var noteDocument = JsonDocument.Parse(noteJson.ToArray());
|
||||
var (metadata, detailUri) = ParseNoteMetadata(noteDocument.RootElement);
|
||||
|
||||
using var vendorsDocument = JsonDocument.Parse(vendorsJson.ToArray());
|
||||
var vendors = ParseVendors(vendorsDocument.RootElement, detailUri);
|
||||
|
||||
using var vulnerabilitiesDocument = JsonDocument.Parse(vulnerabilitiesJson.ToArray());
|
||||
var vulnerabilities = ParseVulnerabilities(vulnerabilitiesDocument.RootElement);
|
||||
|
||||
using var statusesDocument = JsonDocument.Parse(vendorStatusesJson.ToArray());
|
||||
var statuses = ParseVendorStatuses(statusesDocument.RootElement);
|
||||
|
||||
return new CertCcNoteDto(metadata, vendors, statuses, vulnerabilities);
|
||||
}
|
||||
|
||||
public static CertCcNoteDto ParseNote(ReadOnlySpan<byte> noteJson)
|
||||
{
|
||||
using var noteDocument = JsonDocument.Parse(noteJson.ToArray());
|
||||
var (metadata, _) = ParseNoteMetadata(noteDocument.RootElement);
|
||||
return new CertCcNoteDto(metadata, Array.Empty<CertCcVendorDto>(), Array.Empty<CertCcVendorStatusDto>(), Array.Empty<CertCcVulnerabilityDto>());
|
||||
}
|
||||
|
||||
private static (CertCcNoteMetadata Metadata, Uri DetailUri) ParseNoteMetadata(JsonElement root)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
throw new JsonException("CERT/CC note payload must be a JSON object.");
|
||||
}
|
||||
|
||||
var vuId = GetString(root, "vuid");
|
||||
var idNumber = GetString(root, "idnumber") ?? throw new JsonException("CERT/CC note missing idnumber.");
|
||||
var title = GetString(root, "name") ?? throw new JsonException("CERT/CC note missing name.");
|
||||
var detailUri = BuildDetailUri(idNumber);
|
||||
|
||||
var overview = NormalizeMarkdownToPlainText(root, "overview", detailUri);
|
||||
var summary = NormalizeMarkdownToPlainText(root, "clean_desc", detailUri);
|
||||
if (string.IsNullOrWhiteSpace(summary))
|
||||
{
|
||||
summary = NormalizeMarkdownToPlainText(root, "impact", detailUri);
|
||||
}
|
||||
|
||||
var published = ParseDate(root, "publicdate") ?? ParseDate(root, "datefirstpublished");
|
||||
var updated = ParseDate(root, "dateupdated");
|
||||
var created = ParseDate(root, "datecreated");
|
||||
var revision = ParseInt(root, "revision");
|
||||
|
||||
var cveIds = ExtractCveIds(root, "cveids");
|
||||
var references = ExtractReferenceList(root, "public", detailUri);
|
||||
|
||||
var metadata = new CertCcNoteMetadata(
|
||||
VuId: string.IsNullOrWhiteSpace(vuId) ? null : vuId.Trim(),
|
||||
IdNumber: idNumber.Trim(),
|
||||
Title: title.Trim(),
|
||||
Overview: overview,
|
||||
Summary: summary,
|
||||
Published: published?.ToUniversalTime(),
|
||||
Updated: updated?.ToUniversalTime(),
|
||||
Created: created?.ToUniversalTime(),
|
||||
Revision: revision,
|
||||
CveIds: cveIds,
|
||||
PublicUrls: references,
|
||||
PrimaryUrl: detailUri.ToString());
|
||||
|
||||
return (metadata, detailUri);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertCcVendorDto> ParseVendors(JsonElement root, Uri baseUri)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorDto>();
|
||||
}
|
||||
|
||||
var parsed = new List<CertCcVendorDto>(root.GetArrayLength());
|
||||
foreach (var element in root.EnumerateArray())
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vendor = GetString(element, "vendor");
|
||||
if (string.IsNullOrWhiteSpace(vendor))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var statement = NormalizeFreeformText(GetString(element, "statement"));
|
||||
var addendum = NormalizeFreeformText(GetString(element, "addendum"));
|
||||
var references = ExtractReferenceStringList(GetString(element, "references"), baseUri);
|
||||
|
||||
parsed.Add(new CertCcVendorDto(
|
||||
vendor.Trim(),
|
||||
ContactDate: ParseDate(element, "contact_date"),
|
||||
StatementDate: ParseDate(element, "statement_date"),
|
||||
Updated: ParseDate(element, "dateupdated"),
|
||||
Statement: statement,
|
||||
Addendum: addendum,
|
||||
References: references));
|
||||
}
|
||||
|
||||
if (parsed.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorDto>();
|
||||
}
|
||||
|
||||
return parsed
|
||||
.OrderBy(static vendor => vendor.Vendor, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertCcVulnerabilityDto> ParseVulnerabilities(JsonElement root)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVulnerabilityDto>();
|
||||
}
|
||||
|
||||
var parsed = new List<CertCcVulnerabilityDto>(root.GetArrayLength());
|
||||
foreach (var element in root.EnumerateArray())
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var cve = GetString(element, "cve");
|
||||
if (string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
parsed.Add(new CertCcVulnerabilityDto(
|
||||
NormalizeCve(cve),
|
||||
Description: NormalizeFreeformText(GetString(element, "description")),
|
||||
DateAdded: ParseDate(element, "date_added"),
|
||||
DateUpdated: ParseDate(element, "dateupdated")));
|
||||
}
|
||||
|
||||
if (parsed.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVulnerabilityDto>();
|
||||
}
|
||||
|
||||
return parsed
|
||||
.OrderBy(static vuln => vuln.CveId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertCcVendorStatusDto> ParseVendorStatuses(JsonElement root)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorStatusDto>();
|
||||
}
|
||||
|
||||
var parsed = new List<CertCcVendorStatusDto>(root.GetArrayLength());
|
||||
foreach (var element in root.EnumerateArray())
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vendor = GetString(element, "vendor");
|
||||
var cve = GetString(element, "vul");
|
||||
var status = GetString(element, "status");
|
||||
if (string.IsNullOrWhiteSpace(vendor) || string.IsNullOrWhiteSpace(cve) || string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var references = ExtractReferenceStringList(GetString(element, "references"), baseUri: null);
|
||||
parsed.Add(new CertCcVendorStatusDto(
|
||||
vendor.Trim(),
|
||||
NormalizeCve(cve),
|
||||
status.Trim(),
|
||||
NormalizeFreeformText(GetString(element, "statement")),
|
||||
references,
|
||||
DateAdded: ParseDate(element, "date_added"),
|
||||
DateUpdated: ParseDate(element, "dateupdated")));
|
||||
}
|
||||
|
||||
if (parsed.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorStatusDto>();
|
||||
}
|
||||
|
||||
return parsed
|
||||
.OrderBy(static entry => entry.CveId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static entry => entry.Vendor, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeMarkdownToPlainText(JsonElement element, string propertyName, Uri baseUri)
|
||||
=> NormalizeMarkdownToPlainText(GetString(element, propertyName), baseUri);
|
||||
|
||||
private static string? NormalizeMarkdownToPlainText(string? markdown, Uri baseUri)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(markdown))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = NormalizeLineEndings(markdown.Trim());
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var html = Markdig.Markdown.ToHtml(normalized, MarkdownPipeline);
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var sanitized = HtmlSanitizer.Sanitize(html, baseUri);
|
||||
if (string.IsNullOrWhiteSpace(sanitized))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var plain = ConvertHtmlToPlainText(sanitized);
|
||||
return string.IsNullOrWhiteSpace(plain) ? null : plain;
|
||||
}
|
||||
|
||||
private static string? NormalizeFreeformText(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = NormalizeLineEndings(value).Trim();
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var lines = normalized
|
||||
.Split('\n')
|
||||
.Select(static line => line.TrimEnd())
|
||||
.ToArray();
|
||||
|
||||
return string.Join('\n', lines).Trim();
|
||||
}
|
||||
|
||||
private static string ConvertHtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var decoded = WebUtility.HtmlDecode(html);
|
||||
decoded = decoded
|
||||
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
|
||||
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
decoded = Regex.Replace(decoded, "</p>", "\n\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
decoded = Regex.Replace(decoded, "</div>", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
decoded = Regex.Replace(decoded, "<li>", "- ", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
decoded = Regex.Replace(decoded, "</li>", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
decoded = Regex.Replace(decoded, "</tr>", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
decoded = Regex.Replace(decoded, "</td>", " \t", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
decoded = HtmlTagRegex.Replace(decoded, string.Empty);
|
||||
decoded = NormalizeLineEndings(decoded);
|
||||
|
||||
var lines = decoded
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(static line => line.Trim())
|
||||
.ToArray();
|
||||
|
||||
return string.Join('\n', lines).Trim();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractReferenceList(JsonElement element, string propertyName, Uri baseUri)
|
||||
{
|
||||
if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var candidate in raw.EnumerateArray())
|
||||
{
|
||||
if (candidate.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var text = candidate.GetString();
|
||||
if (UrlNormalizer.TryNormalize(text, baseUri, out var normalized, stripFragment: true, forceHttps: false) && normalized is not null)
|
||||
{
|
||||
references.Add(normalized.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
if (references.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return references
|
||||
.OrderBy(static url => url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractReferenceStringList(string? value, Uri? baseUri)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var buffer = ArrayPool<string>.Shared.Rent(16);
|
||||
try
|
||||
{
|
||||
var count = 0;
|
||||
var span = value.AsSpan();
|
||||
var start = 0;
|
||||
|
||||
for (var index = 0; index < span.Length; index++)
|
||||
{
|
||||
var ch = span[index];
|
||||
if (ch == '\r' || ch == '\n')
|
||||
{
|
||||
if (index > start)
|
||||
{
|
||||
AppendSegment(span, start, index - start, baseUri, buffer, ref count);
|
||||
}
|
||||
|
||||
if (ch == '\r' && index + 1 < span.Length && span[index + 1] == '\n')
|
||||
{
|
||||
index++;
|
||||
}
|
||||
|
||||
start = index + 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (start < span.Length)
|
||||
{
|
||||
AppendSegment(span, start, span.Length - start, baseUri, buffer, ref count);
|
||||
}
|
||||
|
||||
if (count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return buffer.AsSpan(0, count)
|
||||
.ToArray()
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static url => url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<string>.Shared.Return(buffer, clearArray: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendSegment(ReadOnlySpan<char> span, int start, int length, Uri? baseUri, string[] buffer, ref int count)
|
||||
{
|
||||
var segment = span.Slice(start, length).ToString().Trim();
|
||||
if (segment.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!UrlNormalizer.TryNormalize(segment, baseUri, out var normalized, stripFragment: true, forceHttps: false) || normalized is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (count >= buffer.Length)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
buffer[count++] = normalized.ToString();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCveIds(JsonElement element, string propertyName)
|
||||
{
|
||||
if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var values = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var entry in raw.EnumerateArray())
|
||||
{
|
||||
if (entry.ValueKind != JsonValueKind.String)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var text = entry.GetString();
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
values.Add(NormalizeCve(text));
|
||||
}
|
||||
|
||||
if (values.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return values
|
||||
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string NormalizeCve(string value)
|
||||
{
|
||||
var trimmed = value.Trim();
|
||||
if (!trimmed.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trimmed = $"CVE-{trimmed}";
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(trimmed.Length);
|
||||
foreach (var ch in trimmed)
|
||||
{
|
||||
builder.Append(char.ToUpperInvariant(ch));
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement element, string propertyName)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty(propertyName, out var property))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return property.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => property.GetString(),
|
||||
JsonValueKind.Number => property.ToString(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(JsonElement element, string propertyName)
|
||||
{
|
||||
var text = GetString(element, propertyName);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
? parsed.ToUniversalTime()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static int? ParseInt(JsonElement element, string propertyName)
|
||||
{
|
||||
if (!element.TryGetProperty(propertyName, out var property))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (property.ValueKind == JsonValueKind.Number && property.TryGetInt32(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
var text = GetString(element, propertyName);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : (int?)null;
|
||||
}
|
||||
|
||||
private static Uri BuildDetailUri(string idNumber)
|
||||
{
|
||||
var sanitized = idNumber.Trim();
|
||||
return new Uri($"https://www.kb.cert.org/vuls/id/{sanitized}", UriKind.Absolute);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
{
|
||||
if (value.IndexOf('\r') < 0)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcSummaryParser
|
||||
{
|
||||
public static IReadOnlyList<string> ParseNotes(byte[] payload)
|
||||
{
|
||||
if (payload is null || payload.Length == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
|
||||
{
|
||||
AllowTrailingCommas = true,
|
||||
CommentHandling = JsonCommentHandling.Skip,
|
||||
});
|
||||
|
||||
var notesElement = document.RootElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
|
||||
JsonValueKind.Array => document.RootElement,
|
||||
JsonValueKind.Null or JsonValueKind.Undefined => default,
|
||||
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
|
||||
};
|
||||
|
||||
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>(notesElement.GetArrayLength());
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var element in notesElement.EnumerateArray())
|
||||
{
|
||||
var token = ExtractToken(element);
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = token.Trim();
|
||||
var dedupKey = CreateDedupKey(normalized);
|
||||
if (seen.Add(dedupKey))
|
||||
{
|
||||
results.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0 ? Array.Empty<string>() : results;
|
||||
}
|
||||
|
||||
private static string CreateDedupKey(string token)
|
||||
{
|
||||
var digits = string.Concat(token.Where(char.IsDigit));
|
||||
return digits.Length > 0
|
||||
? digits
|
||||
: token.Trim().ToUpperInvariant();
|
||||
}
|
||||
|
||||
private static string? ExtractToken(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => element.GetString(),
|
||||
JsonValueKind.Number => element.TryGetInt64(out var number)
|
||||
? number.ToString(CultureInfo.InvariantCulture)
|
||||
: element.GetRawText(),
|
||||
JsonValueKind.Object => ExtractFromObject(element),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static string? ExtractFromObject(JsonElement element)
|
||||
{
|
||||
foreach (var propertyName in PropertyCandidates)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var value = property.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static readonly string[] PropertyCandidates =
|
||||
{
|
||||
"note",
|
||||
"notes",
|
||||
"id",
|
||||
"idnumber",
|
||||
"noteId",
|
||||
"vu",
|
||||
"vuid",
|
||||
"vuId",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
public sealed record CertCcSummaryPlan(
|
||||
TimeWindow Window,
|
||||
IReadOnlyList<CertCcSummaryRequest> Requests,
|
||||
TimeWindowCursorState NextState);
|
||||
|
||||
public enum CertCcSummaryScope
|
||||
{
|
||||
Monthly,
|
||||
Yearly,
|
||||
}
|
||||
|
||||
public sealed record CertCcSummaryRequest(
|
||||
Uri Uri,
|
||||
CertCcSummaryScope Scope,
|
||||
int Year,
|
||||
int? Month);
|
||||
@@ -0,0 +1,96 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.CertCc.Configuration;
|
||||
using StellaOps.Concelier.Connector.Common.Cursors;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
|
||||
/// </summary>
|
||||
public sealed class CertCcSummaryPlanner
|
||||
{
|
||||
private readonly CertCcOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public CertCcSummaryPlanner(
|
||||
IOptions<CertCcOptions> options,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
|
||||
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
|
||||
|
||||
var months = EnumerateYearMonths(window.Start, window.End)
|
||||
.Take(_options.MaxMonthlySummaries)
|
||||
.ToArray();
|
||||
|
||||
if (months.Length == 0)
|
||||
{
|
||||
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
|
||||
}
|
||||
|
||||
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
|
||||
foreach (var month in months)
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildMonthlyUri(month.Year, month.Month),
|
||||
CertCcSummaryScope.Monthly,
|
||||
month.Year,
|
||||
month.Month));
|
||||
}
|
||||
|
||||
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
|
||||
{
|
||||
requests.Add(new CertCcSummaryRequest(
|
||||
BuildYearlyUri(year),
|
||||
CertCcSummaryScope.Yearly,
|
||||
year,
|
||||
Month: null));
|
||||
}
|
||||
|
||||
return new CertCcSummaryPlan(window, requests, nextState);
|
||||
}
|
||||
|
||||
private Uri BuildMonthlyUri(int year, int month)
|
||||
{
|
||||
var path = $"{year:D4}/{month:D2}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private Uri BuildYearlyUri(int year)
|
||||
{
|
||||
var path = $"{year:D4}/summary/";
|
||||
return new Uri(_options.BaseApiUri, path);
|
||||
}
|
||||
|
||||
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
|
||||
{
|
||||
if (end <= start)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
|
||||
{
|
||||
limit = limit.AddMonths(1);
|
||||
}
|
||||
|
||||
while (cursor < limit)
|
||||
{
|
||||
yield return (cursor.Year, cursor.Month);
|
||||
cursor = cursor.AddMonths(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.CertCc.Internal;
|
||||
|
||||
internal static class CertCcVendorStatementParser
|
||||
{
|
||||
private static readonly string[] PairSeparators =
|
||||
{
|
||||
"\t",
|
||||
" - ",
|
||||
" – ",
|
||||
" — ",
|
||||
" : ",
|
||||
": ",
|
||||
" :",
|
||||
":",
|
||||
};
|
||||
|
||||
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
|
||||
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
|
||||
|
||||
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
|
||||
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(statement))
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
var patches = new List<CertCcVendorPatch>();
|
||||
var lines = statement
|
||||
.Replace("\r\n", "\n", StringComparison.Ordinal)
|
||||
.Replace('\r', '\n')
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
foreach (var rawLine in lines)
|
||||
{
|
||||
var line = rawLine.Trim();
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
line = TrimBulletPrefix(line);
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var versions = ExtractVersions(versionSegment);
|
||||
if (versions.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var products = ExtractProducts(productSegment);
|
||||
if (products.Count == 0)
|
||||
{
|
||||
products.Add(string.Empty);
|
||||
}
|
||||
|
||||
if (versions.Count == products.Count)
|
||||
{
|
||||
for (var index = 0; index < versions.Count; index++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
|
||||
{
|
||||
var groupSize = products.Count / versions.Count;
|
||||
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
|
||||
{
|
||||
var start = versionIndex * groupSize;
|
||||
var end = start + groupSize;
|
||||
var version = versions[versionIndex];
|
||||
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var primaryVersion = versions[0];
|
||||
foreach (var product in products)
|
||||
{
|
||||
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
|
||||
}
|
||||
}
|
||||
|
||||
if (patches.Count == 0)
|
||||
{
|
||||
return Array.Empty<CertCcVendorPatch>();
|
||||
}
|
||||
|
||||
return patches
|
||||
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
|
||||
.Distinct(CertCcVendorPatch.Comparer)
|
||||
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string TrimBulletPrefix(string value)
|
||||
{
|
||||
var trimmed = value.TrimStart(BulletPrefixes).Trim();
|
||||
return trimmed.Length == 0 ? value.Trim() : trimmed;
|
||||
}
|
||||
|
||||
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
|
||||
{
|
||||
foreach (var separator in PairSeparators)
|
||||
{
|
||||
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
productSegment = parts[0];
|
||||
versionSegment = parts[1];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (whitespaceSplit.Length >= 2)
|
||||
{
|
||||
productSegment = string.Join(' ', whitespaceSplit[..^1]);
|
||||
versionSegment = whitespaceSplit[^1];
|
||||
return true;
|
||||
}
|
||||
|
||||
productSegment = string.Empty;
|
||||
versionSegment = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static List<string> ExtractProducts(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var normalized = segment.Replace('\t', ' ').Trim();
|
||||
var tokens = normalized
|
||||
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(static token => token.Trim())
|
||||
.Where(static token => token.Length > 0)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
private static List<string> ExtractVersions(string segment)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(segment))
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var matches = VersionTokenRegex.Matches(segment);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
var versions = new List<string>(matches.Count);
|
||||
foreach (Match match in matches)
|
||||
{
|
||||
if (match.Groups.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = match.Groups[1].Value.Trim();
|
||||
if (value.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
versions.Add(value);
|
||||
}
|
||||
|
||||
return versions
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(32)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
|
||||
{
|
||||
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
|
||||
|
||||
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
|
||||
{
|
||||
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(CertCcVendorPatch obj)
|
||||
{
|
||||
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
|
||||
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
|
||||
return HashCode.Combine(product, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user