Restructure solution layout by module

This commit is contained in:
master
2025-10-28 15:10:40 +02:00
parent 95daa159c4
commit d870da18ce
4103 changed files with 192899 additions and 187024 deletions

View File

@@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Kev.Internal;
internal sealed record KevCatalogDto
{
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("catalogVersion")]
public string? CatalogVersion { get; init; }
[JsonPropertyName("dateReleased")]
public DateTimeOffset? DateReleased { get; init; }
[JsonPropertyName("count")]
public int Count { get; init; }
[JsonPropertyName("vulnerabilities")]
public IReadOnlyList<KevVulnerabilityDto> Vulnerabilities { get; init; } = Array.Empty<KevVulnerabilityDto>();
}
internal sealed record KevVulnerabilityDto
{
[JsonPropertyName("cveID")]
public string? CveId { get; init; }
[JsonPropertyName("vendorProject")]
public string? VendorProject { get; init; }
[JsonPropertyName("product")]
public string? Product { get; init; }
[JsonPropertyName("vulnerabilityName")]
public string? VulnerabilityName { get; init; }
[JsonPropertyName("dateAdded")]
public string? DateAdded { get; init; }
[JsonPropertyName("shortDescription")]
public string? ShortDescription { get; init; }
[JsonPropertyName("requiredAction")]
public string? RequiredAction { get; init; }
[JsonPropertyName("dueDate")]
public string? DueDate { get; init; }
[JsonPropertyName("knownRansomwareCampaignUse")]
public string? KnownRansomwareCampaignUse { get; init; }
[JsonPropertyName("notes")]
public string? Notes { get; init; }
[JsonPropertyName("cwes")]
public IReadOnlyList<string> Cwes { get; init; } = Array.Empty<string>();
}

View File

@@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Kev.Internal;
internal sealed record KevCursor(
string? CatalogVersion,
DateTimeOffset? CatalogReleased,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static KevCursor Empty { get; } = new(null, null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())),
};
if (!string.IsNullOrEmpty(CatalogVersion))
{
document["catalogVersion"] = CatalogVersion;
}
if (CatalogReleased.HasValue)
{
document["catalogReleased"] = CatalogReleased.Value.UtcDateTime;
}
return document;
}
public static KevCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var version = document.TryGetValue("catalogVersion", out var versionValue)
? versionValue.AsString
: null;
var released = document.TryGetValue("catalogReleased", out var releasedValue)
? ParseDate(releasedValue)
: null;
return new KevCursor(
version,
released,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public KevCursor WithCatalogMetadata(string? version, DateTimeOffset? released)
=> this with
{
CatalogVersion = string.IsNullOrWhiteSpace(version) ? null : version.Trim(),
CatalogReleased = released?.ToUniversalTime(),
};
public KevCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public KevCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

View File

@@ -0,0 +1,113 @@
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Kev.Internal;
public sealed class KevDiagnostics : IDisposable
{
public const string MeterName = "StellaOps.Concelier.Connector.Kev";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchSuccess;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _parsedEntries;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseAnomalies;
private readonly Counter<long> _mappedAdvisories;
public KevDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>(
name: "kev.fetch.attempts",
unit: "operations",
description: "Number of KEV fetch attempts performed.");
_fetchSuccess = _meter.CreateCounter<long>(
name: "kev.fetch.success",
unit: "operations",
description: "Number of KEV fetch attempts that produced new catalog content.");
_fetchFailures = _meter.CreateCounter<long>(
name: "kev.fetch.failures",
unit: "operations",
description: "Number of KEV fetch attempts that failed.");
_fetchUnchanged = _meter.CreateCounter<long>(
name: "kev.fetch.unchanged",
unit: "operations",
description: "Number of KEV fetch attempts returning HTTP 304 / unchanged catalog.");
_parsedEntries = _meter.CreateCounter<long>(
name: "kev.parse.entries",
unit: "entries",
description: "Number of KEV vulnerabilities parsed from the catalog.");
_parseFailures = _meter.CreateCounter<long>(
name: "kev.parse.failures",
unit: "documents",
description: "Number of KEV catalog parse operations that failed or were quarantined.");
_parseAnomalies = _meter.CreateCounter<long>(
name: "kev.parse.anomalies",
unit: "entries",
description: "Number of KEV entries skipped or flagged during parsing due to anomalies.");
_mappedAdvisories = _meter.CreateCounter<long>(
name: "kev.map.advisories",
unit: "advisories",
description: "Number of KEV advisories emitted during mapping.");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchSuccess() => _fetchSuccess.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void CatalogParsed(string? catalogVersion, int entryCount)
{
if (entryCount <= 0)
{
return;
}
_parsedEntries.Add(entryCount, new KeyValuePair<string, object?>("catalogVersion", catalogVersion ?? string.Empty));
}
public void ParseFailure(string reason, string? catalogVersion = null)
{
var tags = string.IsNullOrWhiteSpace(catalogVersion)
? new[] { new KeyValuePair<string, object?>("reason", reason) }
: new[]
{
new KeyValuePair<string, object?>("reason", reason),
new KeyValuePair<string, object?>("catalogVersion", catalogVersion)
};
_parseFailures.Add(1, tags);
}
public void RecordAnomaly(string reason, string? catalogVersion = null)
{
var tags = string.IsNullOrWhiteSpace(catalogVersion)
? new[] { new KeyValuePair<string, object?>("reason", reason) }
: new[]
{
new KeyValuePair<string, object?>("reason", reason),
new KeyValuePair<string, object?>("catalogVersion", catalogVersion)
};
_parseAnomalies.Add(1, tags);
}
public void AdvisoriesMapped(string? catalogVersion, int advisoryCount)
{
if (advisoryCount <= 0)
{
return;
}
_mappedAdvisories.Add(advisoryCount, new KeyValuePair<string, object?>("catalogVersion", catalogVersion ?? string.Empty));
}
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,373 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.Kev.Internal;
internal static class KevMapper
{
public static IReadOnlyList<Advisory> Map(
KevCatalogDto catalog,
string sourceName,
Uri feedUri,
DateTimeOffset fetchedAt,
DateTimeOffset validatedAt)
{
ArgumentNullException.ThrowIfNull(catalog);
ArgumentNullException.ThrowIfNull(sourceName);
ArgumentNullException.ThrowIfNull(feedUri);
var advisories = new List<Advisory>();
var fetchProvenance = new AdvisoryProvenance(sourceName, "document", feedUri.ToString(), fetchedAt);
var mappingProvenance = new AdvisoryProvenance(
sourceName,
"mapping",
catalog.CatalogVersion ?? feedUri.ToString(),
validatedAt);
if (catalog.Vulnerabilities is null || catalog.Vulnerabilities.Count == 0)
{
return advisories;
}
foreach (var entry in catalog.Vulnerabilities)
{
if (entry is null)
{
continue;
}
var cveId = Normalize(entry.CveId);
if (string.IsNullOrEmpty(cveId))
{
continue;
}
var advisoryKey = $"kev/{cveId.ToLowerInvariant()}";
var title = Normalize(entry.VulnerabilityName) ?? cveId;
var summary = Normalize(entry.ShortDescription);
var published = ParseDate(entry.DateAdded);
var dueDate = ParseDate(entry.DueDate);
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { cveId };
var references = BuildReferences(entry, sourceName, mappingProvenance, feedUri, cveId).ToArray();
var affectedPackages = BuildAffectedPackages(
entry,
catalog,
sourceName,
mappingProvenance,
published,
dueDate).ToArray();
var provenance = new[]
{
fetchProvenance,
mappingProvenance
};
advisories.Add(new Advisory(
advisoryKey,
title,
summary,
language: "en",
published,
modified: catalog.DateReleased?.ToUniversalTime(),
severity: null,
exploitKnown: true,
aliases,
references,
affectedPackages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance));
}
return advisories
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.ToArray();
}
private static IEnumerable<AdvisoryReference> BuildReferences(
KevVulnerabilityDto entry,
string sourceName,
AdvisoryProvenance mappingProvenance,
Uri feedUri,
string cveId)
{
var references = new List<AdvisoryReference>();
var provenance = new AdvisoryProvenance(sourceName, "reference", cveId, mappingProvenance.RecordedAt);
var catalogUrl = BuildCatalogSearchUrl(cveId);
if (catalogUrl is not null)
{
TryAddReference(references, catalogUrl, "advisory", "cisa-kev", provenance);
}
TryAddReference(references, feedUri.ToString(), "reference", "cisa-kev-feed", provenance);
foreach (var url in ExtractUrls(entry.Notes))
{
TryAddReference(references, url, "reference", "kev.notes", provenance);
}
return references
.GroupBy(static r => r.Url, StringComparer.OrdinalIgnoreCase)
.Select(static group => group
.OrderBy(static r => r.Kind, StringComparer.Ordinal)
.ThenBy(static r => r.SourceTag, StringComparer.Ordinal)
.First())
.OrderBy(static r => r.Kind, StringComparer.Ordinal)
.ThenBy(static r => r.Url, StringComparer.Ordinal)
.ToArray();
}
private static void TryAddReference(
ICollection<AdvisoryReference> references,
string? url,
string kind,
string? sourceTag,
AdvisoryProvenance provenance)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed)
|| (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps))
{
return;
}
try
{
references.Add(new AdvisoryReference(parsed.ToString(), kind, sourceTag, null, provenance));
}
catch (ArgumentException)
{
// Ignore invalid references while leaving traceability via diagnostics elsewhere.
}
}
private static string? BuildCatalogSearchUrl(string cveId)
{
if (string.IsNullOrWhiteSpace(cveId))
{
return null;
}
var builder = new StringBuilder("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=");
builder.Append(Uri.EscapeDataString(cveId));
return builder.ToString();
}
private static IEnumerable<AffectedPackage> BuildAffectedPackages(
KevVulnerabilityDto entry,
KevCatalogDto catalog,
string sourceName,
AdvisoryProvenance mappingProvenance,
DateTimeOffset? published,
DateTimeOffset? dueDate)
{
var identifier = BuildIdentifier(entry) ?? entry.CveId ?? "kev";
var rangeExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
void TryAddExtension(string key, string? value, int maxLength = 512)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var trimmed = value.Trim();
if (trimmed.Length > maxLength)
{
trimmed = trimmed[..maxLength].Trim();
}
if (trimmed.Length > 0)
{
rangeExtensions[key] = trimmed;
}
}
TryAddExtension("kev.vendorProject", entry.VendorProject, 256);
TryAddExtension("kev.product", entry.Product, 256);
TryAddExtension("kev.requiredAction", entry.RequiredAction);
TryAddExtension("kev.knownRansomwareCampaignUse", entry.KnownRansomwareCampaignUse, 64);
TryAddExtension("kev.notes", entry.Notes);
TryAddExtension("kev.catalogVersion", catalog.CatalogVersion, 64);
if (catalog.DateReleased.HasValue)
{
TryAddExtension("kev.catalogReleased", catalog.DateReleased.Value.ToString("O", CultureInfo.InvariantCulture));
}
if (published.HasValue)
{
TryAddExtension("kev.dateAdded", published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
}
if (dueDate.HasValue)
{
TryAddExtension("kev.dueDate", dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
}
if (entry.Cwes is { Count: > 0 })
{
TryAddExtension("kev.cwe", string.Join(",", entry.Cwes.Where(static cwe => !string.IsNullOrWhiteSpace(cwe)).OrderBy(static cwe => cwe, StringComparer.Ordinal)));
}
if (rangeExtensions.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var rangeProvenance = new AdvisoryProvenance(sourceName, "kev-range", identifier, mappingProvenance.RecordedAt);
var range = new AffectedVersionRange(
rangeKind: AffectedPackageTypes.Vendor,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: null,
provenance: rangeProvenance,
primitives: new RangePrimitives(null, null, null, rangeExtensions));
var normalizedVersions = BuildNormalizedVersions(identifier, catalog, published, dueDate);
var affectedPackage = new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: new[] { range },
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { mappingProvenance },
normalizedVersions: normalizedVersions);
return new[] { affectedPackage };
}
private static string? BuildIdentifier(KevVulnerabilityDto entry)
{
var vendor = Normalize(entry.VendorProject);
var product = Normalize(entry.Product);
if (!string.IsNullOrEmpty(vendor) && !string.IsNullOrEmpty(product))
{
return $"{vendor}::{product}";
}
return vendor ?? product;
}
private static IEnumerable<string> ExtractUrls(string? notes)
{
if (string.IsNullOrWhiteSpace(notes))
{
return Array.Empty<string>();
}
var tokens = notes.Split(new[] { ';', ',', ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries);
var results = new List<string>();
foreach (var token in tokens)
{
var trimmed = token.Trim().TrimEnd('.', ')', ';', ',');
if (trimmed.Length == 0)
{
continue;
}
if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)
&& (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps))
{
results.Add(uri.ToString());
}
}
return results.Count == 0
? Array.Empty<string>()
: results.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(static value => value, StringComparer.Ordinal).ToArray();
}
private static string? Normalize(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var trimmed = value.Trim();
return trimmed.Length == 0 ? null : trimmed;
}
private static DateTimeOffset? ParseDate(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed.ToUniversalTime();
}
if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var date))
{
return new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Utc));
}
return null;
}
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
string identifier,
KevCatalogDto catalog,
DateTimeOffset? published,
DateTimeOffset? dueDate)
{
var rules = new List<NormalizedVersionRule>();
var notes = Validation.TrimToNull(identifier);
if (!string.IsNullOrWhiteSpace(catalog.CatalogVersion))
{
rules.Add(new NormalizedVersionRule(
scheme: "kev.catalog",
type: NormalizedVersionRuleTypes.Exact,
value: catalog.CatalogVersion.Trim(),
notes: notes));
}
if (published.HasValue)
{
rules.Add(new NormalizedVersionRule(
scheme: "kev.date-added",
type: NormalizedVersionRuleTypes.Exact,
value: published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
notes: notes));
}
if (dueDate.HasValue)
{
rules.Add(new NormalizedVersionRule(
scheme: "kev.due-date",
type: NormalizedVersionRuleTypes.LessThanOrEqual,
max: dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
maxInclusive: true,
notes: notes));
}
return rules.Count == 0
? Array.Empty<NormalizedVersionRule>()
: rules
.OrderBy(static rule => rule.Scheme, StringComparer.Ordinal)
.ThenBy(static rule => rule.Type, StringComparer.Ordinal)
.ThenBy(static rule => rule.Value ?? rule.Max ?? string.Empty, StringComparer.Ordinal)
.ToArray();
}
}

View File

@@ -0,0 +1,25 @@
using System.IO;
using System.Reflection;
using System.Threading;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Kev.Internal;
internal static class KevSchemaProvider
{
private const string ResourceName = "StellaOps.Concelier.Connector.Kev.Schemas.kev-catalog.schema.json";
private static readonly Lazy<JsonSchema> CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication);
public static JsonSchema Schema => CachedSchema.Value;
private static JsonSchema LoadSchema()
{
var assembly = typeof(KevSchemaProvider).GetTypeInfo().Assembly;
using var stream = assembly.GetManifestResourceStream(ResourceName)
?? throw new InvalidOperationException($"Embedded schema '{ResourceName}' was not found.");
using var reader = new StreamReader(stream);
var schemaJson = reader.ReadToEnd();
return JsonSchema.FromText(schemaJson);
}
}