Restructure solution layout by module
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Implement the CISA Known Exploited Vulnerabilities (KEV) catalogue connector to ingest KEV entries for enrichment and policy checks.
|
||||
|
||||
## Scope
|
||||
- Integrate with the official KEV JSON feed; understand schema, update cadence, and pagination (if any).
|
||||
- Implement fetch job with incremental updates, checksum validation, and cursor persistence.
|
||||
- Parse KEV entries (CVE ID, vendor/product, required actions, due dates).
|
||||
- Map entries into canonical `Advisory` (or augmentation) records with aliases, references, affected packages, and range primitives capturing enforcement metadata.
|
||||
- Deliver deterministic fixtures and regression tests.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP client, fetch service, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores, source state).
|
||||
- `Concelier.Models` (advisory + range primitive types).
|
||||
- `Concelier.Testing` (integration fixtures & snapshots).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `kev:fetch`, `kev:parse`, `kev:map`.
|
||||
- Persist upstream `catalogLastUpdated` / ETag to detect changes.
|
||||
- Alias list must include CVE ID; references should point to CISA KEV listing and vendor advisories.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- KEV feed ingestion and canonical mapping.
|
||||
- Range primitives capturing remediation due dates or vendor requirements.
|
||||
|
||||
Out of scope:
|
||||
- Compliance policy enforcement (handled elsewhere).
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch timestamps, updated entry counts, and mapping stats.
|
||||
- Handle data anomalies and record failures with backoff.
|
||||
- Validate JSON payloads before persistence.
|
||||
- Structured informational logs should surface the catalog version, release timestamp, and advisory counts for each successful parse/map cycle.
|
||||
|
||||
## Operational Notes
|
||||
- HTTP allowlist is limited to `www.cisa.gov`; operators should mirror / proxy that hostname for air-gapped deployments.
|
||||
- CISA publishes KEV updates daily (catalogVersion follows `yyyy.MM.dd`). Expect releases near 16:30–17:00 UTC and retain overlap when scheduling fetches.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Kev.Tests` covering fetch/parse/map with KEV JSON fixtures.
|
||||
- Snapshot canonical output; allow fixture regeneration via env flag.
|
||||
- Ensure deterministic ordering/time normalisation.
|
||||
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Configuration;
|
||||
|
||||
public sealed class KevOptions
|
||||
{
|
||||
public static string HttpClientName => "source.kev";
|
||||
|
||||
/// <summary>
|
||||
/// Official CISA Known Exploited Vulnerabilities JSON feed.
|
||||
/// </summary>
|
||||
public Uri FeedUri { get; set; } = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to KEV feed requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
[MemberNotNull(nameof(FeedUri))]
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("FeedUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestTimeout must be greater than zero.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
internal sealed record KevCatalogDto
|
||||
{
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("catalogVersion")]
|
||||
public string? CatalogVersion { get; init; }
|
||||
|
||||
[JsonPropertyName("dateReleased")]
|
||||
public DateTimeOffset? DateReleased { get; init; }
|
||||
|
||||
[JsonPropertyName("count")]
|
||||
public int Count { get; init; }
|
||||
|
||||
[JsonPropertyName("vulnerabilities")]
|
||||
public IReadOnlyList<KevVulnerabilityDto> Vulnerabilities { get; init; } = Array.Empty<KevVulnerabilityDto>();
|
||||
}
|
||||
|
||||
internal sealed record KevVulnerabilityDto
|
||||
{
|
||||
[JsonPropertyName("cveID")]
|
||||
public string? CveId { get; init; }
|
||||
|
||||
[JsonPropertyName("vendorProject")]
|
||||
public string? VendorProject { get; init; }
|
||||
|
||||
[JsonPropertyName("product")]
|
||||
public string? Product { get; init; }
|
||||
|
||||
[JsonPropertyName("vulnerabilityName")]
|
||||
public string? VulnerabilityName { get; init; }
|
||||
|
||||
[JsonPropertyName("dateAdded")]
|
||||
public string? DateAdded { get; init; }
|
||||
|
||||
[JsonPropertyName("shortDescription")]
|
||||
public string? ShortDescription { get; init; }
|
||||
|
||||
[JsonPropertyName("requiredAction")]
|
||||
public string? RequiredAction { get; init; }
|
||||
|
||||
[JsonPropertyName("dueDate")]
|
||||
public string? DueDate { get; init; }
|
||||
|
||||
[JsonPropertyName("knownRansomwareCampaignUse")]
|
||||
public string? KnownRansomwareCampaignUse { get; init; }
|
||||
|
||||
[JsonPropertyName("notes")]
|
||||
public string? Notes { get; init; }
|
||||
|
||||
[JsonPropertyName("cwes")]
|
||||
public IReadOnlyList<string> Cwes { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
internal sealed record KevCursor(
|
||||
string? CatalogVersion,
|
||||
DateTimeOffset? CatalogReleased,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
public static KevCursor Empty { get; } = new(null, null, Array.Empty<Guid>(), Array.Empty<Guid>());
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(CatalogVersion))
|
||||
{
|
||||
document["catalogVersion"] = CatalogVersion;
|
||||
}
|
||||
|
||||
if (CatalogReleased.HasValue)
|
||||
{
|
||||
document["catalogReleased"] = CatalogReleased.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static KevCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var version = document.TryGetValue("catalogVersion", out var versionValue)
|
||||
? versionValue.AsString
|
||||
: null;
|
||||
|
||||
var released = document.TryGetValue("catalogReleased", out var releasedValue)
|
||||
? ParseDate(releasedValue)
|
||||
: null;
|
||||
|
||||
return new KevCursor(
|
||||
version,
|
||||
released,
|
||||
ReadGuidArray(document, "pendingDocuments"),
|
||||
ReadGuidArray(document, "pendingMappings"));
|
||||
}
|
||||
|
||||
public KevCursor WithCatalogMetadata(string? version, DateTimeOffset? released)
|
||||
=> this with
|
||||
{
|
||||
CatalogVersion = string.IsNullOrWhiteSpace(version) ? null : version.Trim(),
|
||||
CatalogReleased = released?.ToUniversalTime(),
|
||||
};
|
||||
|
||||
public KevCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
public KevCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
public sealed class KevDiagnostics : IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Concelier.Connector.Kev";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _parsedEntries;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _parseAnomalies;
|
||||
private readonly Counter<long> _mappedAdvisories;
|
||||
|
||||
public KevDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "kev.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of KEV fetch attempts performed.");
|
||||
_fetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "kev.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of KEV fetch attempts that produced new catalog content.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "kev.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of KEV fetch attempts that failed.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "kev.fetch.unchanged",
|
||||
unit: "operations",
|
||||
description: "Number of KEV fetch attempts returning HTTP 304 / unchanged catalog.");
|
||||
_parsedEntries = _meter.CreateCounter<long>(
|
||||
name: "kev.parse.entries",
|
||||
unit: "entries",
|
||||
description: "Number of KEV vulnerabilities parsed from the catalog.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "kev.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of KEV catalog parse operations that failed or were quarantined.");
|
||||
_parseAnomalies = _meter.CreateCounter<long>(
|
||||
name: "kev.parse.anomalies",
|
||||
unit: "entries",
|
||||
description: "Number of KEV entries skipped or flagged during parsing due to anomalies.");
|
||||
_mappedAdvisories = _meter.CreateCounter<long>(
|
||||
name: "kev.map.advisories",
|
||||
unit: "advisories",
|
||||
description: "Number of KEV advisories emitted during mapping.");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchSuccess() => _fetchSuccess.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void CatalogParsed(string? catalogVersion, int entryCount)
|
||||
{
|
||||
if (entryCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_parsedEntries.Add(entryCount, new KeyValuePair<string, object?>("catalogVersion", catalogVersion ?? string.Empty));
|
||||
}
|
||||
|
||||
public void ParseFailure(string reason, string? catalogVersion = null)
|
||||
{
|
||||
var tags = string.IsNullOrWhiteSpace(catalogVersion)
|
||||
? new[] { new KeyValuePair<string, object?>("reason", reason) }
|
||||
: new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("reason", reason),
|
||||
new KeyValuePair<string, object?>("catalogVersion", catalogVersion)
|
||||
};
|
||||
|
||||
_parseFailures.Add(1, tags);
|
||||
}
|
||||
|
||||
public void RecordAnomaly(string reason, string? catalogVersion = null)
|
||||
{
|
||||
var tags = string.IsNullOrWhiteSpace(catalogVersion)
|
||||
? new[] { new KeyValuePair<string, object?>("reason", reason) }
|
||||
: new[]
|
||||
{
|
||||
new KeyValuePair<string, object?>("reason", reason),
|
||||
new KeyValuePair<string, object?>("catalogVersion", catalogVersion)
|
||||
};
|
||||
|
||||
_parseAnomalies.Add(1, tags);
|
||||
}
|
||||
|
||||
public void AdvisoriesMapped(string? catalogVersion, int advisoryCount)
|
||||
{
|
||||
if (advisoryCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_mappedAdvisories.Add(advisoryCount, new KeyValuePair<string, object?>("catalogVersion", catalogVersion ?? string.Empty));
|
||||
}
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
internal static class KevMapper
|
||||
{
|
||||
public static IReadOnlyList<Advisory> Map(
|
||||
KevCatalogDto catalog,
|
||||
string sourceName,
|
||||
Uri feedUri,
|
||||
DateTimeOffset fetchedAt,
|
||||
DateTimeOffset validatedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(catalog);
|
||||
ArgumentNullException.ThrowIfNull(sourceName);
|
||||
ArgumentNullException.ThrowIfNull(feedUri);
|
||||
|
||||
var advisories = new List<Advisory>();
|
||||
var fetchProvenance = new AdvisoryProvenance(sourceName, "document", feedUri.ToString(), fetchedAt);
|
||||
var mappingProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"mapping",
|
||||
catalog.CatalogVersion ?? feedUri.ToString(),
|
||||
validatedAt);
|
||||
|
||||
if (catalog.Vulnerabilities is null || catalog.Vulnerabilities.Count == 0)
|
||||
{
|
||||
return advisories;
|
||||
}
|
||||
|
||||
foreach (var entry in catalog.Vulnerabilities)
|
||||
{
|
||||
if (entry is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var cveId = Normalize(entry.CveId);
|
||||
if (string.IsNullOrEmpty(cveId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisoryKey = $"kev/{cveId.ToLowerInvariant()}";
|
||||
var title = Normalize(entry.VulnerabilityName) ?? cveId;
|
||||
var summary = Normalize(entry.ShortDescription);
|
||||
var published = ParseDate(entry.DateAdded);
|
||||
var dueDate = ParseDate(entry.DueDate);
|
||||
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { cveId };
|
||||
|
||||
var references = BuildReferences(entry, sourceName, mappingProvenance, feedUri, cveId).ToArray();
|
||||
|
||||
var affectedPackages = BuildAffectedPackages(
|
||||
entry,
|
||||
catalog,
|
||||
sourceName,
|
||||
mappingProvenance,
|
||||
published,
|
||||
dueDate).ToArray();
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
fetchProvenance,
|
||||
mappingProvenance
|
||||
};
|
||||
|
||||
advisories.Add(new Advisory(
|
||||
advisoryKey,
|
||||
title,
|
||||
summary,
|
||||
language: "en",
|
||||
published,
|
||||
modified: catalog.DateReleased?.ToUniversalTime(),
|
||||
severity: null,
|
||||
exploitKnown: true,
|
||||
aliases,
|
||||
references,
|
||||
affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance));
|
||||
}
|
||||
|
||||
return advisories
|
||||
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<AdvisoryReference> BuildReferences(
|
||||
KevVulnerabilityDto entry,
|
||||
string sourceName,
|
||||
AdvisoryProvenance mappingProvenance,
|
||||
Uri feedUri,
|
||||
string cveId)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var provenance = new AdvisoryProvenance(sourceName, "reference", cveId, mappingProvenance.RecordedAt);
|
||||
|
||||
var catalogUrl = BuildCatalogSearchUrl(cveId);
|
||||
if (catalogUrl is not null)
|
||||
{
|
||||
TryAddReference(references, catalogUrl, "advisory", "cisa-kev", provenance);
|
||||
}
|
||||
|
||||
TryAddReference(references, feedUri.ToString(), "reference", "cisa-kev-feed", provenance);
|
||||
|
||||
foreach (var url in ExtractUrls(entry.Notes))
|
||||
{
|
||||
TryAddReference(references, url, "reference", "kev.notes", provenance);
|
||||
}
|
||||
|
||||
return references
|
||||
.GroupBy(static r => r.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(static group => group
|
||||
.OrderBy(static r => r.Kind, StringComparer.Ordinal)
|
||||
.ThenBy(static r => r.SourceTag, StringComparer.Ordinal)
|
||||
.First())
|
||||
.OrderBy(static r => r.Kind, StringComparer.Ordinal)
|
||||
.ThenBy(static r => r.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static void TryAddReference(
|
||||
ICollection<AdvisoryReference> references,
|
||||
string? url,
|
||||
string kind,
|
||||
string? sourceTag,
|
||||
AdvisoryProvenance provenance)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed)
|
||||
|| (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(parsed.ToString(), kind, sourceTag, null, provenance));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// Ignore invalid references while leaving traceability via diagnostics elsewhere.
|
||||
}
|
||||
}
|
||||
|
||||
private static string? BuildCatalogSearchUrl(string cveId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var builder = new StringBuilder("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=");
|
||||
builder.Append(Uri.EscapeDataString(cveId));
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static IEnumerable<AffectedPackage> BuildAffectedPackages(
|
||||
KevVulnerabilityDto entry,
|
||||
KevCatalogDto catalog,
|
||||
string sourceName,
|
||||
AdvisoryProvenance mappingProvenance,
|
||||
DateTimeOffset? published,
|
||||
DateTimeOffset? dueDate)
|
||||
{
|
||||
var identifier = BuildIdentifier(entry) ?? entry.CveId ?? "kev";
|
||||
var rangeExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void TryAddExtension(string key, string? value, int maxLength = 512)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (trimmed.Length > maxLength)
|
||||
{
|
||||
trimmed = trimmed[..maxLength].Trim();
|
||||
}
|
||||
|
||||
if (trimmed.Length > 0)
|
||||
{
|
||||
rangeExtensions[key] = trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
TryAddExtension("kev.vendorProject", entry.VendorProject, 256);
|
||||
TryAddExtension("kev.product", entry.Product, 256);
|
||||
TryAddExtension("kev.requiredAction", entry.RequiredAction);
|
||||
TryAddExtension("kev.knownRansomwareCampaignUse", entry.KnownRansomwareCampaignUse, 64);
|
||||
TryAddExtension("kev.notes", entry.Notes);
|
||||
TryAddExtension("kev.catalogVersion", catalog.CatalogVersion, 64);
|
||||
|
||||
if (catalog.DateReleased.HasValue)
|
||||
{
|
||||
TryAddExtension("kev.catalogReleased", catalog.DateReleased.Value.ToString("O", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
if (published.HasValue)
|
||||
{
|
||||
TryAddExtension("kev.dateAdded", published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
if (dueDate.HasValue)
|
||||
{
|
||||
TryAddExtension("kev.dueDate", dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
if (entry.Cwes is { Count: > 0 })
|
||||
{
|
||||
TryAddExtension("kev.cwe", string.Join(",", entry.Cwes.Where(static cwe => !string.IsNullOrWhiteSpace(cwe)).OrderBy(static cwe => cwe, StringComparer.Ordinal)));
|
||||
}
|
||||
|
||||
if (rangeExtensions.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var rangeProvenance = new AdvisoryProvenance(sourceName, "kev-range", identifier, mappingProvenance.RecordedAt);
|
||||
var range = new AffectedVersionRange(
|
||||
rangeKind: AffectedPackageTypes.Vendor,
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: null,
|
||||
provenance: rangeProvenance,
|
||||
primitives: new RangePrimitives(null, null, null, rangeExtensions));
|
||||
|
||||
var normalizedVersions = BuildNormalizedVersions(identifier, catalog, published, dueDate);
|
||||
|
||||
var affectedPackage = new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: new[] { range },
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { mappingProvenance },
|
||||
normalizedVersions: normalizedVersions);
|
||||
|
||||
return new[] { affectedPackage };
|
||||
}
|
||||
|
||||
private static string? BuildIdentifier(KevVulnerabilityDto entry)
|
||||
{
|
||||
var vendor = Normalize(entry.VendorProject);
|
||||
var product = Normalize(entry.Product);
|
||||
|
||||
if (!string.IsNullOrEmpty(vendor) && !string.IsNullOrEmpty(product))
|
||||
{
|
||||
return $"{vendor}::{product}";
|
||||
}
|
||||
|
||||
return vendor ?? product;
|
||||
}
|
||||
|
||||
private static IEnumerable<string> ExtractUrls(string? notes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(notes))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var tokens = notes.Split(new[] { ';', ',', ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
var trimmed = token.Trim().TrimEnd('.', ')', ';', ',');
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)
|
||||
&& (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps))
|
||||
{
|
||||
results.Add(uri.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: results.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(static value => value, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
return trimmed.Length == 0 ? null : trimmed;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return parsed.ToUniversalTime();
|
||||
}
|
||||
|
||||
if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var date))
|
||||
{
|
||||
return new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(
|
||||
string identifier,
|
||||
KevCatalogDto catalog,
|
||||
DateTimeOffset? published,
|
||||
DateTimeOffset? dueDate)
|
||||
{
|
||||
var rules = new List<NormalizedVersionRule>();
|
||||
var notes = Validation.TrimToNull(identifier);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(catalog.CatalogVersion))
|
||||
{
|
||||
rules.Add(new NormalizedVersionRule(
|
||||
scheme: "kev.catalog",
|
||||
type: NormalizedVersionRuleTypes.Exact,
|
||||
value: catalog.CatalogVersion.Trim(),
|
||||
notes: notes));
|
||||
}
|
||||
|
||||
if (published.HasValue)
|
||||
{
|
||||
rules.Add(new NormalizedVersionRule(
|
||||
scheme: "kev.date-added",
|
||||
type: NormalizedVersionRuleTypes.Exact,
|
||||
value: published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
|
||||
notes: notes));
|
||||
}
|
||||
|
||||
if (dueDate.HasValue)
|
||||
{
|
||||
rules.Add(new NormalizedVersionRule(
|
||||
scheme: "kev.due-date",
|
||||
type: NormalizedVersionRuleTypes.LessThanOrEqual,
|
||||
max: dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
|
||||
maxInclusive: true,
|
||||
notes: notes));
|
||||
}
|
||||
|
||||
return rules.Count == 0
|
||||
? Array.Empty<NormalizedVersionRule>()
|
||||
: rules
|
||||
.OrderBy(static rule => rule.Scheme, StringComparer.Ordinal)
|
||||
.ThenBy(static rule => rule.Type, StringComparer.Ordinal)
|
||||
.ThenBy(static rule => rule.Value ?? rule.Max ?? string.Empty, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using Json.Schema;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
internal static class KevSchemaProvider
|
||||
{
|
||||
private const string ResourceName = "StellaOps.Concelier.Connector.Kev.Schemas.kev-catalog.schema.json";
|
||||
|
||||
private static readonly Lazy<JsonSchema> CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
public static JsonSchema Schema => CachedSchema.Value;
|
||||
|
||||
private static JsonSchema LoadSchema()
|
||||
{
|
||||
var assembly = typeof(KevSchemaProvider).GetTypeInfo().Assembly;
|
||||
using var stream = assembly.GetManifestResourceStream(ResourceName)
|
||||
?? throw new InvalidOperationException($"Embedded schema '{ResourceName}' was not found.");
|
||||
using var reader = new StreamReader(stream);
|
||||
var schemaJson = reader.ReadToEnd();
|
||||
return JsonSchema.FromText(schemaJson);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev;
|
||||
|
||||
internal static class KevJobKinds
|
||||
{
|
||||
public const string Fetch = "source:kev:fetch";
|
||||
public const string Parse = "source:kev:parse";
|
||||
public const string Map = "source:kev:map";
|
||||
}
|
||||
|
||||
internal sealed class KevFetchJob : IJob
|
||||
{
|
||||
private readonly KevConnector _connector;
|
||||
|
||||
public KevFetchJob(KevConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KevParseJob : IJob
|
||||
{
|
||||
private readonly KevConnector _connector;
|
||||
|
||||
public KevParseJob(KevConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KevMapJob : IJob
|
||||
{
|
||||
private readonly KevConnector _connector;
|
||||
|
||||
public KevMapJob(KevConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.Json;
|
||||
using StellaOps.Concelier.Connector.Kev.Configuration;
|
||||
using StellaOps.Concelier.Connector.Kev.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev;
|
||||
|
||||
public sealed class KevConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private const string SchemaVersion = "kev.catalog.v1";
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly KevOptions _options;
|
||||
private readonly IJsonSchemaValidator _schemaValidator;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<KevConnector> _logger;
|
||||
private readonly KevDiagnostics _diagnostics;
|
||||
|
||||
public KevConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<KevOptions> options,
|
||||
IJsonSchemaValidator schemaValidator,
|
||||
KevDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<KevConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => KevConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
try
|
||||
{
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, _options.FeedUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var request = new SourceFetchRequest(
|
||||
KevOptions.HttpClientName,
|
||||
SourceName,
|
||||
_options.FeedUri)
|
||||
{
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["kev.cursor.catalogVersion"] = cursor.CatalogVersion ?? string.Empty,
|
||||
["kev.cursor.catalogReleased"] = cursor.CatalogReleased?.ToString("O") ?? string.Empty,
|
||||
},
|
||||
ETag = existing?.Etag,
|
||||
LastModified = existing?.LastModified,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
AcceptHeaders = new[] { "application/json", "text/json" },
|
||||
};
|
||||
|
||||
_diagnostics.FetchAttempt();
|
||||
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
_logger.LogInformation(
|
||||
"KEV catalog not modified (catalogVersion={CatalogVersion}, etag={Etag})",
|
||||
cursor.CatalogVersion ?? "(unknown)",
|
||||
existing?.Etag ?? "(none)");
|
||||
await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), "KEV feed returned no content.", cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
_diagnostics.FetchSuccess();
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var pendingDocumentsBefore = pendingDocuments.Count;
|
||||
var pendingMappingsBefore = pendingMappings.Count;
|
||||
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
var document = result.Document;
|
||||
var lastModified = document.LastModified?.ToUniversalTime().ToString("O") ?? "(unknown)";
|
||||
_logger.LogInformation(
|
||||
"Fetched KEV catalog document {DocumentId} (etag={Etag}, lastModified={LastModified}) pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter}",
|
||||
document.Id,
|
||||
document.Etag ?? "(none)",
|
||||
lastModified,
|
||||
pendingDocumentsBefore,
|
||||
pendingDocuments.Count,
|
||||
pendingMappingsBefore,
|
||||
pendingMappings.Count);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "KEV fetch failed for {Uri}", _options.FeedUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var latestCatalogVersion = cursor.CatalogVersion;
|
||||
var latestCatalogReleased = cursor.CatalogReleased;
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_diagnostics.ParseFailure("missingPayload", cursor.CatalogVersion);
|
||||
_logger.LogWarning("KEV document {DocumentId} missing GridFS payload", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("download", cursor.CatalogVersion);
|
||||
_logger.LogError(ex, "KEV parse failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
KevCatalogDto? catalog = null;
|
||||
string? catalogVersion = null;
|
||||
try
|
||||
{
|
||||
using var jsonDocument = JsonDocument.Parse(rawBytes);
|
||||
catalogVersion = TryGetCatalogVersion(jsonDocument.RootElement);
|
||||
_schemaValidator.Validate(jsonDocument, KevSchemaProvider.Schema, document.Uri);
|
||||
catalog = jsonDocument.RootElement.Deserialize<KevCatalogDto>(SerializerOptions);
|
||||
}
|
||||
catch (JsonSchemaValidationException ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("schema", catalogVersion);
|
||||
_logger.LogWarning(ex, "KEV schema validation failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("invalidJson", catalogVersion);
|
||||
_logger.LogError(ex, "KEV JSON parsing failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure("deserialize", catalogVersion);
|
||||
_logger.LogError(ex, "KEV catalog deserialization failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (catalog is null)
|
||||
{
|
||||
_diagnostics.ParseFailure("emptyCatalog", catalogVersion);
|
||||
_logger.LogWarning("KEV catalog payload was empty for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var entryCount = catalog.Vulnerabilities?.Count ?? 0;
|
||||
var released = catalog.DateReleased?.ToUniversalTime();
|
||||
RecordCatalogAnomalies(catalog);
|
||||
|
||||
try
|
||||
{
|
||||
var payloadJson = JsonSerializer.Serialize(catalog, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(payloadJson);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Parsed KEV catalog document {DocumentId} (version={CatalogVersion}, released={Released}, entries={EntryCount})",
|
||||
document.Id,
|
||||
catalog.CatalogVersion ?? "(unknown)",
|
||||
released,
|
||||
entryCount);
|
||||
_diagnostics.CatalogParsed(catalog.CatalogVersion, entryCount);
|
||||
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
SchemaVersion,
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
|
||||
latestCatalogVersion = catalog.CatalogVersion ?? latestCatalogVersion;
|
||||
latestCatalogReleased = catalog.DateReleased ?? latestCatalogReleased;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "KEV DTO persistence failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithCatalogMetadata(latestCatalogVersion, latestCatalogReleased);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dtoRecord is null || document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
KevCatalogDto? catalog;
|
||||
try
|
||||
{
|
||||
var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
catalog = JsonSerializer.Deserialize<KevCatalogDto>(dtoJson, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "KEV mapping: failed to deserialize DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (catalog is null)
|
||||
{
|
||||
_logger.LogWarning("KEV mapping: DTO payload was empty for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var feedUri = TryParseUri(document.Uri) ?? _options.FeedUri;
|
||||
var advisories = KevMapper.Map(catalog, SourceName, feedUri, document.FetchedAt, dtoRecord.ValidatedAt);
|
||||
var entryCount = catalog.Vulnerabilities?.Count ?? 0;
|
||||
var mappedCount = advisories.Count;
|
||||
var skippedCount = Math.Max(0, entryCount - mappedCount);
|
||||
_logger.LogInformation(
|
||||
"Mapped {MappedCount}/{EntryCount} KEV advisories from catalog version {CatalogVersion} (skipped={SkippedCount})",
|
||||
mappedCount,
|
||||
entryCount,
|
||||
catalog.CatalogVersion ?? "(unknown)",
|
||||
skippedCount);
|
||||
_diagnostics.AdvisoriesMapped(catalog.CatalogVersion, mappedCount);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<KevCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? KevCursor.Empty : KevCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(KevCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken);
|
||||
}
|
||||
|
||||
private void RecordCatalogAnomalies(KevCatalogDto catalog)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(catalog);
|
||||
|
||||
var version = catalog.CatalogVersion;
|
||||
var vulnerabilities = catalog.Vulnerabilities ?? Array.Empty<KevVulnerabilityDto>();
|
||||
|
||||
if (catalog.Count != vulnerabilities.Count)
|
||||
{
|
||||
_diagnostics.RecordAnomaly("countMismatch", version);
|
||||
}
|
||||
|
||||
foreach (var entry in vulnerabilities)
|
||||
{
|
||||
if (entry is null)
|
||||
{
|
||||
_diagnostics.RecordAnomaly("nullEntry", version);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(entry.CveId))
|
||||
{
|
||||
_diagnostics.RecordAnomaly("missingCveId", version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string? TryGetCatalogVersion(JsonElement root)
|
||||
{
|
||||
if (root.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("catalogVersion", out var versionElement) && versionElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return versionElement.GetString();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Uri? TryParseUri(string? value)
|
||||
=> Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev;
|
||||
|
||||
public sealed class KevConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "kev";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<KevConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Kev.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev;
|
||||
|
||||
public sealed class KevDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:kev";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddKevConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<KevFetchJob>();
|
||||
services.AddTransient<KevParseJob>();
|
||||
services.AddTransient<KevMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, KevJobKinds.Fetch, typeof(KevFetchJob));
|
||||
EnsureJob(options, KevJobKinds.Parse, typeof(KevParseJob));
|
||||
EnsureJob(options, KevJobKinds.Map, typeof(KevMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Kev.Configuration;
|
||||
using StellaOps.Concelier.Connector.Kev.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Kev;
|
||||
|
||||
public static class KevServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddKevConnector(this IServiceCollection services, Action<KevOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<KevOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(KevOptions.HttpClientName, (provider, clientOptions) =>
|
||||
{
|
||||
var opts = provider.GetRequiredService<IOptions<KevOptions>>().Value;
|
||||
clientOptions.BaseAddress = opts.FeedUri;
|
||||
clientOptions.Timeout = opts.RequestTimeout;
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.Kev/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(opts.FeedUri.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
});
|
||||
|
||||
services.TryAddSingleton<KevDiagnostics>();
|
||||
services.AddTransient<KevConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "CISA Known Exploited Vulnerabilities catalog",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"catalogVersion",
|
||||
"dateReleased",
|
||||
"count",
|
||||
"vulnerabilities"
|
||||
],
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"catalogVersion": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"dateReleased": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"vulnerabilities": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cveID",
|
||||
"vendorProject",
|
||||
"product"
|
||||
],
|
||||
"properties": {
|
||||
"cveID": {
|
||||
"type": "string",
|
||||
"pattern": "^CVE-\\d{4}-\\d{4,}$"
|
||||
},
|
||||
"vendorProject": {
|
||||
"type": "string"
|
||||
},
|
||||
"product": {
|
||||
"type": "string"
|
||||
},
|
||||
"vulnerabilityName": {
|
||||
"type": "string"
|
||||
},
|
||||
"dateAdded": {
|
||||
"type": "string"
|
||||
},
|
||||
"shortDescription": {
|
||||
"type": "string"
|
||||
},
|
||||
"requiredAction": {
|
||||
"type": "string"
|
||||
},
|
||||
"dueDate": {
|
||||
"type": "string"
|
||||
},
|
||||
"knownRansomwareCampaignUse": {
|
||||
"type": "string"
|
||||
},
|
||||
"notes": {
|
||||
"type": "string"
|
||||
},
|
||||
"cwes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
||||
<_Parameter1>StellaOps.Concelier.Connector.Kev.Tests</_Parameter1>
|
||||
</AssemblyAttribute>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Schemas\kev-catalog.schema.json" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,12 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Review KEV JSON schema & cadence|BE-Conn-KEV|Research|**DONE** – Feed defaults lock to the public JSON catalog; AGENTS notes call out daily cadence and allowlist requirements.|
|
||||
|Fetch & cursor implementation|BE-Conn-KEV|Source.Common, Storage.Mongo|**DONE** – SourceFetchService drives ETag/Last-Modified aware fetches with SourceState cursor tracking documents + catalog metadata.|
|
||||
|DTO/parser implementation|BE-Conn-KEV|Source.Common|**DONE** – `KevCatalogDto`/`KevVulnerabilityDto` deserialize payloads with logging for catalog version/releases before DTO persistence.|
|
||||
|Canonical mapping & range primitives|BE-Conn-KEV|Models|**DONE** – Mapper produces vendor RangePrimitives (due dates, CWE list, ransomware flag, catalog metadata) and deduplicated references.|
|
||||
|Deterministic fixtures/tests|QA|Testing|**DONE** – End-to-end fetch→parse→map test with canned catalog + snapshot (`UPDATE_KEV_FIXTURES=1`) guards determinism.|
|
||||
|Telemetry & docs|DevEx|Docs|**DONE** – Connector emits structured logs + meters for catalog entries/advisories and AGENTS docs cover cadence/allowlist guidance.|
|
||||
|Schema validation & anomaly surfacing|BE-Conn-KEV, QA|Source.Common|**DONE (2025-10-12)** – Wired `IJsonSchemaValidator` + embedded schema, added failure reasons (`schema`, `download`, `invalidJson`, etc.), anomaly counters (`missingCveId`, `countMismatch`, `nullEntry`), and kept `dotnet test src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Kev.Tests` passing.|
|
||||
|Metrics export wiring|DevOps, DevEx|Observability|**DONE (2025-10-12)** – Added `kev.fetch.*` counters, parse failure/anomaly tags, refreshed ops runbook + Grafana dashboard (`docs/ops/concelier-cve-kev-grafana-dashboard.json`) with PromQL guidance.|
|
||||
|FEEDCONN-KEV-02-003 Normalized versions propagation|BE-Conn-KEV|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Validated catalog/date/due normalized rules emission + ordering; fixtures assert rule set and `dotnet test src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Kev.Tests` remains green.|
|
||||
Reference in New Issue
Block a user