Rename Concelier Source modules to Connector

This commit is contained in:
2025-10-18 20:11:18 +03:00
parent 0137856fdb
commit 6524626230
789 changed files with 1489 additions and 1489 deletions

View File

@@ -0,0 +1,26 @@
# AGENTS
## Role
Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories, including CVSS/CWE/CPE as aliases/references.
## Scope
- Windowed fetch by modified range (6-12h default) with pagination; respect rate limits.
- Parse NVD JSON; validate against schema; extract CVSS v3/v4 metrics, CWE IDs, configurations.cpeMatch.
- Map to Advisory: primary id='CVE-YYYY-NNNN'; references; AffectedPackage entries for CPE (type=cpe) and optional vendor tags.
- Optional change-history capture: store previous payload hashes and diff summaries for auditing modified CVEs.
- Watermark: last successful modified_end; handle partial windows with overlap to avoid misses.
## Participants
- Merge engine reconciles NVD with PSIRT/OVAL (NVD yields to OVAL for OS packages).
- KEV connector may flag some CVEs; NVD severity is preserved but not overridden by KEV.
- Exporters consume canonical advisories.
## Interfaces & contracts
- Job kinds: nvd:fetch, nvd:parse, nvd:map.
- Input params: windowHours, since, until; safe defaults in ConcelierOptions.
- Output: raw documents, sanitized DTOs, mapped advisories + provenance (document, parser).
## In/Out of scope
In: registry-level data, references, generic CPEs.
Out: authoritative distro package ranges; vendor patch states.
## Observability & security expectations
- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms tagged `concelier.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits.
## Tests
- Author and review coverage in `../StellaOps.Concelier.Connector.Nvd.Tests`.
- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`.
- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios.

View File

@@ -0,0 +1,57 @@
namespace StellaOps.Concelier.Connector.Nvd.Configuration;
public sealed class NvdOptions
{
/// <summary>
/// Name of the HttpClient registered for NVD fetches.
/// </summary>
public const string HttpClientName = "nvd";
/// <summary>
/// Base API endpoint for CVE feed queries.
/// </summary>
public Uri BaseEndpoint { get; set; } = new("https://services.nvd.nist.gov/rest/json/cves/2.0");
/// <summary>
/// Duration of each modified window fetch.
/// </summary>
public TimeSpan WindowSize { get; set; } = TimeSpan.FromHours(4);
/// <summary>
/// Overlap added when advancing the sliding window to cover upstream delays.
/// </summary>
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum look-back period used when the connector first starts or state is empty.
/// </summary>
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(7);
public void Validate()
{
if (BaseEndpoint is null)
{
throw new InvalidOperationException("NVD base endpoint must be configured.");
}
if (!BaseEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("NVD base endpoint must be an absolute URI.");
}
if (WindowSize <= TimeSpan.Zero)
{
throw new InvalidOperationException("Window size must be positive.");
}
if (WindowOverlap < TimeSpan.Zero || WindowOverlap >= WindowSize)
{
throw new InvalidOperationException("Window overlap must be non-negative and less than the window size.");
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill duration must be positive.");
}
}
}

View File

@@ -0,0 +1,64 @@
using System.Linq;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.Nvd.Internal;
internal sealed record NvdCursor(
TimeWindowCursorState Window,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static NvdCursor Empty { get; } = new(TimeWindowCursorState.Empty, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument();
Window.WriteTo(document);
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString()));
return document;
}
public static NvdCursor FromBsonDocument(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var window = TimeWindowCursorState.FromBsonDocument(document);
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new NvdCursor(window, pendingDocuments, pendingMappings);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.AsString, out var guid))
{
results.Add(guid);
}
}
return results;
}
public NvdCursor WithWindow(TimeWindow window)
=> this with { Window = Window.WithWindow(window) };
public NvdCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public NvdCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
}

View File

@@ -0,0 +1,76 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Nvd.Internal;
public sealed class NvdDiagnostics : IDisposable
{
public const string MeterName = "StellaOps.Concelier.Connector.Nvd";
public const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchAttempts;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
public NvdDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchAttempts = _meter.CreateCounter<long>(
name: "nvd.fetch.attempts",
unit: "operations",
description: "Number of NVD fetch operations attempted, including paginated windows.");
_fetchDocuments = _meter.CreateCounter<long>(
name: "nvd.fetch.documents",
unit: "documents",
description: "Count of NVD documents fetched and persisted.");
_fetchFailures = _meter.CreateCounter<long>(
name: "nvd.fetch.failures",
unit: "operations",
description: "Count of NVD fetch attempts that resulted in an error or missing document.");
_fetchUnchanged = _meter.CreateCounter<long>(
name: "nvd.fetch.unchanged",
unit: "operations",
description: "Count of NVD fetch attempts returning 304 Not Modified.");
_parseSuccess = _meter.CreateCounter<long>(
name: "nvd.parse.success",
unit: "documents",
description: "Count of NVD documents successfully validated and converted into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "nvd.parse.failures",
unit: "documents",
description: "Count of NVD documents that failed parsing due to missing content or read errors.");
_parseQuarantine = _meter.CreateCounter<long>(
name: "nvd.parse.quarantine",
unit: "documents",
description: "Count of NVD documents quarantined due to schema validation failures.");
_mapSuccess = _meter.CreateCounter<long>(
name: "nvd.map.success",
unit: "advisories",
description: "Count of canonical advisories produced by NVD mapping.");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void ParseQuarantine() => _parseQuarantine.Add(1);
public void MapSuccess(long count = 1) => _mapSuccess.Add(count);
public Meter Meter => _meter;
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,774 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.Json;
using NuGet.Versioning;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Normalization.Identifiers;
using StellaOps.Concelier.Normalization.Cvss;
using StellaOps.Concelier.Normalization.Text;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Nvd.Internal;
internal static class NvdMapper
{
public static IReadOnlyList<Advisory> Map(JsonDocument document, DocumentRecord sourceDocument, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(sourceDocument);
if (!document.RootElement.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array)
{
return Array.Empty<Advisory>();
}
var advisories = new List<Advisory>(vulnerabilities.GetArrayLength());
var index = 0;
foreach (var vulnerability in vulnerabilities.EnumerateArray())
{
if (!vulnerability.TryGetProperty("cve", out var cve) || cve.ValueKind != JsonValueKind.Object)
{
index++;
continue;
}
if (!cve.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
index++;
continue;
}
var cveId = idElement.GetString();
var advisoryKey = string.IsNullOrWhiteSpace(cveId)
? $"nvd:{sourceDocument.Id:N}:{index}"
: cveId;
var published = TryGetDateTime(cve, "published");
var modified = TryGetDateTime(cve, "lastModified");
var description = GetNormalizedDescription(cve);
var weaknessMetadata = GetWeaknessMetadata(cve);
var references = GetReferences(cve, sourceDocument, recordedAt, weaknessMetadata);
var affectedPackages = GetAffectedPackages(cve, cveId, sourceDocument, recordedAt);
var cvssMetrics = GetCvssMetrics(cve, sourceDocument, recordedAt, out var severity);
var weaknesses = BuildWeaknesses(weaknessMetadata, recordedAt);
var canonicalMetricId = cvssMetrics.Count > 0
? $"{cvssMetrics[0].Version}|{cvssMetrics[0].Vector}"
: null;
var provenance = new[]
{
new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"document",
sourceDocument.Uri,
sourceDocument.FetchedAt,
new[] { ProvenanceFieldMasks.Advisory }),
new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"mapping",
string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory }),
};
var title = string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId;
var aliasCandidates = new List<string>(capacity: 2);
if (!string.IsNullOrWhiteSpace(cveId))
{
aliasCandidates.Add(cveId);
}
aliasCandidates.Add(advisoryKey);
var advisory = new Advisory(
advisoryKey: advisoryKey,
title: title,
summary: string.IsNullOrEmpty(description.Text) ? null : description.Text,
language: description.Language,
published: published,
modified: modified,
severity: severity,
exploitKnown: false,
aliases: aliasCandidates,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: cvssMetrics,
provenance: provenance,
description: string.IsNullOrEmpty(description.Text) ? null : description.Text,
cwes: weaknesses,
canonicalMetricId: canonicalMetricId);
advisories.Add(advisory);
index++;
}
return advisories;
}
private static NormalizedDescription GetNormalizedDescription(JsonElement cve)
{
var candidates = new List<LocalizedText>();
if (cve.TryGetProperty("descriptions", out var descriptions) && descriptions.ValueKind == JsonValueKind.Array)
{
foreach (var item in descriptions.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Object)
{
continue;
}
var text = item.TryGetProperty("value", out var valueElement) && valueElement.ValueKind == JsonValueKind.String
? valueElement.GetString()
: null;
var lang = item.TryGetProperty("lang", out var langElement) && langElement.ValueKind == JsonValueKind.String
? langElement.GetString()
: null;
if (!string.IsNullOrWhiteSpace(text))
{
candidates.Add(new LocalizedText(text, lang));
}
}
}
return DescriptionNormalizer.Normalize(candidates);
}
private static DateTimeOffset? TryGetDateTime(JsonElement element, string propertyName)
{
if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String)
{
return null;
}
return DateTimeOffset.TryParse(property.GetString(), out var parsed) ? parsed : null;
}
private static IReadOnlyList<AdvisoryReference> GetReferences(
JsonElement cve,
DocumentRecord document,
DateTimeOffset recordedAt,
IReadOnlyList<WeaknessMetadata> weaknesses)
{
var references = new List<AdvisoryReference>();
if (!cve.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array)
{
AppendWeaknessReferences(references, weaknesses, recordedAt);
return references;
}
foreach (var reference in referencesElement.EnumerateArray())
{
if (!reference.TryGetProperty("url", out var urlElement) || urlElement.ValueKind != JsonValueKind.String)
{
continue;
}
var url = urlElement.GetString();
if (string.IsNullOrWhiteSpace(url) || !Validation.LooksLikeHttpUrl(url))
{
continue;
}
var sourceTag = reference.TryGetProperty("source", out var sourceElement) ? sourceElement.GetString() : null;
string? kind = null;
if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array)
{
kind = tagsElement.EnumerateArray().Select(static t => t.GetString()).FirstOrDefault(static tag => !string.IsNullOrWhiteSpace(tag))?.ToLowerInvariant();
}
references.Add(new AdvisoryReference(
url: url,
kind: kind,
sourceTag: sourceTag,
summary: null,
provenance: new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"reference",
url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
AppendWeaknessReferences(references, weaknesses, recordedAt);
return references;
}
private static IReadOnlyList<WeaknessMetadata> GetWeaknessMetadata(JsonElement cve)
{
if (!cve.TryGetProperty("weaknesses", out var weaknesses) || weaknesses.ValueKind != JsonValueKind.Array)
{
return Array.Empty<WeaknessMetadata>();
}
var list = new List<WeaknessMetadata>(weaknesses.GetArrayLength());
foreach (var weakness in weaknesses.EnumerateArray())
{
if (!weakness.TryGetProperty("description", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array)
{
continue;
}
string? cweId = null;
string? name = null;
foreach (var description in descriptions.EnumerateArray())
{
if (description.ValueKind != JsonValueKind.Object)
{
continue;
}
if (!description.TryGetProperty("value", out var valueElement) || valueElement.ValueKind != JsonValueKind.String)
{
continue;
}
var value = valueElement.GetString();
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
var trimmed = value.Trim();
if (trimmed.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase))
{
cweId ??= trimmed.ToUpperInvariant();
}
else
{
name ??= trimmed;
}
}
if (string.IsNullOrWhiteSpace(cweId))
{
continue;
}
list.Add(new WeaknessMetadata(cweId, name));
}
return list.Count == 0 ? Array.Empty<WeaknessMetadata>() : list;
}
private static IReadOnlyList<AdvisoryWeakness> BuildWeaknesses(IReadOnlyList<WeaknessMetadata> metadata, DateTimeOffset recordedAt)
{
if (metadata.Count == 0)
{
return Array.Empty<AdvisoryWeakness>();
}
var list = new List<AdvisoryWeakness>(metadata.Count);
foreach (var entry in metadata)
{
var provenance = new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"weakness",
entry.CweId,
recordedAt,
new[] { ProvenanceFieldMasks.Weaknesses });
var provenanceArray = ImmutableArray.Create(provenance);
list.Add(new AdvisoryWeakness(
taxonomy: "cwe",
identifier: entry.CweId,
name: entry.Name,
uri: BuildCweUrl(entry.CweId),
provenance: provenanceArray));
}
return list;
}
private static void AppendWeaknessReferences(
List<AdvisoryReference> references,
IReadOnlyList<WeaknessMetadata> weaknesses,
DateTimeOffset recordedAt)
{
if (weaknesses.Count == 0)
{
return;
}
var existing = new HashSet<string>(references.Select(reference => reference.Url), StringComparer.OrdinalIgnoreCase);
foreach (var weakness in weaknesses)
{
var url = BuildCweUrl(weakness.CweId);
if (url is null || existing.Contains(url))
{
continue;
}
var provenance = new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"reference",
url,
recordedAt,
new[] { ProvenanceFieldMasks.References });
references.Add(new AdvisoryReference(url, "weakness", weakness.CweId, weakness.Name, provenance));
existing.Add(url);
}
}
private static IReadOnlyList<AffectedPackage> GetAffectedPackages(JsonElement cve, string? cveId, DocumentRecord document, DateTimeOffset recordedAt)
{
var packages = new Dictionary<string, PackageAccumulator>(StringComparer.Ordinal);
if (!cve.TryGetProperty("configurations", out var configurations) || configurations.ValueKind != JsonValueKind.Object)
{
return Array.Empty<AffectedPackage>();
}
if (!configurations.TryGetProperty("nodes", out var nodes) || nodes.ValueKind != JsonValueKind.Array)
{
return Array.Empty<AffectedPackage>();
}
foreach (var node in nodes.EnumerateArray())
{
if (!node.TryGetProperty("cpeMatch", out var matches) || matches.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var match in matches.EnumerateArray())
{
if (match.TryGetProperty("vulnerable", out var vulnerableElement) && vulnerableElement.ValueKind == JsonValueKind.False)
{
continue;
}
if (!match.TryGetProperty("criteria", out var criteriaElement) || criteriaElement.ValueKind != JsonValueKind.String)
{
continue;
}
var criteria = criteriaElement.GetString();
if (string.IsNullOrWhiteSpace(criteria))
{
continue;
}
var identifier = IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe) && !string.IsNullOrWhiteSpace(normalizedCpe)
? normalizedCpe
: criteria.Trim();
var provenance = new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"cpe",
document.Uri,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
if (!packages.TryGetValue(identifier, out var accumulator))
{
accumulator = new PackageAccumulator();
packages[identifier] = accumulator;
}
var range = BuildVersionRange(match, criteria, provenance);
if (range is not null)
{
accumulator.Ranges.Add(range);
}
accumulator.Provenance.Add(provenance);
}
}
if (packages.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
return packages
.OrderBy(static kvp => kvp.Key, StringComparer.Ordinal)
.Select(kvp =>
{
var ranges = kvp.Value.Ranges.Count == 0
? Array.Empty<AffectedVersionRange>()
: kvp.Value.Ranges
.OrderBy(static range => range, AffectedVersionRangeComparer.Instance)
.ToArray();
var provenance = kvp.Value.Provenance
.OrderBy(static p => p.Source, StringComparer.Ordinal)
.ThenBy(static p => p.Kind, StringComparer.Ordinal)
.ThenBy(static p => p.Value, StringComparer.Ordinal)
.ThenBy(static p => p.RecordedAt.UtcDateTime)
.ToArray();
var normalizedNote = string.IsNullOrWhiteSpace(cveId)
? $"nvd:{document.Id:N}"
: $"nvd:{cveId}";
var normalizedVersions = new List<NormalizedVersionRule>(ranges.Length);
foreach (var range in ranges)
{
var rule = range.ToNormalizedVersionRule(normalizedNote);
if (rule is not null)
{
normalizedVersions.Add(rule);
}
}
return new AffectedPackage(
type: AffectedPackageTypes.Cpe,
identifier: kvp.Key,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: provenance,
normalizedVersions: normalizedVersions.Count == 0
? Array.Empty<NormalizedVersionRule>()
: normalizedVersions.ToArray());
})
.ToArray();
}
private static IReadOnlyList<CvssMetric> GetCvssMetrics(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt, out string? severity)
{
severity = null;
if (!cve.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Object)
{
return Array.Empty<CvssMetric>();
}
var sources = new[] { "cvssMetricV31", "cvssMetricV30", "cvssMetricV2" };
foreach (var source in sources)
{
if (!metrics.TryGetProperty(source, out var array) || array.ValueKind != JsonValueKind.Array)
{
continue;
}
var list = new List<CvssMetric>();
foreach (var item in array.EnumerateArray())
{
if (!item.TryGetProperty("cvssData", out var data) || data.ValueKind != JsonValueKind.Object)
{
continue;
}
if (!data.TryGetProperty("vectorString", out var vectorElement) || vectorElement.ValueKind != JsonValueKind.String)
{
continue;
}
if (!data.TryGetProperty("baseScore", out var scoreElement) || scoreElement.ValueKind != JsonValueKind.Number)
{
continue;
}
if (!data.TryGetProperty("baseSeverity", out var severityElement) || severityElement.ValueKind != JsonValueKind.String)
{
continue;
}
var vector = vectorElement.GetString() ?? string.Empty;
var baseScore = scoreElement.GetDouble();
var baseSeverity = severityElement.GetString();
var versionToken = source switch
{
"cvssMetricV30" => "3.0",
"cvssMetricV31" => "3.1",
_ => "2.0",
};
if (!CvssMetricNormalizer.TryNormalize(versionToken, vector, baseScore, baseSeverity, out var normalized))
{
continue;
}
severity ??= normalized.BaseSeverity;
list.Add(normalized.ToModel(new AdvisoryProvenance(
NvdConnectorPlugin.SourceName,
"cvss",
normalized.Vector,
recordedAt,
new[] { ProvenanceFieldMasks.CvssMetrics })));
}
if (list.Count > 0)
{
return list;
}
}
return Array.Empty<CvssMetric>();
}
private static AffectedVersionRange? BuildVersionRange(JsonElement match, string criteria, AdvisoryProvenance provenance)
{
static string? ReadString(JsonElement parent, string property)
{
if (!parent.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.String)
{
return null;
}
var text = value.GetString();
return string.IsNullOrWhiteSpace(text) ? null : text.Trim();
}
var version = ReadString(match, "version");
if (string.Equals(version, "*", StringComparison.Ordinal))
{
version = null;
}
version ??= TryExtractVersionFromCriteria(criteria);
var versionStartIncluding = ReadString(match, "versionStartIncluding");
var versionStartExcluding = ReadString(match, "versionStartExcluding");
var versionEndIncluding = ReadString(match, "versionEndIncluding");
var versionEndExcluding = ReadString(match, "versionEndExcluding");
var vendorExtensions = new Dictionary<string, string>(StringComparer.Ordinal);
if (versionStartIncluding is not null)
{
vendorExtensions["versionStartIncluding"] = versionStartIncluding;
}
if (versionStartExcluding is not null)
{
vendorExtensions["versionStartExcluding"] = versionStartExcluding;
}
if (versionEndIncluding is not null)
{
vendorExtensions["versionEndIncluding"] = versionEndIncluding;
}
if (versionEndExcluding is not null)
{
vendorExtensions["versionEndExcluding"] = versionEndExcluding;
}
if (version is not null)
{
vendorExtensions["version"] = version;
}
string? introduced = null;
string? fixedVersion = null;
string? lastAffected = null;
string? exactVersion = null;
var expressionParts = new List<string>();
var introducedInclusive = true;
var fixedInclusive = false;
var lastInclusive = true;
if (versionStartIncluding is not null)
{
introduced = versionStartIncluding;
introducedInclusive = true;
expressionParts.Add($">={versionStartIncluding}");
}
if (versionStartExcluding is not null)
{
if (introduced is null)
{
introduced = versionStartExcluding;
introducedInclusive = false;
}
expressionParts.Add($">{versionStartExcluding}");
}
if (versionEndExcluding is not null)
{
fixedVersion = versionEndExcluding;
fixedInclusive = false;
expressionParts.Add($"<{versionEndExcluding}");
}
if (versionEndIncluding is not null)
{
lastAffected = versionEndIncluding;
lastInclusive = true;
expressionParts.Add($"<={versionEndIncluding}");
}
if (version is not null)
{
introduced = version;
introducedInclusive = true;
lastAffected = version;
lastInclusive = true;
exactVersion = version;
expressionParts.Add($"=={version}");
}
if (introduced is null && fixedVersion is null && lastAffected is null && vendorExtensions.Count == 0)
{
return null;
}
var rangeExpression = expressionParts.Count > 0 ? string.Join(' ', expressionParts) : null;
IReadOnlyDictionary<string, string>? extensions = vendorExtensions.Count == 0 ? null : vendorExtensions;
SemVerPrimitive? semVerPrimitive = null;
if (TryBuildSemVerPrimitive(
introduced,
introducedInclusive,
fixedVersion,
fixedInclusive,
lastAffected,
lastInclusive,
exactVersion,
rangeExpression,
out var primitive))
{
semVerPrimitive = primitive;
}
var primitives = semVerPrimitive is null && extensions is null
? null
: new RangePrimitives(semVerPrimitive, null, null, extensions);
var provenanceValue = provenance.Value ?? criteria;
var rangeProvenance = new AdvisoryProvenance(
provenance.Source,
provenance.Kind,
provenanceValue,
provenance.RecordedAt,
new[] { ProvenanceFieldMasks.VersionRanges });
return new AffectedVersionRange(
rangeKind: "cpe",
introducedVersion: introduced,
fixedVersion: fixedVersion,
lastAffectedVersion: lastAffected,
rangeExpression: rangeExpression,
provenance: rangeProvenance,
primitives);
}
private static bool TryBuildSemVerPrimitive(
string? introduced,
bool introducedInclusive,
string? fixedVersion,
bool fixedInclusive,
string? lastAffected,
bool lastInclusive,
string? exactVersion,
string? constraintExpression,
out SemVerPrimitive? primitive)
{
primitive = null;
if (!TryNormalizeSemVer(introduced, out var normalizedIntroduced)
|| !TryNormalizeSemVer(fixedVersion, out var normalizedFixed)
|| !TryNormalizeSemVer(lastAffected, out var normalizedLast)
|| !TryNormalizeSemVer(exactVersion, out var normalizedExact))
{
return false;
}
if (normalizedIntroduced is null && normalizedFixed is null && normalizedLast is null && normalizedExact is null)
{
return false;
}
primitive = new SemVerPrimitive(
Introduced: normalizedIntroduced,
IntroducedInclusive: normalizedIntroduced is null ? true : introducedInclusive,
Fixed: normalizedFixed,
FixedInclusive: normalizedFixed is null ? false : fixedInclusive,
LastAffected: normalizedLast,
LastAffectedInclusive: normalizedLast is null ? false : lastInclusive,
ConstraintExpression: constraintExpression,
ExactValue: normalizedExact);
return true;
}
private static bool TryNormalizeSemVer(string? value, out string? normalized)
{
normalized = null;
if (string.IsNullOrWhiteSpace(value))
{
return true;
}
var trimmed = value.Trim();
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 1)
{
trimmed = trimmed[1..];
}
if (!NuGetVersion.TryParse(trimmed, out var parsed))
{
return false;
}
normalized = parsed.ToNormalizedString();
return true;
}
private static string? BuildCweUrl(string cweId)
{
var dashIndex = cweId.IndexOf('-');
if (dashIndex < 0 || dashIndex == cweId.Length - 1)
{
return null;
}
var digits = new StringBuilder();
for (var i = dashIndex + 1; i < cweId.Length; i++)
{
var ch = cweId[i];
if (char.IsDigit(ch))
{
digits.Append(ch);
}
}
return digits.Length == 0 ? null : $"https://cwe.mitre.org/data/definitions/{digits}.html";
}
private static string? TryExtractVersionFromCriteria(string criteria)
{
if (string.IsNullOrWhiteSpace(criteria))
{
return null;
}
var segments = criteria.Split(':');
if (segments.Length < 6)
{
return null;
}
var version = segments[5];
if (string.IsNullOrWhiteSpace(version))
{
return null;
}
if (string.Equals(version, "*", StringComparison.Ordinal) || string.Equals(version, "-", StringComparison.Ordinal))
{
return null;
}
return version;
}
private readonly record struct WeaknessMetadata(string CweId, string? Name);
private sealed class PackageAccumulator
{
public List<AffectedVersionRange> Ranges { get; } = new();
public List<AdvisoryProvenance> Provenance { get; } = new();
}
}

View File

@@ -0,0 +1,25 @@
using System.IO;
using System.Reflection;
using System.Threading;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Nvd.Internal;
internal static class NvdSchemaProvider
{
private static readonly Lazy<JsonSchema> Cached = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication);
public static JsonSchema Schema => Cached.Value;
private static JsonSchema LoadSchema()
{
var assembly = typeof(NvdSchemaProvider).GetTypeInfo().Assembly;
const string resourceName = "StellaOps.Concelier.Connector.Nvd.Schemas.nvd-vulnerability.schema.json";
using var stream = assembly.GetManifestResourceStream(resourceName)
?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found.");
using var reader = new StreamReader(stream);
var schemaText = reader.ReadToEnd();
return JsonSchema.FromText(schemaText);
}
}

View File

@@ -0,0 +1,565 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Json;
using StellaOps.Concelier.Connector.Common.Cursors;
using StellaOps.Concelier.Connector.Nvd.Configuration;
using StellaOps.Concelier.Connector.Nvd.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Storage.Mongo.ChangeHistory;
using StellaOps.Plugin;
using Json.Schema;
namespace StellaOps.Concelier.Connector.Nvd;
public sealed class NvdConnector : IFeedConnector
{
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
private readonly IDtoStore _dtoStore;
private readonly IAdvisoryStore _advisoryStore;
private readonly IChangeHistoryStore _changeHistoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly IJsonSchemaValidator _schemaValidator;
private readonly NvdOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NvdConnector> _logger;
private readonly NvdDiagnostics _diagnostics;
private static readonly JsonSchema Schema = NvdSchemaProvider.Schema;
public NvdConnector(
SourceFetchService fetchService,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
IChangeHistoryStore changeHistoryStore,
ISourceStateRepository stateRepository,
IJsonSchemaValidator schemaValidator,
IOptions<NvdOptions> options,
NvdDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<NvdConnector> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_changeHistoryStore = changeHistoryStore ?? throw new ArgumentNullException(nameof(changeHistoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => NvdConnectorPlugin.SourceName;
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var windowOptions = new TimeWindowCursorOptions
{
WindowSize = _options.WindowSize,
Overlap = _options.WindowOverlap,
InitialBackfill = _options.InitialBackfill,
};
var window = TimeWindowCursorPlanner.GetNextWindow(now, cursor.Window, windowOptions);
var requestUri = BuildRequestUri(window);
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["windowStart"] = window.Start.ToString("O"),
["windowEnd"] = window.End.ToString("O"),
};
metadata["startIndex"] = "0";
try
{
_diagnostics.FetchAttempt();
var result = await _fetchService.FetchAsync(
new SourceFetchRequest(
NvdOptions.HttpClientName,
SourceName,
requestUri)
{
Metadata = metadata
},
cancellationToken).ConfigureAwait(false);
if (result.IsNotModified)
{
_diagnostics.FetchUnchanged();
_logger.LogDebug("NVD window {Start} - {End} returned 304", window.Start, window.End);
await UpdateCursorAsync(cursor.WithWindow(window), cancellationToken).ConfigureAwait(false);
return;
}
if (!result.IsSuccess || result.Document is null)
{
_diagnostics.FetchFailure();
return;
}
_diagnostics.FetchDocument();
var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments)
{
result.Document.Id
};
var additionalDocuments = await FetchAdditionalPagesAsync(
window,
metadata,
result.Document,
cancellationToken).ConfigureAwait(false);
foreach (var documentId in additionalDocuments)
{
pendingDocuments.Add(documentId);
}
var updated = cursor
.WithWindow(window)
.WithPendingDocuments(pendingDocuments)
.WithPendingMappings(cursor.PendingMappings);
await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.FetchFailure();
_logger.LogError(ex, "NVD fetch failed for {Uri}", requestUri);
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
throw;
}
}
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingDocuments.Count == 0)
{
return;
}
var remainingFetch = cursor.PendingDocuments.ToList();
var pendingMapping = cursor.PendingMappings.ToList();
foreach (var documentId in cursor.PendingDocuments)
{
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
_diagnostics.ParseFailure();
remainingFetch.Remove(documentId);
pendingMapping.Remove(documentId);
continue;
}
if (!document.GridFsId.HasValue)
{
_logger.LogWarning("Document {DocumentId} is missing GridFS content; skipping", documentId);
_diagnostics.ParseFailure();
remainingFetch.Remove(documentId);
pendingMapping.Remove(documentId);
continue;
}
var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
try
{
using var jsonDocument = JsonDocument.Parse(rawBytes);
try
{
_schemaValidator.Validate(jsonDocument, Schema, document.Uri);
}
catch (JsonSchemaValidationException ex)
{
_logger.LogWarning(ex, "NVD schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingFetch.Remove(documentId);
pendingMapping.Remove(documentId);
_diagnostics.ParseQuarantine();
continue;
}
var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement);
var payload = BsonDocument.Parse(sanitized);
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
document.Id,
SourceName,
"nvd.cve.v2",
payload,
_timeProvider.GetUtcNow());
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
_diagnostics.ParseSuccess();
remainingFetch.Remove(documentId);
if (!pendingMapping.Contains(documentId))
{
pendingMapping.Add(documentId);
}
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse NVD JSON payload for document {DocumentId} ({Uri})", document.Id, document.Uri);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingFetch.Remove(documentId);
pendingMapping.Remove(documentId);
_diagnostics.ParseFailure();
}
}
var updatedCursor = cursor
.WithPendingDocuments(remainingFetch)
.WithPendingMappings(pendingMapping);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
{
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
if (cursor.PendingMappings.Count == 0)
{
return;
}
var pendingMapping = cursor.PendingMappings.ToList();
var now = _timeProvider.GetUtcNow();
foreach (var documentId in cursor.PendingMappings)
{
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
if (dto is null || document is null)
{
pendingMapping.Remove(documentId);
continue;
}
var json = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
{
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
});
using var jsonDocument = JsonDocument.Parse(json);
var advisories = NvdMapper.Map(jsonDocument, document, now)
.GroupBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal)
.Select(static group => group.First())
.ToArray();
var mappedCount = 0L;
foreach (var advisory in advisories)
{
if (string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
{
_logger.LogWarning("Skipping advisory with missing key for document {DocumentId} ({Uri})", document.Id, document.Uri);
continue;
}
var previous = await _advisoryStore.FindAsync(advisory.AdvisoryKey, cancellationToken).ConfigureAwait(false);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
if (previous is not null)
{
await RecordChangeHistoryAsync(advisory, previous, document, now, cancellationToken).ConfigureAwait(false);
}
mappedCount++;
}
if (mappedCount > 0)
{
_diagnostics.MapSuccess(mappedCount);
}
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
pendingMapping.Remove(documentId);
}
var updatedCursor = cursor.WithPendingMappings(pendingMapping);
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
}
private async Task<IReadOnlyCollection<Guid>> FetchAdditionalPagesAsync(
TimeWindow window,
IReadOnlyDictionary<string, string> baseMetadata,
DocumentRecord firstDocument,
CancellationToken cancellationToken)
{
if (firstDocument.GridFsId is null)
{
return Array.Empty<Guid>();
}
byte[] rawBytes;
try
{
rawBytes = await _rawDocumentStorage.DownloadAsync(firstDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Unable to download NVD first page {DocumentId} to evaluate pagination", firstDocument.Id);
return Array.Empty<Guid>();
}
try
{
using var jsonDocument = JsonDocument.Parse(rawBytes);
var root = jsonDocument.RootElement;
if (!TryReadInt32(root, "totalResults", out var totalResults) || !TryReadInt32(root, "resultsPerPage", out var resultsPerPage))
{
return Array.Empty<Guid>();
}
if (resultsPerPage <= 0 || totalResults <= resultsPerPage)
{
return Array.Empty<Guid>();
}
var fetchedDocuments = new List<Guid>();
foreach (var startIndex in PaginationPlanner.EnumerateAdditionalPages(totalResults, resultsPerPage))
{
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var kvp in baseMetadata)
{
metadata[kvp.Key] = kvp.Value;
}
metadata["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture);
var request = new SourceFetchRequest(
NvdOptions.HttpClientName,
SourceName,
BuildRequestUri(window, startIndex))
{
Metadata = metadata
};
SourceFetchResult pageResult;
try
{
_diagnostics.FetchAttempt();
pageResult = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_diagnostics.FetchFailure();
_logger.LogError(ex, "NVD fetch failed for page starting at {StartIndex}", startIndex);
throw;
}
if (pageResult.IsNotModified)
{
_diagnostics.FetchUnchanged();
continue;
}
if (!pageResult.IsSuccess || pageResult.Document is null)
{
_diagnostics.FetchFailure();
_logger.LogWarning("NVD fetch for page starting at {StartIndex} returned status {Status}", startIndex, pageResult.StatusCode);
continue;
}
_diagnostics.FetchDocument();
fetchedDocuments.Add(pageResult.Document.Id);
}
return fetchedDocuments;
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse NVD first page {DocumentId} while determining pagination", firstDocument.Id);
return Array.Empty<Guid>();
}
}
private static bool TryReadInt32(JsonElement root, string propertyName, out int value)
{
value = 0;
if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.Number)
{
return false;
}
if (property.TryGetInt32(out var intValue))
{
value = intValue;
return true;
}
if (property.TryGetInt64(out var longValue))
{
if (longValue > int.MaxValue)
{
value = int.MaxValue;
return true;
}
value = (int)longValue;
return true;
}
return false;
}
private async Task RecordChangeHistoryAsync(
Advisory current,
Advisory previous,
DocumentRecord document,
DateTimeOffset capturedAt,
CancellationToken cancellationToken)
{
if (current.Equals(previous))
{
return;
}
var currentSnapshot = SnapshotSerializer.ToSnapshot(current);
var previousSnapshot = SnapshotSerializer.ToSnapshot(previous);
if (string.Equals(currentSnapshot, previousSnapshot, StringComparison.Ordinal))
{
return;
}
var changes = ComputeChanges(previousSnapshot, currentSnapshot);
if (changes.Count == 0)
{
return;
}
var documentHash = string.IsNullOrWhiteSpace(document.Sha256)
? ComputeHash(currentSnapshot)
: document.Sha256;
var record = new ChangeHistoryRecord(
Guid.NewGuid(),
SourceName,
current.AdvisoryKey,
document.Id,
documentHash,
ComputeHash(currentSnapshot),
ComputeHash(previousSnapshot),
currentSnapshot,
previousSnapshot,
changes,
capturedAt);
await _changeHistoryStore.AddAsync(record, cancellationToken).ConfigureAwait(false);
}
private static IReadOnlyList<ChangeHistoryFieldChange> ComputeChanges(string previousSnapshot, string currentSnapshot)
{
using var previousDocument = JsonDocument.Parse(previousSnapshot);
using var currentDocument = JsonDocument.Parse(currentSnapshot);
var previousRoot = previousDocument.RootElement;
var currentRoot = currentDocument.RootElement;
var fields = new HashSet<string>(StringComparer.Ordinal);
foreach (var property in previousRoot.EnumerateObject())
{
fields.Add(property.Name);
}
foreach (var property in currentRoot.EnumerateObject())
{
fields.Add(property.Name);
}
var changes = new List<ChangeHistoryFieldChange>();
foreach (var field in fields.OrderBy(static name => name, StringComparer.Ordinal))
{
var hasPrevious = previousRoot.TryGetProperty(field, out var previousValue);
var hasCurrent = currentRoot.TryGetProperty(field, out var currentValue);
if (!hasPrevious && hasCurrent)
{
changes.Add(new ChangeHistoryFieldChange(field, "Added", null, SerializeElement(currentValue)));
continue;
}
if (hasPrevious && !hasCurrent)
{
changes.Add(new ChangeHistoryFieldChange(field, "Removed", SerializeElement(previousValue), null));
continue;
}
if (hasPrevious && hasCurrent && !JsonElement.DeepEquals(previousValue, currentValue))
{
changes.Add(new ChangeHistoryFieldChange(field, "Modified", SerializeElement(previousValue), SerializeElement(currentValue)));
}
}
return changes;
}
private static string SerializeElement(JsonElement element)
=> JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false });
private static string ComputeHash(string snapshot)
{
var bytes = Encoding.UTF8.GetBytes(snapshot);
var hash = SHA256.HashData(bytes);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private async Task<NvdCursor> GetCursorAsync(CancellationToken cancellationToken)
{
var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
return NvdCursor.FromBsonDocument(record?.Cursor);
}
private async Task UpdateCursorAsync(NvdCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
}
private Uri BuildRequestUri(TimeWindow window, int startIndex = 0)
{
var builder = new UriBuilder(_options.BaseEndpoint);
var parameters = new Dictionary<string, string>
{
["lastModifiedStartDate"] = window.Start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["lastModifiedEndDate"] = window.End.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"),
["resultsPerPage"] = "2000",
};
if (startIndex > 0)
{
parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture);
}
builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}"));
return builder.Uri;
}
}

View File

@@ -0,0 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Nvd;
public sealed class NvdConnectorPlugin : IConnectorPlugin
{
public string Name => SourceName;
public static string SourceName => "nvd";
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<NvdConnector>(services);
}
}

View File

@@ -0,0 +1,35 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Nvd.Configuration;
using StellaOps.Concelier.Connector.Nvd.Internal;
namespace StellaOps.Concelier.Connector.Nvd;
public static class NvdServiceCollectionExtensions
{
public static IServiceCollection AddNvdConnector(this IServiceCollection services, Action<NvdOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<NvdOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(NvdOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<NvdOptions>>().Value;
clientOptions.BaseAddress = options.BaseEndpoint;
clientOptions.Timeout = TimeSpan.FromSeconds(30);
clientOptions.UserAgent = "StellaOps.Concelier.Nvd/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.AddSingleton<NvdDiagnostics>();
services.AddTransient<NvdConnector>();
return services;
}
}

View File

@@ -0,0 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Nvd.Tests")]
[assembly: InternalsVisibleTo("FixtureUpdater")]

View File

@@ -0,0 +1,115 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["vulnerabilities"],
"properties": {
"resultsPerPage": { "type": "integer", "minimum": 0 },
"startIndex": { "type": "integer", "minimum": 0 },
"totalResults": { "type": "integer", "minimum": 0 },
"vulnerabilities": {
"type": "array",
"items": {
"type": "object",
"required": ["cve"],
"properties": {
"cve": {
"type": "object",
"required": ["id", "published", "lastModified", "descriptions"],
"properties": {
"id": { "type": "string" },
"published": { "type": "string", "format": "date-time" },
"lastModified": { "type": "string", "format": "date-time" },
"vulnStatus": { "type": "string" },
"sourceIdentifier": { "type": "string" },
"descriptions": {
"type": "array",
"items": {
"type": "object",
"required": ["lang", "value"],
"properties": {
"lang": { "type": "string" },
"value": { "type": "string" }
}
}
},
"references": {
"type": "array",
"items": {
"type": "object",
"required": ["url"],
"properties": {
"url": { "type": "string", "format": "uri" },
"source": { "type": "string" },
"tags": {
"type": "array",
"items": { "type": "string" }
}
}
}
},
"metrics": {
"type": "object",
"properties": {
"cvssMetricV2": { "$ref": "#/definitions/cvssMetricArray" },
"cvssMetricV30": { "$ref": "#/definitions/cvssMetricArray" },
"cvssMetricV31": { "$ref": "#/definitions/cvssMetricArray" }
}
},
"configurations": {
"type": "object",
"properties": {
"nodes": {
"type": "array",
"items": {
"type": "object",
"properties": {
"cpeMatch": {
"type": "array",
"items": {
"type": "object",
"properties": {
"vulnerable": { "type": "boolean" },
"criteria": { "type": "string" }
},
"required": ["criteria"],
"additionalProperties": true
}
}
},
"additionalProperties": true
}
}
},
"additionalProperties": true
}
},
"additionalProperties": true
}
},
"additionalProperties": true
}
}
},
"additionalProperties": true,
"definitions": {
"cvssMetricArray": {
"type": "array",
"items": {
"type": "object",
"properties": {
"cvssData": {
"type": "object",
"required": ["vectorString", "baseScore", "baseSeverity"],
"properties": {
"vectorString": { "type": "string" },
"baseScore": { "type": "number" },
"baseSeverity": { "type": "string" }
},
"additionalProperties": true
}
},
"additionalProperties": true
}
}
}
}

View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="Schemas\nvd-vulnerability.schema.json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,18 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Fetch job with sliding modified windows|BE-Conn-Nvd|Source.Common|**DONE** windowed fetch implemented with overlap and raw doc persistence.|
|DTO schema + validation|BE-Conn-Nvd|Source.Common|**DONE** schema validator enforced before DTO persistence.|
|Mapper to canonical model|BE-Conn-Nvd|Models|**DONE** `NvdMapper` populates CVSS/CWE/CPE data.<br>2025-10-11 research trail: upcoming normalized rules must serialize as `[{"scheme":"semver","type":"range","min":"<floor>","minInclusive":true,"max":"<ceiling>","maxInclusive":false,"notes":"nvd:CVE-2025-XXXX"}]`; keep notes consistent with CVE IDs for provenance joins.|
|Watermark repo usage|BE-Conn-Nvd|Storage.Mongo|**DONE** cursor tracks windowStart/windowEnd and updates SourceState.|
|Integration test fixture isolation|QA|Storage.Mongo|**DONE** connector tests reset Mongo/time fixtures between runs to avoid cross-test bleed.|
|Tests: golden pages + resume|QA|Tests|**DONE** snapshot and resume coverage added across `NvdConnectorTests`.|
|Observability|BE-Conn-Nvd|Core|**DONE** `NvdDiagnostics` meter tracks attempts/documents/failures with collector tests.|
|Change history snapshotting|BE-Conn-Nvd|Storage.Mongo|DONE connector now records per-CVE snapshots with top-level diff metadata whenever canonical advisories change.|
|Pagination for windows over page limit|BE-Conn-Nvd|Source.Common|**DONE** additional page fetcher honors `startIndex`; covered by multipage tests.|
|Schema validation quarantine path|BE-Conn-Nvd|Storage.Mongo|**DONE** schema failures mark documents failed and metrics assert quarantine.|
|FEEDCONN-NVD-04-002 Conflict regression fixtures|BE-Conn-Nvd, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** Published `conflict-nvd.canonical.json` + mapper test; includes CVSS 3.1 + CWE reference and normalized CPE range feeding the conflict triple. Validation: `dotnet test src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj --filter NvdConflictFixtureTests`.|
|FEEDCONN-NVD-02-004 NVD CVSS & CWE precedence payloads|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** CVSS metrics now carry provenance masks, CWE weaknesses emit normalized references, and fixtures cover the additional precedence data.|
|FEEDCONN-NVD-02-005 NVD merge/export parity regression|BE-Conn-Nvd, BE-Merge|Merge `FEEDMERGE-ENGINE-04-003`|**DONE (2025-10-12)** Canonical merge parity fixtures captured, regression test validates credit/reference union, and exporter snapshot check guarantees parity through JSON exports.|
|FEEDCONN-NVD-02-002 Normalized versions rollout|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** SemVer primitives + normalized rules emitting for parseable ranges, fixtures/tests refreshed, coordination pinged via FEEDMERGE-COORD-02-900.|
|FEEDCONN-NVD-04-003 Description/CWE/metric parity rollout|BE-Conn-Nvd|Models, Core|**DONE (2025-10-15)** Mapper now surfaces normalized description text, CWE weaknesses, and canonical CVSS metric id. Snapshots (`conflict-nvd.canonical.json`) refreshed and completion relayed to Merge coordination.|