Rename Concelier Source modules to Connector
This commit is contained in:
		
							
								
								
									
										26
									
								
								src/StellaOps.Concelier.Connector.Nvd/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										26
									
								
								src/StellaOps.Concelier.Connector.Nvd/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,26 @@ | ||||
| # AGENTS | ||||
| ## Role | ||||
| Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories, including CVSS/CWE/CPE as aliases/references. | ||||
| ## Scope | ||||
| - Windowed fetch by modified range (6-12h default) with pagination; respect rate limits. | ||||
| - Parse NVD JSON; validate against schema; extract CVSS v3/v4 metrics, CWE IDs, configurations.cpeMatch. | ||||
| - Map to Advisory: primary id='CVE-YYYY-NNNN'; references; AffectedPackage entries for CPE (type=cpe) and optional vendor tags. | ||||
| - Optional change-history capture: store previous payload hashes and diff summaries for auditing modified CVEs. | ||||
| - Watermark: last successful modified_end; handle partial windows with overlap to avoid misses. | ||||
| ## Participants | ||||
| - Merge engine reconciles NVD with PSIRT/OVAL (NVD yields to OVAL for OS packages). | ||||
| - KEV connector may flag some CVEs; NVD severity is preserved but not overridden by KEV. | ||||
| - Exporters consume canonical advisories. | ||||
| ## Interfaces & contracts | ||||
| - Job kinds: nvd:fetch, nvd:parse, nvd:map. | ||||
| - Input params: windowHours, since, until; safe defaults in ConcelierOptions. | ||||
| - Output: raw documents, sanitized DTOs, mapped advisories + provenance (document, parser). | ||||
| ## In/Out of scope | ||||
| In: registry-level data, references, generic CPEs. | ||||
| Out: authoritative distro package ranges; vendor patch states. | ||||
| ## Observability & security expectations | ||||
| - Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms tagged `concelier.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits. | ||||
| ## Tests | ||||
| - Author and review coverage in `../StellaOps.Concelier.Connector.Nvd.Tests`. | ||||
| - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. | ||||
| - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. | ||||
| @@ -0,0 +1,57 @@ | ||||
| namespace StellaOps.Concelier.Connector.Nvd.Configuration; | ||||
|  | ||||
| public sealed class NvdOptions | ||||
| { | ||||
|     /// <summary> | ||||
|     /// Name of the HttpClient registered for NVD fetches. | ||||
|     /// </summary> | ||||
|     public const string HttpClientName = "nvd"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Base API endpoint for CVE feed queries. | ||||
|     /// </summary> | ||||
|     public Uri BaseEndpoint { get; set; } = new("https://services.nvd.nist.gov/rest/json/cves/2.0"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Duration of each modified window fetch. | ||||
|     /// </summary> | ||||
|     public TimeSpan WindowSize { get; set; } = TimeSpan.FromHours(4); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Overlap added when advancing the sliding window to cover upstream delays. | ||||
|     /// </summary> | ||||
|     public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Maximum look-back period used when the connector first starts or state is empty. | ||||
|     /// </summary> | ||||
|     public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(7); | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (BaseEndpoint is null) | ||||
|         { | ||||
|             throw new InvalidOperationException("NVD base endpoint must be configured."); | ||||
|         } | ||||
|  | ||||
|         if (!BaseEndpoint.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("NVD base endpoint must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (WindowSize <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Window size must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (WindowOverlap < TimeSpan.Zero || WindowOverlap >= WindowSize) | ||||
|         { | ||||
|             throw new InvalidOperationException("Window overlap must be non-negative and less than the window size."); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfill <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Initial backfill duration must be positive."); | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										64
									
								
								src/StellaOps.Concelier.Connector.Nvd/Internal/NvdCursor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										64
									
								
								src/StellaOps.Concelier.Connector.Nvd/Internal/NvdCursor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,64 @@ | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Concelier.Connector.Common.Cursors; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd.Internal; | ||||
|  | ||||
| internal sealed record NvdCursor( | ||||
|     TimeWindowCursorState Window, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings) | ||||
| { | ||||
|     public static NvdCursor Empty { get; } = new(TimeWindowCursorState.Empty, Array.Empty<Guid>(), Array.Empty<Guid>()); | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument(); | ||||
|         Window.WriteTo(document); | ||||
|         document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); | ||||
|         document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static NvdCursor FromBsonDocument(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var window = TimeWindowCursorState.FromBsonDocument(document); | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|  | ||||
|         return new NvdCursor(window, pendingDocuments, pendingMappings); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return Array.Empty<Guid>(); | ||||
|         } | ||||
|  | ||||
|         var results = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.AsString, out var guid)) | ||||
|             { | ||||
|                 results.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
|  | ||||
|     public NvdCursor WithWindow(TimeWindow window) | ||||
|         => this with { Window = Window.WithWindow(window) }; | ||||
|  | ||||
|     public NvdCursor WithPendingDocuments(IEnumerable<Guid> ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() }; | ||||
|  | ||||
|     public NvdCursor WithPendingMappings(IEnumerable<Guid> ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() }; | ||||
| } | ||||
| @@ -0,0 +1,76 @@ | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd.Internal; | ||||
|  | ||||
| public sealed class NvdDiagnostics : IDisposable | ||||
| { | ||||
|     public const string MeterName = "StellaOps.Concelier.Connector.Nvd"; | ||||
|     public const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _fetchAttempts; | ||||
|     private readonly Counter<long> _fetchDocuments; | ||||
|     private readonly Counter<long> _fetchFailures; | ||||
|     private readonly Counter<long> _fetchUnchanged; | ||||
|     private readonly Counter<long> _parseSuccess; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Counter<long> _parseQuarantine; | ||||
|     private readonly Counter<long> _mapSuccess; | ||||
|  | ||||
|     public NvdDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _fetchAttempts = _meter.CreateCounter<long>( | ||||
|             name: "nvd.fetch.attempts", | ||||
|             unit: "operations", | ||||
|             description: "Number of NVD fetch operations attempted, including paginated windows."); | ||||
|         _fetchDocuments = _meter.CreateCounter<long>( | ||||
|             name: "nvd.fetch.documents", | ||||
|             unit: "documents", | ||||
|             description: "Count of NVD documents fetched and persisted."); | ||||
|         _fetchFailures = _meter.CreateCounter<long>( | ||||
|             name: "nvd.fetch.failures", | ||||
|             unit: "operations", | ||||
|             description: "Count of NVD fetch attempts that resulted in an error or missing document."); | ||||
|         _fetchUnchanged = _meter.CreateCounter<long>( | ||||
|             name: "nvd.fetch.unchanged", | ||||
|             unit: "operations", | ||||
|             description: "Count of NVD fetch attempts returning 304 Not Modified."); | ||||
|         _parseSuccess = _meter.CreateCounter<long>( | ||||
|             name: "nvd.parse.success", | ||||
|             unit: "documents", | ||||
|             description: "Count of NVD documents successfully validated and converted into DTOs."); | ||||
|         _parseFailures = _meter.CreateCounter<long>( | ||||
|             name: "nvd.parse.failures", | ||||
|             unit: "documents", | ||||
|             description: "Count of NVD documents that failed parsing due to missing content or read errors."); | ||||
|         _parseQuarantine = _meter.CreateCounter<long>( | ||||
|             name: "nvd.parse.quarantine", | ||||
|             unit: "documents", | ||||
|             description: "Count of NVD documents quarantined due to schema validation failures."); | ||||
|         _mapSuccess = _meter.CreateCounter<long>( | ||||
|             name: "nvd.map.success", | ||||
|             unit: "advisories", | ||||
|             description: "Count of canonical advisories produced by NVD mapping."); | ||||
|     } | ||||
|  | ||||
|     public void FetchAttempt() => _fetchAttempts.Add(1); | ||||
|  | ||||
|     public void FetchDocument() => _fetchDocuments.Add(1); | ||||
|  | ||||
|     public void FetchFailure() => _fetchFailures.Add(1); | ||||
|  | ||||
|     public void FetchUnchanged() => _fetchUnchanged.Add(1); | ||||
|  | ||||
|     public void ParseSuccess() => _parseSuccess.Add(1); | ||||
|  | ||||
|     public void ParseFailure() => _parseFailures.Add(1); | ||||
|  | ||||
|     public void ParseQuarantine() => _parseQuarantine.Add(1); | ||||
|  | ||||
|     public void MapSuccess(long count = 1) => _mapSuccess.Add(count); | ||||
|  | ||||
|     public Meter Meter => _meter; | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
| } | ||||
							
								
								
									
										774
									
								
								src/StellaOps.Concelier.Connector.Nvd/Internal/NvdMapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										774
									
								
								src/StellaOps.Concelier.Connector.Nvd/Internal/NvdMapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,774 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Collections.Immutable; | ||||
| using System.Linq; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using NuGet.Versioning; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Normalization.Identifiers; | ||||
| using StellaOps.Concelier.Normalization.Cvss; | ||||
| using StellaOps.Concelier.Normalization.Text; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd.Internal; | ||||
|  | ||||
| internal static class NvdMapper | ||||
| { | ||||
|     public static IReadOnlyList<Advisory> Map(JsonDocument document, DocumentRecord sourceDocument, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|         ArgumentNullException.ThrowIfNull(sourceDocument); | ||||
|  | ||||
|         if (!document.RootElement.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<Advisory>(); | ||||
|         } | ||||
|  | ||||
|         var advisories = new List<Advisory>(vulnerabilities.GetArrayLength()); | ||||
|         var index = 0; | ||||
|         foreach (var vulnerability in vulnerabilities.EnumerateArray()) | ||||
|         { | ||||
|             if (!vulnerability.TryGetProperty("cve", out var cve) || cve.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 index++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!cve.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String) | ||||
|             { | ||||
|                 index++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var cveId = idElement.GetString(); | ||||
|             var advisoryKey = string.IsNullOrWhiteSpace(cveId) | ||||
|                 ? $"nvd:{sourceDocument.Id:N}:{index}" | ||||
|                 : cveId; | ||||
|  | ||||
|             var published = TryGetDateTime(cve, "published"); | ||||
|             var modified = TryGetDateTime(cve, "lastModified"); | ||||
|             var description = GetNormalizedDescription(cve); | ||||
|  | ||||
|             var weaknessMetadata = GetWeaknessMetadata(cve); | ||||
|             var references = GetReferences(cve, sourceDocument, recordedAt, weaknessMetadata); | ||||
|             var affectedPackages = GetAffectedPackages(cve, cveId, sourceDocument, recordedAt); | ||||
|             var cvssMetrics = GetCvssMetrics(cve, sourceDocument, recordedAt, out var severity); | ||||
|             var weaknesses = BuildWeaknesses(weaknessMetadata, recordedAt); | ||||
|             var canonicalMetricId = cvssMetrics.Count > 0 | ||||
|                 ? $"{cvssMetrics[0].Version}|{cvssMetrics[0].Vector}" | ||||
|                 : null; | ||||
|  | ||||
|             var provenance = new[] | ||||
|             { | ||||
|                 new AdvisoryProvenance( | ||||
|                     NvdConnectorPlugin.SourceName, | ||||
|                     "document", | ||||
|                     sourceDocument.Uri, | ||||
|                     sourceDocument.FetchedAt, | ||||
|                     new[] { ProvenanceFieldMasks.Advisory }), | ||||
|                 new AdvisoryProvenance( | ||||
|                     NvdConnectorPlugin.SourceName, | ||||
|                     "mapping", | ||||
|                     string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.Advisory }), | ||||
|             }; | ||||
|  | ||||
|             var title = string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId; | ||||
|  | ||||
|             var aliasCandidates = new List<string>(capacity: 2); | ||||
|             if (!string.IsNullOrWhiteSpace(cveId)) | ||||
|             { | ||||
|                 aliasCandidates.Add(cveId); | ||||
|             } | ||||
|  | ||||
|             aliasCandidates.Add(advisoryKey); | ||||
|  | ||||
|             var advisory = new Advisory( | ||||
|                 advisoryKey: advisoryKey, | ||||
|                 title: title, | ||||
|                 summary: string.IsNullOrEmpty(description.Text) ? null : description.Text, | ||||
|                 language: description.Language, | ||||
|                 published: published, | ||||
|                 modified: modified, | ||||
|                 severity: severity, | ||||
|                 exploitKnown: false, | ||||
|                 aliases: aliasCandidates, | ||||
|                 references: references, | ||||
|                 affectedPackages: affectedPackages, | ||||
|                 cvssMetrics: cvssMetrics, | ||||
|                 provenance: provenance, | ||||
|                 description: string.IsNullOrEmpty(description.Text) ? null : description.Text, | ||||
|                 cwes: weaknesses, | ||||
|                 canonicalMetricId: canonicalMetricId); | ||||
|  | ||||
|             advisories.Add(advisory); | ||||
|             index++; | ||||
|         } | ||||
|  | ||||
|         return advisories; | ||||
|     } | ||||
|  | ||||
|     private static NormalizedDescription GetNormalizedDescription(JsonElement cve) | ||||
|     { | ||||
|         var candidates = new List<LocalizedText>(); | ||||
|  | ||||
|         if (cve.TryGetProperty("descriptions", out var descriptions) && descriptions.ValueKind == JsonValueKind.Array) | ||||
|         { | ||||
|             foreach (var item in descriptions.EnumerateArray()) | ||||
|             { | ||||
|                 if (item.ValueKind != JsonValueKind.Object) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var text = item.TryGetProperty("value", out var valueElement) && valueElement.ValueKind == JsonValueKind.String | ||||
|                     ? valueElement.GetString() | ||||
|                     : null; | ||||
|                 var lang = item.TryGetProperty("lang", out var langElement) && langElement.ValueKind == JsonValueKind.String | ||||
|                     ? langElement.GetString() | ||||
|                     : null; | ||||
|  | ||||
|                 if (!string.IsNullOrWhiteSpace(text)) | ||||
|                 { | ||||
|                     candidates.Add(new LocalizedText(text, lang)); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return DescriptionNormalizer.Normalize(candidates); | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? TryGetDateTime(JsonElement element, string propertyName) | ||||
|     { | ||||
|         if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return DateTimeOffset.TryParse(property.GetString(), out var parsed) ? parsed : null; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> GetReferences( | ||||
|         JsonElement cve, | ||||
|         DocumentRecord document, | ||||
|         DateTimeOffset recordedAt, | ||||
|         IReadOnlyList<WeaknessMetadata> weaknesses) | ||||
|     { | ||||
|         var references = new List<AdvisoryReference>(); | ||||
|         if (!cve.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             AppendWeaknessReferences(references, weaknesses, recordedAt); | ||||
|             return references; | ||||
|         } | ||||
|  | ||||
|         foreach (var reference in referencesElement.EnumerateArray()) | ||||
|         { | ||||
|             if (!reference.TryGetProperty("url", out var urlElement) || urlElement.ValueKind != JsonValueKind.String) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var url = urlElement.GetString(); | ||||
|             if (string.IsNullOrWhiteSpace(url) || !Validation.LooksLikeHttpUrl(url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var sourceTag = reference.TryGetProperty("source", out var sourceElement) ? sourceElement.GetString() : null; | ||||
|             string? kind = null; | ||||
|             if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) | ||||
|             { | ||||
|                 kind = tagsElement.EnumerateArray().Select(static t => t.GetString()).FirstOrDefault(static tag => !string.IsNullOrWhiteSpace(tag))?.ToLowerInvariant(); | ||||
|             } | ||||
|  | ||||
|             references.Add(new AdvisoryReference( | ||||
|                 url: url, | ||||
|                 kind: kind, | ||||
|                 sourceTag: sourceTag, | ||||
|                 summary: null, | ||||
|                 provenance: new AdvisoryProvenance( | ||||
|                     NvdConnectorPlugin.SourceName, | ||||
|                     "reference", | ||||
|                     url, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.References }))); | ||||
|         } | ||||
|  | ||||
|         AppendWeaknessReferences(references, weaknesses, recordedAt); | ||||
|         return references; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<WeaknessMetadata> GetWeaknessMetadata(JsonElement cve) | ||||
|     { | ||||
|         if (!cve.TryGetProperty("weaknesses", out var weaknesses) || weaknesses.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<WeaknessMetadata>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<WeaknessMetadata>(weaknesses.GetArrayLength()); | ||||
|         foreach (var weakness in weaknesses.EnumerateArray()) | ||||
|         { | ||||
|             if (!weakness.TryGetProperty("description", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             string? cweId = null; | ||||
|             string? name = null; | ||||
|  | ||||
|             foreach (var description in descriptions.EnumerateArray()) | ||||
|             { | ||||
|                 if (description.ValueKind != JsonValueKind.Object) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!description.TryGetProperty("value", out var valueElement) || valueElement.ValueKind != JsonValueKind.String) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var value = valueElement.GetString(); | ||||
|                 if (string.IsNullOrWhiteSpace(value)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var trimmed = value.Trim(); | ||||
|                 if (trimmed.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase)) | ||||
|                 { | ||||
|                     cweId ??= trimmed.ToUpperInvariant(); | ||||
|                 } | ||||
|                 else | ||||
|                 { | ||||
|                     name ??= trimmed; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             if (string.IsNullOrWhiteSpace(cweId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             list.Add(new WeaknessMetadata(cweId, name)); | ||||
|         } | ||||
|  | ||||
|         return list.Count == 0 ? Array.Empty<WeaknessMetadata>() : list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryWeakness> BuildWeaknesses(IReadOnlyList<WeaknessMetadata> metadata, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (metadata.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryWeakness>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<AdvisoryWeakness>(metadata.Count); | ||||
|         foreach (var entry in metadata) | ||||
|         { | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 NvdConnectorPlugin.SourceName, | ||||
|                 "weakness", | ||||
|                 entry.CweId, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.Weaknesses }); | ||||
|  | ||||
|             var provenanceArray = ImmutableArray.Create(provenance); | ||||
|             list.Add(new AdvisoryWeakness( | ||||
|                 taxonomy: "cwe", | ||||
|                 identifier: entry.CweId, | ||||
|                 name: entry.Name, | ||||
|                 uri: BuildCweUrl(entry.CweId), | ||||
|                 provenance: provenanceArray)); | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static void AppendWeaknessReferences( | ||||
|         List<AdvisoryReference> references, | ||||
|         IReadOnlyList<WeaknessMetadata> weaknesses, | ||||
|         DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (weaknesses.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var existing = new HashSet<string>(references.Select(reference => reference.Url), StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         foreach (var weakness in weaknesses) | ||||
|         { | ||||
|             var url = BuildCweUrl(weakness.CweId); | ||||
|             if (url is null || existing.Contains(url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 NvdConnectorPlugin.SourceName, | ||||
|                 "reference", | ||||
|                 url, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.References }); | ||||
|  | ||||
|             references.Add(new AdvisoryReference(url, "weakness", weakness.CweId, weakness.Name, provenance)); | ||||
|             existing.Add(url); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> GetAffectedPackages(JsonElement cve, string? cveId, DocumentRecord document, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var packages = new Dictionary<string, PackageAccumulator>(StringComparer.Ordinal); | ||||
|         if (!cve.TryGetProperty("configurations", out var configurations) || configurations.ValueKind != JsonValueKind.Object) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         if (!configurations.TryGetProperty("nodes", out var nodes) || nodes.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         foreach (var node in nodes.EnumerateArray()) | ||||
|         { | ||||
|             if (!node.TryGetProperty("cpeMatch", out var matches) || matches.ValueKind != JsonValueKind.Array) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             foreach (var match in matches.EnumerateArray()) | ||||
|             { | ||||
|                 if (match.TryGetProperty("vulnerable", out var vulnerableElement) && vulnerableElement.ValueKind == JsonValueKind.False) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!match.TryGetProperty("criteria", out var criteriaElement) || criteriaElement.ValueKind != JsonValueKind.String) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var criteria = criteriaElement.GetString(); | ||||
|                 if (string.IsNullOrWhiteSpace(criteria)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var identifier = IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe) && !string.IsNullOrWhiteSpace(normalizedCpe) | ||||
|                     ? normalizedCpe | ||||
|                     : criteria.Trim(); | ||||
|  | ||||
|                 var provenance = new AdvisoryProvenance( | ||||
|                     NvdConnectorPlugin.SourceName, | ||||
|                     "cpe", | ||||
|                     document.Uri, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.AffectedPackages }); | ||||
|                 if (!packages.TryGetValue(identifier, out var accumulator)) | ||||
|                 { | ||||
|                     accumulator = new PackageAccumulator(); | ||||
|                     packages[identifier] = accumulator; | ||||
|                 } | ||||
|  | ||||
|                 var range = BuildVersionRange(match, criteria, provenance); | ||||
|                 if (range is not null) | ||||
|                 { | ||||
|                     accumulator.Ranges.Add(range); | ||||
|                 } | ||||
|  | ||||
|                 accumulator.Provenance.Add(provenance); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (packages.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         return packages | ||||
|             .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) | ||||
|             .Select(kvp => | ||||
|             { | ||||
|                 var ranges = kvp.Value.Ranges.Count == 0 | ||||
|                     ? Array.Empty<AffectedVersionRange>() | ||||
|                     : kvp.Value.Ranges | ||||
|                         .OrderBy(static range => range, AffectedVersionRangeComparer.Instance) | ||||
|                         .ToArray(); | ||||
|  | ||||
|                 var provenance = kvp.Value.Provenance | ||||
|                     .OrderBy(static p => p.Source, StringComparer.Ordinal) | ||||
|                     .ThenBy(static p => p.Kind, StringComparer.Ordinal) | ||||
|                     .ThenBy(static p => p.Value, StringComparer.Ordinal) | ||||
|                     .ThenBy(static p => p.RecordedAt.UtcDateTime) | ||||
|                     .ToArray(); | ||||
|  | ||||
|                 var normalizedNote = string.IsNullOrWhiteSpace(cveId) | ||||
|                     ? $"nvd:{document.Id:N}" | ||||
|                     : $"nvd:{cveId}"; | ||||
|  | ||||
|                 var normalizedVersions = new List<NormalizedVersionRule>(ranges.Length); | ||||
|                 foreach (var range in ranges) | ||||
|                 { | ||||
|                     var rule = range.ToNormalizedVersionRule(normalizedNote); | ||||
|                     if (rule is not null) | ||||
|                     { | ||||
|                         normalizedVersions.Add(rule); | ||||
|                     } | ||||
|                 } | ||||
|  | ||||
|                 return new AffectedPackage( | ||||
|                     type: AffectedPackageTypes.Cpe, | ||||
|                     identifier: kvp.Key, | ||||
|                     platform: null, | ||||
|                     versionRanges: ranges, | ||||
|                     statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                     provenance: provenance, | ||||
|                     normalizedVersions: normalizedVersions.Count == 0 | ||||
|                         ? Array.Empty<NormalizedVersionRule>() | ||||
|                         : normalizedVersions.ToArray()); | ||||
|             }) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<CvssMetric> GetCvssMetrics(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt, out string? severity) | ||||
|     { | ||||
|         severity = null; | ||||
|         if (!cve.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Object) | ||||
|         { | ||||
|             return Array.Empty<CvssMetric>(); | ||||
|         } | ||||
|  | ||||
|         var sources = new[] { "cvssMetricV31", "cvssMetricV30", "cvssMetricV2" }; | ||||
|         foreach (var source in sources) | ||||
|         { | ||||
|             if (!metrics.TryGetProperty(source, out var array) || array.ValueKind != JsonValueKind.Array) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var list = new List<CvssMetric>(); | ||||
|             foreach (var item in array.EnumerateArray()) | ||||
|             { | ||||
|                 if (!item.TryGetProperty("cvssData", out var data) || data.ValueKind != JsonValueKind.Object) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!data.TryGetProperty("vectorString", out var vectorElement) || vectorElement.ValueKind != JsonValueKind.String) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!data.TryGetProperty("baseScore", out var scoreElement) || scoreElement.ValueKind != JsonValueKind.Number) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!data.TryGetProperty("baseSeverity", out var severityElement) || severityElement.ValueKind != JsonValueKind.String) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var vector = vectorElement.GetString() ?? string.Empty; | ||||
|                 var baseScore = scoreElement.GetDouble(); | ||||
|                 var baseSeverity = severityElement.GetString(); | ||||
|                 var versionToken = source switch | ||||
|                 { | ||||
|                     "cvssMetricV30" => "3.0", | ||||
|                     "cvssMetricV31" => "3.1", | ||||
|                     _ => "2.0", | ||||
|                 }; | ||||
|  | ||||
|                 if (!CvssMetricNormalizer.TryNormalize(versionToken, vector, baseScore, baseSeverity, out var normalized)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 severity ??= normalized.BaseSeverity; | ||||
|  | ||||
|                 list.Add(normalized.ToModel(new AdvisoryProvenance( | ||||
|                     NvdConnectorPlugin.SourceName, | ||||
|                     "cvss", | ||||
|                     normalized.Vector, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.CvssMetrics }))); | ||||
|             } | ||||
|  | ||||
|             if (list.Count > 0) | ||||
|             { | ||||
|                 return list; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return Array.Empty<CvssMetric>(); | ||||
|     } | ||||
|  | ||||
|     private static AffectedVersionRange? BuildVersionRange(JsonElement match, string criteria, AdvisoryProvenance provenance) | ||||
|     { | ||||
|         static string? ReadString(JsonElement parent, string property) | ||||
|         { | ||||
|             if (!parent.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.String) | ||||
|             { | ||||
|                 return null; | ||||
|             } | ||||
|  | ||||
|             var text = value.GetString(); | ||||
|             return string.IsNullOrWhiteSpace(text) ? null : text.Trim(); | ||||
|         } | ||||
|  | ||||
|         var version = ReadString(match, "version"); | ||||
|         if (string.Equals(version, "*", StringComparison.Ordinal)) | ||||
|         { | ||||
|             version = null; | ||||
|         } | ||||
|  | ||||
|         version ??= TryExtractVersionFromCriteria(criteria); | ||||
|  | ||||
|         var versionStartIncluding = ReadString(match, "versionStartIncluding"); | ||||
|         var versionStartExcluding = ReadString(match, "versionStartExcluding"); | ||||
|         var versionEndIncluding = ReadString(match, "versionEndIncluding"); | ||||
|         var versionEndExcluding = ReadString(match, "versionEndExcluding"); | ||||
|  | ||||
|         var vendorExtensions = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         if (versionStartIncluding is not null) | ||||
|         { | ||||
|             vendorExtensions["versionStartIncluding"] = versionStartIncluding; | ||||
|         } | ||||
|  | ||||
|         if (versionStartExcluding is not null) | ||||
|         { | ||||
|             vendorExtensions["versionStartExcluding"] = versionStartExcluding; | ||||
|         } | ||||
|  | ||||
|         if (versionEndIncluding is not null) | ||||
|         { | ||||
|             vendorExtensions["versionEndIncluding"] = versionEndIncluding; | ||||
|         } | ||||
|  | ||||
|         if (versionEndExcluding is not null) | ||||
|         { | ||||
|             vendorExtensions["versionEndExcluding"] = versionEndExcluding; | ||||
|         } | ||||
|  | ||||
|         if (version is not null) | ||||
|         { | ||||
|             vendorExtensions["version"] = version; | ||||
|         } | ||||
|  | ||||
|         string? introduced = null; | ||||
|         string? fixedVersion = null; | ||||
|         string? lastAffected = null; | ||||
|         string? exactVersion = null; | ||||
|         var expressionParts = new List<string>(); | ||||
|  | ||||
|         var introducedInclusive = true; | ||||
|         var fixedInclusive = false; | ||||
|         var lastInclusive = true; | ||||
|  | ||||
|         if (versionStartIncluding is not null) | ||||
|         { | ||||
|             introduced = versionStartIncluding; | ||||
|             introducedInclusive = true; | ||||
|             expressionParts.Add($">={versionStartIncluding}"); | ||||
|         } | ||||
|  | ||||
|         if (versionStartExcluding is not null) | ||||
|         { | ||||
|             if (introduced is null) | ||||
|             { | ||||
|                 introduced = versionStartExcluding; | ||||
|                 introducedInclusive = false; | ||||
|             } | ||||
|             expressionParts.Add($">{versionStartExcluding}"); | ||||
|         } | ||||
|  | ||||
|         if (versionEndExcluding is not null) | ||||
|         { | ||||
|             fixedVersion = versionEndExcluding; | ||||
|             fixedInclusive = false; | ||||
|             expressionParts.Add($"<{versionEndExcluding}"); | ||||
|         } | ||||
|  | ||||
|         if (versionEndIncluding is not null) | ||||
|         { | ||||
|             lastAffected = versionEndIncluding; | ||||
|             lastInclusive = true; | ||||
|             expressionParts.Add($"<={versionEndIncluding}"); | ||||
|         } | ||||
|  | ||||
|         if (version is not null) | ||||
|         { | ||||
|             introduced = version; | ||||
|             introducedInclusive = true; | ||||
|             lastAffected = version; | ||||
|             lastInclusive = true; | ||||
|             exactVersion = version; | ||||
|             expressionParts.Add($"=={version}"); | ||||
|         } | ||||
|  | ||||
|         if (introduced is null && fixedVersion is null && lastAffected is null && vendorExtensions.Count == 0) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var rangeExpression = expressionParts.Count > 0 ? string.Join(' ', expressionParts) : null; | ||||
|         IReadOnlyDictionary<string, string>? extensions = vendorExtensions.Count == 0 ? null : vendorExtensions; | ||||
|  | ||||
|         SemVerPrimitive? semVerPrimitive = null; | ||||
|         if (TryBuildSemVerPrimitive( | ||||
|             introduced, | ||||
|             introducedInclusive, | ||||
|             fixedVersion, | ||||
|             fixedInclusive, | ||||
|             lastAffected, | ||||
|             lastInclusive, | ||||
|             exactVersion, | ||||
|             rangeExpression, | ||||
|             out var primitive)) | ||||
|         { | ||||
|             semVerPrimitive = primitive; | ||||
|         } | ||||
|  | ||||
|         var primitives = semVerPrimitive is null && extensions is null | ||||
|             ? null | ||||
|             : new RangePrimitives(semVerPrimitive, null, null, extensions); | ||||
|  | ||||
|         var provenanceValue = provenance.Value ?? criteria; | ||||
|         var rangeProvenance = new AdvisoryProvenance( | ||||
|             provenance.Source, | ||||
|             provenance.Kind, | ||||
|             provenanceValue, | ||||
|             provenance.RecordedAt, | ||||
|             new[] { ProvenanceFieldMasks.VersionRanges }); | ||||
|  | ||||
|         return new AffectedVersionRange( | ||||
|             rangeKind: "cpe", | ||||
|             introducedVersion: introduced, | ||||
|             fixedVersion: fixedVersion, | ||||
|             lastAffectedVersion: lastAffected, | ||||
|             rangeExpression: rangeExpression, | ||||
|             provenance: rangeProvenance, | ||||
|             primitives); | ||||
|     } | ||||
|  | ||||
|     private static bool TryBuildSemVerPrimitive( | ||||
|         string? introduced, | ||||
|         bool introducedInclusive, | ||||
|         string? fixedVersion, | ||||
|         bool fixedInclusive, | ||||
|         string? lastAffected, | ||||
|         bool lastInclusive, | ||||
|         string? exactVersion, | ||||
|         string? constraintExpression, | ||||
|         out SemVerPrimitive? primitive) | ||||
|     { | ||||
|         primitive = null; | ||||
|  | ||||
|         if (!TryNormalizeSemVer(introduced, out var normalizedIntroduced) | ||||
|             || !TryNormalizeSemVer(fixedVersion, out var normalizedFixed) | ||||
|             || !TryNormalizeSemVer(lastAffected, out var normalizedLast) | ||||
|             || !TryNormalizeSemVer(exactVersion, out var normalizedExact)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (normalizedIntroduced is null && normalizedFixed is null && normalizedLast is null && normalizedExact is null) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         primitive = new SemVerPrimitive( | ||||
|             Introduced: normalizedIntroduced, | ||||
|             IntroducedInclusive: normalizedIntroduced is null ? true : introducedInclusive, | ||||
|             Fixed: normalizedFixed, | ||||
|             FixedInclusive: normalizedFixed is null ? false : fixedInclusive, | ||||
|             LastAffected: normalizedLast, | ||||
|             LastAffectedInclusive: normalizedLast is null ? false : lastInclusive, | ||||
|             ConstraintExpression: constraintExpression, | ||||
|             ExactValue: normalizedExact); | ||||
|  | ||||
|     return true; | ||||
|     } | ||||
|  | ||||
|     private static bool TryNormalizeSemVer(string? value, out string? normalized) | ||||
|     { | ||||
|         normalized = null; | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         var trimmed = value.Trim(); | ||||
|         if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 1) | ||||
|         { | ||||
|             trimmed = trimmed[1..]; | ||||
|         } | ||||
|  | ||||
|         if (!NuGetVersion.TryParse(trimmed, out var parsed)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         normalized = parsed.ToNormalizedString(); | ||||
|         return true; | ||||
|     } | ||||
|  | ||||
|     private static string? BuildCweUrl(string cweId) | ||||
|     { | ||||
|         var dashIndex = cweId.IndexOf('-'); | ||||
|         if (dashIndex < 0 || dashIndex == cweId.Length - 1) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var digits = new StringBuilder(); | ||||
|         for (var i = dashIndex + 1; i < cweId.Length; i++) | ||||
|         { | ||||
|             var ch = cweId[i]; | ||||
|             if (char.IsDigit(ch)) | ||||
|             { | ||||
|                 digits.Append(ch); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return digits.Length == 0 ? null : $"https://cwe.mitre.org/data/definitions/{digits}.html"; | ||||
|     } | ||||
|  | ||||
|     private static string? TryExtractVersionFromCriteria(string criteria) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(criteria)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var segments = criteria.Split(':'); | ||||
|         if (segments.Length < 6) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var version = segments[5]; | ||||
|         if (string.IsNullOrWhiteSpace(version)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (string.Equals(version, "*", StringComparison.Ordinal) || string.Equals(version, "-", StringComparison.Ordinal)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return version; | ||||
|     } | ||||
|  | ||||
|     private readonly record struct WeaknessMetadata(string CweId, string? Name); | ||||
|  | ||||
|     private sealed class PackageAccumulator | ||||
|     { | ||||
|         public List<AffectedVersionRange> Ranges { get; } = new(); | ||||
|  | ||||
|         public List<AdvisoryProvenance> Provenance { get; } = new(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,25 @@ | ||||
| using System.IO; | ||||
| using System.Reflection; | ||||
| using System.Threading; | ||||
| using Json.Schema; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd.Internal; | ||||
|  | ||||
| internal static class NvdSchemaProvider | ||||
| { | ||||
|     private static readonly Lazy<JsonSchema> Cached = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); | ||||
|  | ||||
|     public static JsonSchema Schema => Cached.Value; | ||||
|  | ||||
|     private static JsonSchema LoadSchema() | ||||
|     { | ||||
|         var assembly = typeof(NvdSchemaProvider).GetTypeInfo().Assembly; | ||||
|         const string resourceName = "StellaOps.Concelier.Connector.Nvd.Schemas.nvd-vulnerability.schema.json"; | ||||
|  | ||||
|         using var stream = assembly.GetManifestResourceStream(resourceName) | ||||
|             ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); | ||||
|         using var reader = new StreamReader(stream); | ||||
|         var schemaText = reader.ReadToEnd(); | ||||
|         return JsonSchema.FromText(schemaText); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										565
									
								
								src/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										565
									
								
								src/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,565 @@ | ||||
| using System.Globalization; | ||||
| using System.Security.Cryptography; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Connector.Common; | ||||
| using StellaOps.Concelier.Connector.Common.Fetch; | ||||
| using StellaOps.Concelier.Connector.Common.Json; | ||||
| using StellaOps.Concelier.Connector.Common.Cursors; | ||||
| using StellaOps.Concelier.Connector.Nvd.Configuration; | ||||
| using StellaOps.Concelier.Connector.Nvd.Internal; | ||||
| using StellaOps.Concelier.Storage.Mongo; | ||||
| using StellaOps.Concelier.Storage.Mongo.Advisories; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
| using StellaOps.Concelier.Storage.Mongo.Dtos; | ||||
| using StellaOps.Concelier.Storage.Mongo.ChangeHistory; | ||||
| using StellaOps.Plugin; | ||||
| using Json.Schema; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd; | ||||
|  | ||||
| public sealed class NvdConnector : IFeedConnector | ||||
| { | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly IChangeHistoryStore _changeHistoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly IJsonSchemaValidator _schemaValidator; | ||||
|     private readonly NvdOptions _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<NvdConnector> _logger; | ||||
|     private readonly NvdDiagnostics _diagnostics; | ||||
|  | ||||
|     private static readonly JsonSchema Schema = NvdSchemaProvider.Schema; | ||||
|  | ||||
|     public NvdConnector( | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         IChangeHistoryStore changeHistoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IJsonSchemaValidator schemaValidator, | ||||
|         IOptions<NvdOptions> options, | ||||
|         NvdDiagnostics diagnostics, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<NvdConnector> logger) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _changeHistoryStore = changeHistoryStore ?? throw new ArgumentNullException(nameof(changeHistoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); | ||||
|         _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => NvdConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|         var windowOptions = new TimeWindowCursorOptions | ||||
|         { | ||||
|             WindowSize = _options.WindowSize, | ||||
|             Overlap = _options.WindowOverlap, | ||||
|             InitialBackfill = _options.InitialBackfill, | ||||
|         }; | ||||
|  | ||||
|         var window = TimeWindowCursorPlanner.GetNextWindow(now, cursor.Window, windowOptions); | ||||
|         var requestUri = BuildRequestUri(window); | ||||
|  | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|         { | ||||
|             ["windowStart"] = window.Start.ToString("O"), | ||||
|             ["windowEnd"] = window.End.ToString("O"), | ||||
|         }; | ||||
|         metadata["startIndex"] = "0"; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             _diagnostics.FetchAttempt(); | ||||
|  | ||||
|             var result = await _fetchService.FetchAsync( | ||||
|                 new SourceFetchRequest( | ||||
|                     NvdOptions.HttpClientName, | ||||
|                     SourceName, | ||||
|                     requestUri) | ||||
|                 { | ||||
|                     Metadata = metadata | ||||
|                 }, | ||||
|                 cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (result.IsNotModified) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|                 _logger.LogDebug("NVD window {Start} - {End} returned 304", window.Start, window.End); | ||||
|                 await UpdateCursorAsync(cursor.WithWindow(window), cancellationToken).ConfigureAwait(false); | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             if (!result.IsSuccess || result.Document is null) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             _diagnostics.FetchDocument(); | ||||
|  | ||||
|             var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments) | ||||
|             { | ||||
|                 result.Document.Id | ||||
|             }; | ||||
|  | ||||
|             var additionalDocuments = await FetchAdditionalPagesAsync( | ||||
|                 window, | ||||
|                 metadata, | ||||
|                 result.Document, | ||||
|                 cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             foreach (var documentId in additionalDocuments) | ||||
|             { | ||||
|                 pendingDocuments.Add(documentId); | ||||
|             } | ||||
|  | ||||
|             var updated = cursor | ||||
|                 .WithWindow(window) | ||||
|                 .WithPendingDocuments(pendingDocuments) | ||||
|                 .WithPendingMappings(cursor.PendingMappings); | ||||
|  | ||||
|             await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _diagnostics.FetchFailure(); | ||||
|             _logger.LogError(ex, "NVD fetch failed for {Uri}", requestUri); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remainingFetch = cursor.PendingDocuments.ToList(); | ||||
|         var pendingMapping = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 remainingFetch.Remove(documentId); | ||||
|                 pendingMapping.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _logger.LogWarning("Document {DocumentId} is missing GridFS content; skipping", documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 remainingFetch.Remove(documentId); | ||||
|                 pendingMapping.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             try | ||||
|             { | ||||
|                 using var jsonDocument = JsonDocument.Parse(rawBytes); | ||||
|                 try | ||||
|                 { | ||||
|                     _schemaValidator.Validate(jsonDocument, Schema, document.Uri); | ||||
|                 } | ||||
|                 catch (JsonSchemaValidationException ex) | ||||
|                 { | ||||
|                     _logger.LogWarning(ex, "NVD schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); | ||||
|                     await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                     remainingFetch.Remove(documentId); | ||||
|                     pendingMapping.Remove(documentId); | ||||
|                     _diagnostics.ParseQuarantine(); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement); | ||||
|                 var payload = BsonDocument.Parse(sanitized); | ||||
|  | ||||
|                 var dtoRecord = new DtoRecord( | ||||
|                     Guid.NewGuid(), | ||||
|                     document.Id, | ||||
|                     SourceName, | ||||
|                     "nvd.cve.v2", | ||||
|                     payload, | ||||
|                     _timeProvider.GetUtcNow()); | ||||
|  | ||||
|                 await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|                 _diagnostics.ParseSuccess(); | ||||
|  | ||||
|                 remainingFetch.Remove(documentId); | ||||
|                 if (!pendingMapping.Contains(documentId)) | ||||
|                 { | ||||
|                     pendingMapping.Add(documentId); | ||||
|                 } | ||||
|             } | ||||
|             catch (JsonException ex) | ||||
|             { | ||||
|                 _logger.LogWarning(ex, "Failed to parse NVD JSON payload for document {DocumentId} ({Uri})", document.Id, document.Uri); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingFetch.Remove(documentId); | ||||
|                 pendingMapping.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remainingFetch) | ||||
|             .WithPendingMappings(pendingMapping); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMapping = cursor.PendingMappings.ToList(); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (dto is null || document is null) | ||||
|             { | ||||
|                 pendingMapping.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var json = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings | ||||
|             { | ||||
|                 OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, | ||||
|             }); | ||||
|  | ||||
|             using var jsonDocument = JsonDocument.Parse(json); | ||||
|             var advisories = NvdMapper.Map(jsonDocument, document, now) | ||||
|                 .GroupBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) | ||||
|                 .Select(static group => group.First()) | ||||
|                 .ToArray(); | ||||
|  | ||||
|             var mappedCount = 0L; | ||||
|             foreach (var advisory in advisories) | ||||
|             { | ||||
|                 if (string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) | ||||
|                 { | ||||
|                     _logger.LogWarning("Skipping advisory with missing key for document {DocumentId} ({Uri})", document.Id, document.Uri); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var previous = await _advisoryStore.FindAsync(advisory.AdvisoryKey, cancellationToken).ConfigureAwait(false); | ||||
|                 await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|                 if (previous is not null) | ||||
|                 { | ||||
|                     await RecordChangeHistoryAsync(advisory, previous, document, now, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 mappedCount++; | ||||
|             } | ||||
|  | ||||
|             if (mappedCount > 0) | ||||
|             { | ||||
|                 _diagnostics.MapSuccess(mappedCount); | ||||
|             } | ||||
|  | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|             pendingMapping.Remove(documentId); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMapping); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private async Task<IReadOnlyCollection<Guid>> FetchAdditionalPagesAsync( | ||||
|         TimeWindow window, | ||||
|         IReadOnlyDictionary<string, string> baseMetadata, | ||||
|         DocumentRecord firstDocument, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (firstDocument.GridFsId is null) | ||||
|         { | ||||
|             return Array.Empty<Guid>(); | ||||
|         } | ||||
|  | ||||
|         byte[] rawBytes; | ||||
|         try | ||||
|         { | ||||
|             rawBytes = await _rawDocumentStorage.DownloadAsync(firstDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _logger.LogWarning(ex, "Unable to download NVD first page {DocumentId} to evaluate pagination", firstDocument.Id); | ||||
|             return Array.Empty<Guid>(); | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             using var jsonDocument = JsonDocument.Parse(rawBytes); | ||||
|             var root = jsonDocument.RootElement; | ||||
|  | ||||
|             if (!TryReadInt32(root, "totalResults", out var totalResults) || !TryReadInt32(root, "resultsPerPage", out var resultsPerPage)) | ||||
|             { | ||||
|                 return Array.Empty<Guid>(); | ||||
|             } | ||||
|  | ||||
|             if (resultsPerPage <= 0 || totalResults <= resultsPerPage) | ||||
|             { | ||||
|                 return Array.Empty<Guid>(); | ||||
|             } | ||||
|  | ||||
|             var fetchedDocuments = new List<Guid>(); | ||||
|  | ||||
|             foreach (var startIndex in PaginationPlanner.EnumerateAdditionalPages(totalResults, resultsPerPage)) | ||||
|             { | ||||
|                 var metadata = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|                 foreach (var kvp in baseMetadata) | ||||
|                 { | ||||
|                     metadata[kvp.Key] = kvp.Value; | ||||
|                 } | ||||
|                 metadata["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); | ||||
|  | ||||
|                 var request = new SourceFetchRequest( | ||||
|                     NvdOptions.HttpClientName, | ||||
|                     SourceName, | ||||
|                     BuildRequestUri(window, startIndex)) | ||||
|                 { | ||||
|                     Metadata = metadata | ||||
|                 }; | ||||
|  | ||||
|                 SourceFetchResult pageResult; | ||||
|                 try | ||||
|                 { | ||||
|                     _diagnostics.FetchAttempt(); | ||||
|                     pageResult = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 catch (Exception ex) | ||||
|                 { | ||||
|                     _diagnostics.FetchFailure(); | ||||
|                     _logger.LogError(ex, "NVD fetch failed for page starting at {StartIndex}", startIndex); | ||||
|                     throw; | ||||
|                 } | ||||
|  | ||||
|                 if (pageResult.IsNotModified) | ||||
|                 { | ||||
|                     _diagnostics.FetchUnchanged(); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!pageResult.IsSuccess || pageResult.Document is null) | ||||
|                 { | ||||
|                     _diagnostics.FetchFailure(); | ||||
|                     _logger.LogWarning("NVD fetch for page starting at {StartIndex} returned status {Status}", startIndex, pageResult.StatusCode); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 _diagnostics.FetchDocument(); | ||||
|                 fetchedDocuments.Add(pageResult.Document.Id); | ||||
|             } | ||||
|  | ||||
|             return fetchedDocuments; | ||||
|         } | ||||
|         catch (JsonException ex) | ||||
|         { | ||||
|             _logger.LogWarning(ex, "Failed to parse NVD first page {DocumentId} while determining pagination", firstDocument.Id); | ||||
|             return Array.Empty<Guid>(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static bool TryReadInt32(JsonElement root, string propertyName, out int value) | ||||
|     { | ||||
|         value = 0; | ||||
|         if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.Number) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (property.TryGetInt32(out var intValue)) | ||||
|         { | ||||
|             value = intValue; | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (property.TryGetInt64(out var longValue)) | ||||
|         { | ||||
|             if (longValue > int.MaxValue) | ||||
|             { | ||||
|                 value = int.MaxValue; | ||||
|                 return true; | ||||
|             } | ||||
|  | ||||
|             value = (int)longValue; | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
|  | ||||
|     private async Task RecordChangeHistoryAsync( | ||||
|         Advisory current, | ||||
|         Advisory previous, | ||||
|         DocumentRecord document, | ||||
|         DateTimeOffset capturedAt, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (current.Equals(previous)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var currentSnapshot = SnapshotSerializer.ToSnapshot(current); | ||||
|         var previousSnapshot = SnapshotSerializer.ToSnapshot(previous); | ||||
|  | ||||
|         if (string.Equals(currentSnapshot, previousSnapshot, StringComparison.Ordinal)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var changes = ComputeChanges(previousSnapshot, currentSnapshot); | ||||
|         if (changes.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var documentHash = string.IsNullOrWhiteSpace(document.Sha256) | ||||
|             ? ComputeHash(currentSnapshot) | ||||
|             : document.Sha256; | ||||
|  | ||||
|         var record = new ChangeHistoryRecord( | ||||
|             Guid.NewGuid(), | ||||
|             SourceName, | ||||
|             current.AdvisoryKey, | ||||
|             document.Id, | ||||
|             documentHash, | ||||
|             ComputeHash(currentSnapshot), | ||||
|             ComputeHash(previousSnapshot), | ||||
|             currentSnapshot, | ||||
|             previousSnapshot, | ||||
|             changes, | ||||
|             capturedAt); | ||||
|  | ||||
|         await _changeHistoryStore.AddAsync(record, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<ChangeHistoryFieldChange> ComputeChanges(string previousSnapshot, string currentSnapshot) | ||||
|     { | ||||
|         using var previousDocument = JsonDocument.Parse(previousSnapshot); | ||||
|         using var currentDocument = JsonDocument.Parse(currentSnapshot); | ||||
|  | ||||
|         var previousRoot = previousDocument.RootElement; | ||||
|         var currentRoot = currentDocument.RootElement; | ||||
|         var fields = new HashSet<string>(StringComparer.Ordinal); | ||||
|  | ||||
|         foreach (var property in previousRoot.EnumerateObject()) | ||||
|         { | ||||
|             fields.Add(property.Name); | ||||
|         } | ||||
|  | ||||
|         foreach (var property in currentRoot.EnumerateObject()) | ||||
|         { | ||||
|             fields.Add(property.Name); | ||||
|         } | ||||
|  | ||||
|         var changes = new List<ChangeHistoryFieldChange>(); | ||||
|         foreach (var field in fields.OrderBy(static name => name, StringComparer.Ordinal)) | ||||
|         { | ||||
|             var hasPrevious = previousRoot.TryGetProperty(field, out var previousValue); | ||||
|             var hasCurrent = currentRoot.TryGetProperty(field, out var currentValue); | ||||
|  | ||||
|             if (!hasPrevious && hasCurrent) | ||||
|             { | ||||
|                 changes.Add(new ChangeHistoryFieldChange(field, "Added", null, SerializeElement(currentValue))); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (hasPrevious && !hasCurrent) | ||||
|             { | ||||
|                 changes.Add(new ChangeHistoryFieldChange(field, "Removed", SerializeElement(previousValue), null)); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (hasPrevious && hasCurrent && !JsonElement.DeepEquals(previousValue, currentValue)) | ||||
|             { | ||||
|                 changes.Add(new ChangeHistoryFieldChange(field, "Modified", SerializeElement(previousValue), SerializeElement(currentValue))); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return changes; | ||||
|     } | ||||
|  | ||||
|     private static string SerializeElement(JsonElement element) | ||||
|         => JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false }); | ||||
|  | ||||
|     private static string ComputeHash(string snapshot) | ||||
|     { | ||||
|         var bytes = Encoding.UTF8.GetBytes(snapshot); | ||||
|         var hash = SHA256.HashData(bytes); | ||||
|         return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; | ||||
|     } | ||||
|  | ||||
|     private async Task<NvdCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return NvdCursor.FromBsonDocument(record?.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(NvdCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var completedAt = _timeProvider.GetUtcNow(); | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private Uri BuildRequestUri(TimeWindow window, int startIndex = 0) | ||||
|     { | ||||
|         var builder = new UriBuilder(_options.BaseEndpoint); | ||||
|  | ||||
|         var parameters = new Dictionary<string, string> | ||||
|         { | ||||
|             ["lastModifiedStartDate"] = window.Start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), | ||||
|             ["lastModifiedEndDate"] = window.End.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), | ||||
|             ["resultsPerPage"] = "2000", | ||||
|         }; | ||||
|  | ||||
|         if (startIndex > 0) | ||||
|         { | ||||
|             parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); | ||||
|         } | ||||
|  | ||||
|         builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}")); | ||||
|         return builder.Uri; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										19
									
								
								src/StellaOps.Concelier.Connector.Nvd/NvdConnectorPlugin.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										19
									
								
								src/StellaOps.Concelier.Connector.Nvd/NvdConnectorPlugin.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,19 @@ | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd; | ||||
|  | ||||
| public sealed class NvdConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public static string SourceName => "nvd"; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => services is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return ActivatorUtilities.CreateInstance<NvdConnector>(services); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,35 @@ | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Common.Http; | ||||
| using StellaOps.Concelier.Connector.Nvd.Configuration; | ||||
| using StellaOps.Concelier.Connector.Nvd.Internal; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Nvd; | ||||
|  | ||||
| public static class NvdServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddNvdConnector(this IServiceCollection services, Action<NvdOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<NvdOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static opts => opts.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(NvdOptions.HttpClientName, (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<NvdOptions>>().Value; | ||||
|             clientOptions.BaseAddress = options.BaseEndpoint; | ||||
|             clientOptions.Timeout = TimeSpan.FromSeconds(30); | ||||
|             clientOptions.UserAgent = "StellaOps.Concelier.Nvd/1.0"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); | ||||
|             clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; | ||||
|         }); | ||||
|  | ||||
|         services.AddSingleton<NvdDiagnostics>(); | ||||
|         services.AddTransient<NvdConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,4 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Nvd.Tests")] | ||||
| [assembly: InternalsVisibleTo("FixtureUpdater")] | ||||
| @@ -0,0 +1,115 @@ | ||||
| { | ||||
|   "$schema": "https://json-schema.org/draft/2020-12/schema", | ||||
|   "type": "object", | ||||
|   "required": ["vulnerabilities"], | ||||
|   "properties": { | ||||
|     "resultsPerPage": { "type": "integer", "minimum": 0 }, | ||||
|     "startIndex": { "type": "integer", "minimum": 0 }, | ||||
|     "totalResults": { "type": "integer", "minimum": 0 }, | ||||
|     "vulnerabilities": { | ||||
|       "type": "array", | ||||
|       "items": { | ||||
|         "type": "object", | ||||
|         "required": ["cve"], | ||||
|         "properties": { | ||||
|           "cve": { | ||||
|             "type": "object", | ||||
|             "required": ["id", "published", "lastModified", "descriptions"], | ||||
|             "properties": { | ||||
|               "id": { "type": "string" }, | ||||
|               "published": { "type": "string", "format": "date-time" }, | ||||
|               "lastModified": { "type": "string", "format": "date-time" }, | ||||
|               "vulnStatus": { "type": "string" }, | ||||
|               "sourceIdentifier": { "type": "string" }, | ||||
|               "descriptions": { | ||||
|                 "type": "array", | ||||
|                 "items": { | ||||
|                   "type": "object", | ||||
|                   "required": ["lang", "value"], | ||||
|                   "properties": { | ||||
|                     "lang": { "type": "string" }, | ||||
|                     "value": { "type": "string" } | ||||
|                   } | ||||
|                 } | ||||
|               }, | ||||
|               "references": { | ||||
|                 "type": "array", | ||||
|                 "items": { | ||||
|                   "type": "object", | ||||
|                   "required": ["url"], | ||||
|                   "properties": { | ||||
|                     "url": { "type": "string", "format": "uri" }, | ||||
|                     "source": { "type": "string" }, | ||||
|                     "tags": { | ||||
|                       "type": "array", | ||||
|                       "items": { "type": "string" } | ||||
|                     } | ||||
|                   } | ||||
|                 } | ||||
|               }, | ||||
|               "metrics": { | ||||
|                 "type": "object", | ||||
|                 "properties": { | ||||
|                   "cvssMetricV2": { "$ref": "#/definitions/cvssMetricArray" }, | ||||
|                   "cvssMetricV30": { "$ref": "#/definitions/cvssMetricArray" }, | ||||
|                   "cvssMetricV31": { "$ref": "#/definitions/cvssMetricArray" } | ||||
|                 } | ||||
|               }, | ||||
|               "configurations": { | ||||
|                 "type": "object", | ||||
|                 "properties": { | ||||
|                   "nodes": { | ||||
|                     "type": "array", | ||||
|                     "items": { | ||||
|                       "type": "object", | ||||
|                       "properties": { | ||||
|                         "cpeMatch": { | ||||
|                           "type": "array", | ||||
|                           "items": { | ||||
|                             "type": "object", | ||||
|                             "properties": { | ||||
|                               "vulnerable": { "type": "boolean" }, | ||||
|                               "criteria": { "type": "string" } | ||||
|                             }, | ||||
|                             "required": ["criteria"], | ||||
|                             "additionalProperties": true | ||||
|                           } | ||||
|                         } | ||||
|                       }, | ||||
|                       "additionalProperties": true | ||||
|                     } | ||||
|                   } | ||||
|                 }, | ||||
|                 "additionalProperties": true | ||||
|               } | ||||
|             }, | ||||
|             "additionalProperties": true | ||||
|           } | ||||
|         }, | ||||
|         "additionalProperties": true | ||||
|       } | ||||
|     } | ||||
|   }, | ||||
|   "additionalProperties": true, | ||||
|   "definitions": { | ||||
|     "cvssMetricArray": { | ||||
|       "type": "array", | ||||
|       "items": { | ||||
|         "type": "object", | ||||
|         "properties": { | ||||
|           "cvssData": { | ||||
|             "type": "object", | ||||
|             "required": ["vectorString", "baseScore", "baseSeverity"], | ||||
|             "properties": { | ||||
|               "vectorString": { "type": "string" }, | ||||
|               "baseScore": { "type": "number" }, | ||||
|               "baseSeverity": { "type": "string" } | ||||
|             }, | ||||
|             "additionalProperties": true | ||||
|           } | ||||
|         }, | ||||
|         "additionalProperties": true | ||||
|       } | ||||
|     } | ||||
|   } | ||||
| } | ||||
| @@ -0,0 +1,17 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|   <ItemGroup> | ||||
|     <EmbeddedResource Include="Schemas\nvd-vulnerability.schema.json" /> | ||||
|   </ItemGroup> | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|     <ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" /> | ||||
|     <ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" /> | ||||
|     <ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" /> | ||||
|     <ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
							
								
								
									
										18
									
								
								src/StellaOps.Concelier.Connector.Nvd/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								src/StellaOps.Concelier.Connector.Nvd/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,18 @@ | ||||
| # TASKS | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |Fetch job with sliding modified windows|BE-Conn-Nvd|Source.Common|**DONE** – windowed fetch implemented with overlap and raw doc persistence.| | ||||
| |DTO schema + validation|BE-Conn-Nvd|Source.Common|**DONE** – schema validator enforced before DTO persistence.| | ||||
| |Mapper to canonical model|BE-Conn-Nvd|Models|**DONE** – `NvdMapper` populates CVSS/CWE/CPE data.<br>2025-10-11 research trail: upcoming normalized rules must serialize as `[{"scheme":"semver","type":"range","min":"<floor>","minInclusive":true,"max":"<ceiling>","maxInclusive":false,"notes":"nvd:CVE-2025-XXXX"}]`; keep notes consistent with CVE IDs for provenance joins.| | ||||
| |Watermark repo usage|BE-Conn-Nvd|Storage.Mongo|**DONE** – cursor tracks windowStart/windowEnd and updates SourceState.| | ||||
| |Integration test fixture isolation|QA|Storage.Mongo|**DONE** – connector tests reset Mongo/time fixtures between runs to avoid cross-test bleed.| | ||||
| |Tests: golden pages + resume|QA|Tests|**DONE** – snapshot and resume coverage added across `NvdConnectorTests`.| | ||||
| |Observability|BE-Conn-Nvd|Core|**DONE** – `NvdDiagnostics` meter tracks attempts/documents/failures with collector tests.| | ||||
| |Change history snapshotting|BE-Conn-Nvd|Storage.Mongo|DONE – connector now records per-CVE snapshots with top-level diff metadata whenever canonical advisories change.| | ||||
| |Pagination for windows over page limit|BE-Conn-Nvd|Source.Common|**DONE** – additional page fetcher honors `startIndex`; covered by multipage tests.| | ||||
| |Schema validation quarantine path|BE-Conn-Nvd|Storage.Mongo|**DONE** – schema failures mark documents failed and metrics assert quarantine.| | ||||
| |FEEDCONN-NVD-04-002 Conflict regression fixtures|BE-Conn-Nvd, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Published `conflict-nvd.canonical.json` + mapper test; includes CVSS 3.1 + CWE reference and normalized CPE range feeding the conflict triple. Validation: `dotnet test src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj --filter NvdConflictFixtureTests`.| | ||||
| |FEEDCONN-NVD-02-004 NVD CVSS & CWE precedence payloads|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – CVSS metrics now carry provenance masks, CWE weaknesses emit normalized references, and fixtures cover the additional precedence data.| | ||||
| |FEEDCONN-NVD-02-005 NVD merge/export parity regression|BE-Conn-Nvd, BE-Merge|Merge `FEEDMERGE-ENGINE-04-003`|**DONE (2025-10-12)** – Canonical merge parity fixtures captured, regression test validates credit/reference union, and exporter snapshot check guarantees parity through JSON exports.| | ||||
| |FEEDCONN-NVD-02-002 Normalized versions rollout|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** – SemVer primitives + normalized rules emitting for parseable ranges, fixtures/tests refreshed, coordination pinged via FEEDMERGE-COORD-02-900.| | ||||
| |FEEDCONN-NVD-04-003 Description/CWE/metric parity rollout|BE-Conn-Nvd|Models, Core|**DONE (2025-10-15)** – Mapper now surfaces normalized description text, CWE weaknesses, and canonical CVSS metric id. Snapshots (`conflict-nvd.canonical.json`) refreshed and completion relayed to Merge coordination.| | ||||
		Reference in New Issue
	
	Block a user