Rename Concelier Source modules to Connector
This commit is contained in:
master
38
src/StellaOps.Concelier.Connector.Cve/AGENTS.md
Normal file
38
src/StellaOps.Concelier.Connector.Cve/AGENTS.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Create a dedicated CVE connector when we need raw CVE stream ingestion outside of NVD/OSV/National feeds (e.g., CVE JSON 5 API or CNA disclosures).
|
||||
|
||||
## Scope
|
||||
- Determine whether this connector should consume the official CVE JSON 5 API, CNA disclosures, or another stream.
|
||||
- Implement fetch/windowing aligned with CVE publication cadence; manage cursors for incremental backfills.
|
||||
- Parse CVE payloads into DTOs capturing descriptions, affected vendors/products, references, and metrics.
|
||||
- Map CVEs into canonical `Advisory` records (aliases, references, affected packages, range primitives).
|
||||
- Deliver deterministic fixtures/tests for fetch/parse/map lifecycle.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP/fetch utilities, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores & source state).
|
||||
- `Concelier.Models` (canonical data model).
|
||||
- `Concelier.Testing` (integration fixtures, snapshot helpers).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `cve:fetch`, `cve:parse`, `cve:map`.
|
||||
- Persist upstream metadata (e.g., `If-Modified-Since`, `cveMetadataDate`) for incremental fetching.
|
||||
- Aliases must include primary CVE ID along with CNA-specific identifiers when available.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- Core pipeline for CVE ingestion with provenance/range primitives.
|
||||
|
||||
Out of scope:
|
||||
- Downstream impact scoring or enrichment (handled by other teams).
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch batch sizes, update timestamps, and mapping counts.
|
||||
- Handle rate limits politely with exponential backoff.
|
||||
- Sanitize and validate payloads before persistence.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Cve.Tests` with canned CVE JSON fixtures covering fetch/parse/map.
|
||||
- Snapshot canonical advisories; include env flag for fixture regeneration.
|
||||
- Ensure deterministic ordering and timestamp handling.
|
||||
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Configuration;
|
||||
|
||||
public sealed class CveOptions
|
||||
{
|
||||
public static string HttpClientName => "source.cve";
|
||||
|
||||
public Uri BaseEndpoint { get; set; } = new("https://cveawg.mitre.org/api/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// CVE Services requires an organisation identifier for authenticated requests.
|
||||
/// </summary>
|
||||
public string ApiOrg { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// CVE Services user identifier. Typically the username registered with the CVE Program.
|
||||
/// </summary>
|
||||
public string ApiUser { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// API key issued by the CVE Program for the configured organisation/user pair.
|
||||
/// </summary>
|
||||
public string ApiKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Optional path containing seed CVE JSON documents used when live credentials are unavailable.
|
||||
/// </summary>
|
||||
public string? SeedDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Results fetched per page when querying CVE Services. Valid range 1-500.
|
||||
/// </summary>
|
||||
public int PageSize { get; set; } = 200;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of pages to fetch in a single run. Guards against runaway backfills.
|
||||
/// </summary>
|
||||
public int MaxPagesPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Sliding look-back window when no previous cursor is available.
|
||||
/// </summary>
|
||||
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay between paginated requests to respect API throttling guidance.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the connector encounters an unrecoverable failure.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(10);
|
||||
|
||||
[MemberNotNull(nameof(BaseEndpoint))]
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("BaseEndpoint must be an absolute URI.");
|
||||
}
|
||||
|
||||
var hasCredentials = !string.IsNullOrWhiteSpace(ApiOrg)
|
||||
&& !string.IsNullOrWhiteSpace(ApiUser)
|
||||
&& !string.IsNullOrWhiteSpace(ApiKey);
|
||||
var hasSeedDirectory = !string.IsNullOrWhiteSpace(SeedDirectory);
|
||||
|
||||
if (!hasCredentials && !hasSeedDirectory)
|
||||
{
|
||||
throw new InvalidOperationException("Api credentials must be provided unless a SeedDirectory is configured.");
|
||||
}
|
||||
|
||||
if (hasCredentials && string.IsNullOrWhiteSpace(ApiOrg))
|
||||
{
|
||||
throw new InvalidOperationException("ApiOrg must be provided.");
|
||||
}
|
||||
|
||||
if (hasCredentials && string.IsNullOrWhiteSpace(ApiUser))
|
||||
{
|
||||
throw new InvalidOperationException("ApiUser must be provided.");
|
||||
}
|
||||
|
||||
if (hasCredentials && string.IsNullOrWhiteSpace(ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("ApiKey must be provided.");
|
||||
}
|
||||
|
||||
if (hasSeedDirectory && !Directory.Exists(SeedDirectory!))
|
||||
{
|
||||
throw new InvalidOperationException($"SeedDirectory '{SeedDirectory}' does not exist.");
|
||||
}
|
||||
|
||||
if (PageSize is < 1 or > 500)
|
||||
{
|
||||
throw new InvalidOperationException("PageSize must be between 1 and 500.");
|
||||
}
|
||||
|
||||
if (MaxPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxPagesPerFetch must be a positive integer.");
|
||||
}
|
||||
|
||||
if (InitialBackfill < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("InitialBackfill cannot be negative.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("FailureBackoff must be greater than zero.");
|
||||
}
|
||||
}
|
||||
|
||||
public bool HasCredentials()
|
||||
=> !string.IsNullOrWhiteSpace(ApiOrg)
|
||||
&& !string.IsNullOrWhiteSpace(ApiUser)
|
||||
&& !string.IsNullOrWhiteSpace(ApiKey);
|
||||
}
|
||||
609
src/StellaOps.Concelier.Connector.Cve/CveConnector.cs
Normal file
609
src/StellaOps.Concelier.Connector.Cve/CveConnector.cs
Normal file
@@ -0,0 +1,609 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Text;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Cve.Configuration;
|
||||
using StellaOps.Concelier.Connector.Cve.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve;
|
||||
|
||||
public sealed class CveConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly CveOptions _options;
|
||||
private readonly CveDiagnostics _diagnostics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<CveConnector> _logger;
|
||||
|
||||
public CveConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<CveOptions> options,
|
||||
CveDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<CveConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => CveConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!_options.HasCredentials())
|
||||
{
|
||||
if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning("CVEs fetch skipped: no credentials configured and no seed data found at {SeedDirectory}.", _options.SeedDirectory ?? "(seed directory not configured)");
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var initialPendingDocuments = pendingDocuments.Count;
|
||||
var initialPendingMappings = pendingMappings.Count;
|
||||
var documentsFetched = 0;
|
||||
var detailFailures = 0;
|
||||
var detailUnchanged = 0;
|
||||
var listSuccessCount = 0;
|
||||
var listUnchangedCount = 0;
|
||||
|
||||
var since = cursor.CurrentWindowStart ?? cursor.LastModifiedExclusive ?? now - _options.InitialBackfill;
|
||||
if (since > now)
|
||||
{
|
||||
since = now;
|
||||
}
|
||||
|
||||
var windowEnd = cursor.CurrentWindowEnd ?? now;
|
||||
if (windowEnd <= since)
|
||||
{
|
||||
windowEnd = since + TimeSpan.FromMinutes(1);
|
||||
}
|
||||
|
||||
var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage;
|
||||
var pagesFetched = 0;
|
||||
var hasMorePages = true;
|
||||
DateTimeOffset? maxModified = cursor.LastModifiedExclusive;
|
||||
|
||||
while (hasMorePages && pagesFetched < _options.MaxPagesPerFetch)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var requestUri = BuildListRequestUri(since, windowEnd, page, _options.PageSize);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["since"] = since.ToString("O"),
|
||||
["until"] = windowEnd.ToString("O"),
|
||||
["page"] = page.ToString(CultureInfo.InvariantCulture),
|
||||
["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture),
|
||||
};
|
||||
|
||||
SourceFetchContentResult listResult;
|
||||
try
|
||||
{
|
||||
_diagnostics.FetchAttempt();
|
||||
listResult = await _fetchService.FetchContentAsync(
|
||||
new SourceFetchRequest(
|
||||
CveOptions.HttpClientName,
|
||||
SourceName,
|
||||
requestUri)
|
||||
{
|
||||
Metadata = metadata,
|
||||
AcceptHeaders = new[] { "application/json" },
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (HttpRequestException ex) when (IsAuthenticationFailure(ex))
|
||||
{
|
||||
_logger.LogWarning("CVEs fetch requires API credentials ({StatusCode}); falling back to seed data if available.", ex.StatusCode);
|
||||
if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning("CVEs fetch aborted: no seed data available (SeedDirectory={SeedDirectory}).", _options.SeedDirectory ?? "(seed directory not configured)");
|
||||
return;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (listResult.IsNotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
listUnchangedCount++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!listResult.IsSuccess || listResult.Content is null)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
break;
|
||||
}
|
||||
|
||||
_diagnostics.FetchSuccess();
|
||||
listSuccessCount++;
|
||||
|
||||
var pageModel = CveListParser.Parse(listResult.Content, page, _options.PageSize);
|
||||
|
||||
if (pageModel.Items.Count == 0)
|
||||
{
|
||||
hasMorePages = false;
|
||||
}
|
||||
|
||||
foreach (var item in pageModel.Items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var detailUri = BuildDetailRequestUri(item.CveId);
|
||||
var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cveId"] = item.CveId,
|
||||
["page"] = page.ToString(CultureInfo.InvariantCulture),
|
||||
["since"] = since.ToString("O"),
|
||||
["until"] = windowEnd.ToString("O"),
|
||||
};
|
||||
|
||||
SourceFetchResult detailResult;
|
||||
try
|
||||
{
|
||||
detailResult = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(
|
||||
CveOptions.HttpClientName,
|
||||
SourceName,
|
||||
detailUri)
|
||||
{
|
||||
Metadata = detailMetadata,
|
||||
AcceptHeaders = new[] { "application/json" },
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (HttpRequestException ex) when (IsAuthenticationFailure(ex))
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogWarning(ex, "Failed fetching CVE record {CveId} due to authentication. Seeding if possible.", item.CveId);
|
||||
if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning("CVE record {CveId} skipped; missing credentials and no seed data available.", item.CveId);
|
||||
continue;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogWarning(ex, "Failed fetching CVE record {CveId}", item.CveId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (detailResult.IsNotModified)
|
||||
{
|
||||
_diagnostics.FetchUnchanged();
|
||||
detailUnchanged++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!detailResult.IsSuccess || detailResult.Document is null)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
detailFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.FetchDocument();
|
||||
if (pendingDocuments.Add(detailResult.Document.Id))
|
||||
{
|
||||
documentsFetched++;
|
||||
}
|
||||
pendingMappings.Add(detailResult.Document.Id);
|
||||
}
|
||||
|
||||
if (pageModel.MaxModified.HasValue)
|
||||
{
|
||||
if (!maxModified.HasValue || pageModel.MaxModified > maxModified)
|
||||
{
|
||||
maxModified = pageModel.MaxModified;
|
||||
}
|
||||
}
|
||||
|
||||
hasMorePages = pageModel.HasMorePages;
|
||||
page = pageModel.NextPageCandidate;
|
||||
pagesFetched++;
|
||||
|
||||
if (hasMorePages && _options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
if (hasMorePages)
|
||||
{
|
||||
updatedCursor = updatedCursor
|
||||
.WithCurrentWindowStart(since)
|
||||
.WithCurrentWindowEnd(windowEnd)
|
||||
.WithNextPage(page);
|
||||
}
|
||||
else
|
||||
{
|
||||
var nextSince = maxModified ?? windowEnd;
|
||||
updatedCursor = updatedCursor
|
||||
.WithLastModifiedExclusive(nextSince)
|
||||
.WithCurrentWindowStart(null)
|
||||
.WithCurrentWindowEnd(null)
|
||||
.WithNextPage(1);
|
||||
}
|
||||
|
||||
var nextWindowStart = hasMorePages ? since : maxModified ?? windowEnd;
|
||||
DateTimeOffset? nextWindowEnd = hasMorePages ? windowEnd : null;
|
||||
var nextPage = hasMorePages ? page : 1;
|
||||
var windowStartString = since.ToString("O");
|
||||
var windowEndString = windowEnd.ToString("O");
|
||||
var nextWindowStartString = nextWindowStart.ToString("O");
|
||||
var nextWindowEndString = nextWindowEnd?.ToString("O") ?? "(none)";
|
||||
|
||||
_logger.LogInformation(
|
||||
"CVEs fetch window {WindowStart}->{WindowEnd} pages={PagesFetched} listSuccess={ListSuccess} detailDocuments={DocumentsFetched} detailFailures={DetailFailures} detailUnchanged={DetailUnchanged} pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter} hasMorePages={HasMorePages} nextWindowStart={NextWindowStart} nextWindowEnd={NextWindowEnd} nextPage={NextPage}",
|
||||
windowStartString,
|
||||
windowEndString,
|
||||
pagesFetched,
|
||||
listSuccessCount,
|
||||
documentsFetched,
|
||||
detailFailures,
|
||||
detailUnchanged,
|
||||
initialPendingDocuments,
|
||||
pendingDocuments.Count,
|
||||
initialPendingMappings,
|
||||
pendingMappings.Count,
|
||||
hasMorePages,
|
||||
nextWindowStartString,
|
||||
nextWindowEndString,
|
||||
nextPage);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning("CVEs document {DocumentId} missing GridFS content", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogError(ex, "Unable to download CVE raw document {DocumentId}", documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
CveRecordDto dto;
|
||||
try
|
||||
{
|
||||
dto = CveRecordParser.Parse(rawBytes);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_diagnostics.ParseQuarantine();
|
||||
_logger.LogError(ex, "Malformed CVE JSON for {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
"cve/5.0",
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
_diagnostics.ParseSuccess();
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dtoRecord is null || document is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping CVE mapping for {DocumentId}: DTO or document missing", documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
CveRecordDto dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<CveRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions)
|
||||
?? throw new InvalidOperationException("Deserialized DTO was null.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize CVE DTO for {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var recordedAt = dtoRecord.ValidatedAt;
|
||||
var advisory = CveMapper.Map(dto, document, recordedAt);
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapSuccess(1);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<bool> TrySeedFromDirectoryAsync(CveCursor cursor, DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var seedDirectory = _options.SeedDirectory;
|
||||
if (string.IsNullOrWhiteSpace(seedDirectory) || !Directory.Exists(seedDirectory))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var detailFiles = Directory.EnumerateFiles(seedDirectory, "CVE-*.json", SearchOption.AllDirectories)
|
||||
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
if (detailFiles.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var seeded = 0;
|
||||
DateTimeOffset? maxModified = cursor.LastModifiedExclusive;
|
||||
|
||||
foreach (var file in detailFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await File.ReadAllBytesAsync(file, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to read CVE seed file {File}", file);
|
||||
continue;
|
||||
}
|
||||
|
||||
CveRecordDto dto;
|
||||
try
|
||||
{
|
||||
dto = CveRecordParser.Parse(payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Seed file {File} did not contain a valid CVE record", file);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(dto.CveId))
|
||||
{
|
||||
_logger.LogWarning("Seed file {File} missing CVE identifier", file);
|
||||
continue;
|
||||
}
|
||||
|
||||
var uri = $"seed://{dto.CveId}";
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri, cancellationToken).ConfigureAwait(false);
|
||||
var documentId = existing?.Id ?? Guid.NewGuid();
|
||||
|
||||
var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
var lastModified = dto.Modified ?? dto.Published ?? now;
|
||||
ObjectId gridId = ObjectId.Empty;
|
||||
|
||||
try
|
||||
{
|
||||
if (existing?.GridFsId is ObjectId existingGrid && existingGrid != ObjectId.Empty)
|
||||
{
|
||||
gridId = existingGrid;
|
||||
}
|
||||
else
|
||||
{
|
||||
gridId = await _rawDocumentStorage.UploadAsync(SourceName, uri, payload, "application/json", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to store CVE seed payload for {CveId}", dto.CveId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["seed.file"] = Path.GetFileName(file),
|
||||
["seed.directory"] = seedDirectory,
|
||||
};
|
||||
|
||||
var document = new DocumentRecord(
|
||||
documentId,
|
||||
SourceName,
|
||||
uri,
|
||||
now,
|
||||
sha256,
|
||||
DocumentStatuses.Mapped,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: lastModified,
|
||||
GridFsId: gridId);
|
||||
|
||||
await _documentStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var advisory = CveMapper.Map(dto, document, now);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!maxModified.HasValue || lastModified > maxModified)
|
||||
{
|
||||
maxModified = lastModified;
|
||||
}
|
||||
|
||||
seeded++;
|
||||
}
|
||||
|
||||
if (seeded == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(Array.Empty<Guid>())
|
||||
.WithPendingMappings(Array.Empty<Guid>())
|
||||
.WithLastModifiedExclusive(maxModified ?? now)
|
||||
.WithCurrentWindowStart(null)
|
||||
.WithCurrentWindowEnd(null)
|
||||
.WithNextPage(1);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogWarning("Seeded {SeededCount} CVE advisories from {SeedDirectory}; live fetch will resume when credentials are configured.", seeded, seedDirectory);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool IsAuthenticationFailure(HttpRequestException exception)
|
||||
=> exception.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden;
|
||||
|
||||
private async Task<CveCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? CveCursor.Empty : CveCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(CveCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Uri BuildListRequestUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize)
|
||||
{
|
||||
var query = $"time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(until.ToString("O"))}&page={page}&size={pageSize}";
|
||||
return new Uri($"cve?{query}", UriKind.Relative);
|
||||
}
|
||||
|
||||
private static Uri BuildDetailRequestUri(string cveId)
|
||||
{
|
||||
var encoded = Uri.EscapeDataString(cveId);
|
||||
return new Uri($"cve/{encoded}", UriKind.Relative);
|
||||
}
|
||||
}
|
||||
19
src/StellaOps.Concelier.Connector.Cve/CveConnectorPlugin.cs
Normal file
19
src/StellaOps.Concelier.Connector.Cve/CveConnectorPlugin.cs
Normal file
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve;
|
||||
|
||||
public sealed class CveConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cve";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<CveConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Cve.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve;
|
||||
|
||||
public sealed class CveDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:cve";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCveConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CveFetchJob>();
|
||||
services.AddTransient<CveParseJob>();
|
||||
services.AddTransient<CveMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CveJobKinds.Fetch, typeof(CveFetchJob));
|
||||
EnsureJob(options, CveJobKinds.Parse, typeof(CveParseJob));
|
||||
EnsureJob(options, CveJobKinds.Map, typeof(CveMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Cve.Configuration;
|
||||
using StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve;
|
||||
|
||||
public static class CveServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCveConnector(this IServiceCollection services, Action<CveOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CveOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CveOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CveOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseEndpoint;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.Cve/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
|
||||
if (options.HasCredentials())
|
||||
{
|
||||
clientOptions.DefaultRequestHeaders["CVE-API-ORG"] = options.ApiOrg;
|
||||
clientOptions.DefaultRequestHeaders["CVE-API-USER"] = options.ApiUser;
|
||||
clientOptions.DefaultRequestHeaders["CVE-API-KEY"] = options.ApiKey;
|
||||
}
|
||||
});
|
||||
|
||||
services.AddSingleton<CveDiagnostics>();
|
||||
services.AddTransient<CveConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
135
src/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs
Normal file
135
src/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs
Normal file
@@ -0,0 +1,135 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
internal sealed record CveCursor(
|
||||
DateTimeOffset? LastModifiedExclusive,
|
||||
DateTimeOffset? CurrentWindowStart,
|
||||
DateTimeOffset? CurrentWindowEnd,
|
||||
int NextPage,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
|
||||
public static CveCursor Empty { get; } = new(
|
||||
LastModifiedExclusive: null,
|
||||
CurrentWindowStart: null,
|
||||
CurrentWindowEnd: null,
|
||||
NextPage: 1,
|
||||
PendingDocuments: EmptyGuidList,
|
||||
PendingMappings: EmptyGuidList);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["nextPage"] = NextPage,
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastModifiedExclusive.HasValue)
|
||||
{
|
||||
document["lastModifiedExclusive"] = LastModifiedExclusive.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (CurrentWindowStart.HasValue)
|
||||
{
|
||||
document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (CurrentWindowEnd.HasValue)
|
||||
{
|
||||
document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CveCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastModifiedExclusive = document.TryGetValue("lastModifiedExclusive", out var lastModifiedValue)
|
||||
? ParseDate(lastModifiedValue)
|
||||
: null;
|
||||
var currentWindowStart = document.TryGetValue("currentWindowStart", out var windowStartValue)
|
||||
? ParseDate(windowStartValue)
|
||||
: null;
|
||||
var currentWindowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue)
|
||||
? ParseDate(windowEndValue)
|
||||
: null;
|
||||
var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32
|
||||
? Math.Max(1, nextPageValue.AsInt32)
|
||||
: 1;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
|
||||
return new CveCursor(
|
||||
LastModifiedExclusive: lastModifiedExclusive,
|
||||
CurrentWindowStart: currentWindowStart,
|
||||
CurrentWindowEnd: currentWindowEnd,
|
||||
NextPage: nextPage,
|
||||
PendingDocuments: pendingDocuments,
|
||||
PendingMappings: pendingMappings);
|
||||
}
|
||||
|
||||
public CveCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public CveCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public CveCursor WithLastModifiedExclusive(DateTimeOffset? timestamp)
|
||||
=> this with { LastModifiedExclusive = timestamp };
|
||||
|
||||
public CveCursor WithCurrentWindowEnd(DateTimeOffset? timestamp)
|
||||
=> this with { CurrentWindowEnd = timestamp };
|
||||
|
||||
public CveCursor WithCurrentWindowStart(DateTimeOffset? timestamp)
|
||||
=> this with { CurrentWindowStart = timestamp };
|
||||
|
||||
public CveCursor WithNextPage(int page)
|
||||
=> this with { NextPage = page < 1 ? 1 : page };
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidList;
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
public sealed class CveDiagnostics : IDisposable
|
||||
{
|
||||
public const string MeterName = "StellaOps.Concelier.Connector.Cve";
|
||||
public const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchDocuments;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _parseQuarantine;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
|
||||
public CveDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "cve.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of CVE fetch operations attempted.");
|
||||
_fetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "cve.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of CVE fetch operations that completed successfully.");
|
||||
_fetchDocuments = _meter.CreateCounter<long>(
|
||||
name: "cve.fetch.documents",
|
||||
unit: "documents",
|
||||
description: "Count of CVE documents fetched and persisted.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "cve.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Count of CVE fetch attempts that resulted in an error.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "cve.fetch.unchanged",
|
||||
unit: "operations",
|
||||
description: "Count of CVE fetch attempts returning 304 Not Modified.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "cve.parse.success",
|
||||
unit: "documents",
|
||||
description: "Count of CVE documents successfully parsed into DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "cve.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Count of CVE documents that could not be parsed.");
|
||||
_parseQuarantine = _meter.CreateCounter<long>(
|
||||
name: "cve.parse.quarantine",
|
||||
unit: "documents",
|
||||
description: "Count of CVE documents quarantined after schema validation errors.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "cve.map.success",
|
||||
unit: "advisories",
|
||||
description: "Count of canonical advisories emitted by the CVE mapper.");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchDocument() => _fetchDocuments.Add(1);
|
||||
|
||||
public void FetchSuccess() => _fetchSuccess.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void ParseSuccess() => _parseSuccess.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void ParseQuarantine() => _parseQuarantine.Add(1);
|
||||
|
||||
public void MapSuccess(long count) => _mapSuccess.Add(count);
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
264
src/StellaOps.Concelier.Connector.Cve/Internal/CveListParser.cs
Normal file
264
src/StellaOps.Concelier.Connector.Cve/Internal/CveListParser.cs
Normal file
@@ -0,0 +1,264 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
internal static class CveListParser
|
||||
{
|
||||
public static CveListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize)
|
||||
{
|
||||
using var document = JsonDocument.Parse(content.ToArray());
|
||||
var root = document.RootElement;
|
||||
|
||||
var items = new List<CveListItem>();
|
||||
DateTimeOffset? maxModified = null;
|
||||
|
||||
foreach (var element in EnumerateItemElements(root))
|
||||
{
|
||||
var cveId = ExtractCveId(element);
|
||||
if (string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var modified = ExtractModified(element);
|
||||
if (modified.HasValue && (!maxModified.HasValue || modified > maxModified))
|
||||
{
|
||||
maxModified = modified;
|
||||
}
|
||||
|
||||
items.Add(new CveListItem(cveId, modified));
|
||||
}
|
||||
|
||||
var hasMore = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage);
|
||||
|
||||
return new CveListPage(items, maxModified, hasMore, nextPage ?? currentPage + 1);
|
||||
}
|
||||
|
||||
private static IEnumerable<JsonElement> EnumerateItemElements(JsonElement root)
|
||||
{
|
||||
if (root.TryGetProperty("data", out var dataElement) && dataElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in dataElement.EnumerateArray())
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("vulnerabilities", out var vulnerabilities) && vulnerabilities.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in vulnerabilities.EnumerateArray())
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (root.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var item in root.EnumerateArray())
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractCveId(JsonElement element)
|
||||
{
|
||||
if (element.TryGetProperty("cveId", out var cveId) && cveId.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return cveId.GetString();
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("cveMetadata", out var metadata))
|
||||
{
|
||||
if (metadata.TryGetProperty("cveId", out var metadataId) && metadataId.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return metadataId.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
if (cve.TryGetProperty("cveMetadata", out var nestedMeta) && nestedMeta.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
if (nestedMeta.TryGetProperty("cveId", out var nestedId) && nestedId.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return nestedId.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
if (cve.TryGetProperty("id", out var cveIdElement) && cveIdElement.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return cveIdElement.GetString();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ExtractModified(JsonElement element)
|
||||
{
|
||||
static DateTimeOffset? Parse(JsonElement candidate)
|
||||
{
|
||||
return candidate.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String when DateTimeOffset.TryParse(candidate.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
=> parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("dateUpdated", out var dateUpdated))
|
||||
{
|
||||
var parsed = Parse(dateUpdated);
|
||||
if (parsed.HasValue)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("cveMetadata", out var metadata))
|
||||
{
|
||||
if (metadata.TryGetProperty("dateUpdated", out var metadataUpdated))
|
||||
{
|
||||
var parsed = Parse(metadataUpdated);
|
||||
if (parsed.HasValue)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
if (cve.TryGetProperty("cveMetadata", out var nestedMeta))
|
||||
{
|
||||
if (nestedMeta.TryGetProperty("dateUpdated", out var nestedUpdated))
|
||||
{
|
||||
var parsed = Parse(nestedUpdated);
|
||||
if (parsed.HasValue)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cve.TryGetProperty("lastModified", out var lastModified))
|
||||
{
|
||||
var parsed = Parse(lastModified);
|
||||
if (parsed.HasValue)
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage)
|
||||
{
|
||||
nextPage = null;
|
||||
|
||||
if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
var totalPages = TryGetInt(pagination, "totalPages")
|
||||
?? TryGetInt(pagination, "pageCount")
|
||||
?? TryGetInt(pagination, "totalPagesCount");
|
||||
if (totalPages.HasValue)
|
||||
{
|
||||
if (currentPage < totalPages.Value)
|
||||
{
|
||||
nextPage = currentPage + 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
var totalCount = TryGetInt(pagination, "totalCount")
|
||||
?? TryGetInt(pagination, "totalResults");
|
||||
var limit = TryGetInt(pagination, "limit")
|
||||
?? TryGetInt(pagination, "itemsPerPage")
|
||||
?? TryGetInt(pagination, "pageSize")
|
||||
?? pageSize;
|
||||
|
||||
if (totalCount.HasValue)
|
||||
{
|
||||
var processed = (currentPage - 1) * limit + itemCount;
|
||||
if (processed < totalCount.Value)
|
||||
{
|
||||
nextPage = currentPage + 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (pagination.TryGetProperty("nextPage", out var nextPageElement))
|
||||
{
|
||||
switch (nextPageElement.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Number when nextPageElement.TryGetInt32(out var value):
|
||||
nextPage = value;
|
||||
return true;
|
||||
case JsonValueKind.String when int.TryParse(nextPageElement.GetString(), out var parsed):
|
||||
nextPage = parsed;
|
||||
return true;
|
||||
case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageElement.GetString()):
|
||||
nextPage = currentPage + 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (root.TryGetProperty("nextPage", out var nextPageValue))
|
||||
{
|
||||
switch (nextPageValue.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Number when nextPageValue.TryGetInt32(out var value):
|
||||
nextPage = value;
|
||||
return true;
|
||||
case JsonValueKind.String when int.TryParse(nextPageValue.GetString(), out var parsed):
|
||||
nextPage = parsed;
|
||||
return true;
|
||||
case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageValue.GetString()):
|
||||
nextPage = currentPage + 1;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (itemCount >= pageSize)
|
||||
{
|
||||
nextPage = currentPage + 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static int? TryGetInt(JsonElement element, string propertyName)
|
||||
{
|
||||
if (!element.TryGetProperty(propertyName, out var value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Number when value.TryGetInt32(out var number) => number,
|
||||
JsonValueKind.String when int.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CveListPage(
|
||||
IReadOnlyList<CveListItem> Items,
|
||||
DateTimeOffset? MaxModified,
|
||||
bool HasMorePages,
|
||||
int NextPageCandidate);
|
||||
|
||||
internal sealed record CveListItem(string CveId, DateTimeOffset? DateUpdated);
|
||||
451
src/StellaOps.Concelier.Connector.Cve/Internal/CveMapper.cs
Normal file
451
src/StellaOps.Concelier.Connector.Cve/Internal/CveMapper.cs
Normal file
@@ -0,0 +1,451 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Cvss;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using NuGet.Versioning;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
internal static class CveMapper
|
||||
{
|
||||
private static readonly string[] SeverityOrder =
|
||||
{
|
||||
"critical",
|
||||
"high",
|
||||
"medium",
|
||||
"low",
|
||||
"informational",
|
||||
"none",
|
||||
"unknown",
|
||||
};
|
||||
|
||||
public static Advisory Map(CveRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt);
|
||||
var mapProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "mapping", dto.CveId, recordedAt);
|
||||
|
||||
var aliases = dto.Aliases
|
||||
.Append(dto.CveId)
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var references = dto.References
|
||||
.Select(reference => CreateReference(reference, recordedAt))
|
||||
.Where(static reference => reference is not null)
|
||||
.Cast<AdvisoryReference>()
|
||||
.ToList();
|
||||
|
||||
var affected = CreateAffectedPackages(dto, recordedAt);
|
||||
var cvssMetrics = CreateCvssMetrics(dto, recordedAt, document.Uri);
|
||||
var severity = DetermineSeverity(cvssMetrics);
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
fetchProvenance,
|
||||
mapProvenance,
|
||||
};
|
||||
|
||||
var title = string.IsNullOrWhiteSpace(dto.Title) ? dto.CveId : dto.Title!;
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.CveId,
|
||||
title: title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language,
|
||||
published: dto.Published,
|
||||
modified: dto.Modified ?? dto.Published,
|
||||
severity: severity,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affected,
|
||||
cvssMetrics: cvssMetrics,
|
||||
provenance: provenance);
|
||||
}
|
||||
|
||||
private static AdvisoryReference? CreateReference(CveReferenceDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dto.Url) || !Validation.LooksLikeHttpUrl(dto.Url))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var kind = dto.Tags.FirstOrDefault();
|
||||
return new AdvisoryReference(
|
||||
dto.Url,
|
||||
kind,
|
||||
dto.Source,
|
||||
summary: null,
|
||||
provenance: new AdvisoryProvenance(CveConnectorPlugin.SourceName, "reference", dto.Url, recordedAt));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> CreateAffectedPackages(CveRecordDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Affected.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Affected.Count);
|
||||
foreach (var affected in dto.Affected)
|
||||
{
|
||||
var vendor = string.IsNullOrWhiteSpace(affected.Vendor) ? "unknown-vendor" : affected.Vendor!.Trim();
|
||||
var product = string.IsNullOrWhiteSpace(affected.Product) ? "unknown-product" : affected.Product!.Trim();
|
||||
var identifier = string.Equals(product, vendor, StringComparison.OrdinalIgnoreCase)
|
||||
? vendor.ToLowerInvariant()
|
||||
: $"{vendor}:{product}".ToLowerInvariant();
|
||||
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected", identifier, recordedAt),
|
||||
};
|
||||
|
||||
var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["vendor"] = vendor,
|
||||
["product"] = product,
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(affected.Platform))
|
||||
{
|
||||
vendorExtensions["platform"] = affected.Platform!;
|
||||
}
|
||||
|
||||
var note = BuildNormalizedNote(dto.CveId, identifier);
|
||||
var (ranges, normalizedVersions) = CreateVersionArtifacts(affected, recordedAt, identifier, vendorExtensions, note);
|
||||
var statuses = CreateStatuses(affected, recordedAt, identifier);
|
||||
|
||||
if (ranges.Count == 0)
|
||||
{
|
||||
var fallbackPrimitives = vendorExtensions.Count == 0
|
||||
? null
|
||||
: new RangePrimitives(null, null, null, vendorExtensions);
|
||||
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "vendor",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: null,
|
||||
provenance: provenance[0],
|
||||
primitives: fallbackPrimitives));
|
||||
}
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
type: AffectedPackageTypes.Vendor,
|
||||
identifier: identifier,
|
||||
platform: affected.Platform,
|
||||
versionRanges: ranges,
|
||||
statuses: statuses,
|
||||
provenance: provenance,
|
||||
normalizedVersions: normalizedVersions.Count == 0
|
||||
? Array.Empty<NormalizedVersionRule>()
|
||||
: normalizedVersions.ToArray()));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> Normalized) CreateVersionArtifacts(
|
||||
CveAffectedDto affected,
|
||||
DateTimeOffset recordedAt,
|
||||
string identifier,
|
||||
IReadOnlyDictionary<string, string> vendorExtensions,
|
||||
string normalizedNote)
|
||||
{
|
||||
var ranges = new List<AffectedVersionRange>();
|
||||
var normalized = new List<NormalizedVersionRule>();
|
||||
|
||||
foreach (var version in affected.Versions)
|
||||
{
|
||||
var range = BuildVersionRange(version, recordedAt, identifier, vendorExtensions);
|
||||
if (range is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ranges.Add(range);
|
||||
|
||||
var rule = range.ToNormalizedVersionRule(normalizedNote);
|
||||
if (rule is not null)
|
||||
{
|
||||
normalized.Add(rule);
|
||||
}
|
||||
}
|
||||
|
||||
return (ranges, normalized);
|
||||
}
|
||||
|
||||
private static AffectedVersionRange? BuildVersionRange(
|
||||
CveVersionDto version,
|
||||
DateTimeOffset recordedAt,
|
||||
string identifier,
|
||||
IReadOnlyDictionary<string, string> baseVendorExtensions)
|
||||
{
|
||||
var vendor = new Dictionary<string, string>(baseVendorExtensions, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddExtension(string key, string? value)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
vendor[key] = value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
AddExtension("version", version.Version);
|
||||
AddExtension("lessThan", version.LessThan);
|
||||
AddExtension("lessThanOrEqual", version.LessThanOrEqual);
|
||||
AddExtension("versionType", version.VersionType);
|
||||
AddExtension("versionRange", version.Range);
|
||||
|
||||
var introduced = Normalize(version.Version);
|
||||
var fixedExclusive = Normalize(version.LessThan);
|
||||
var lastInclusive = Normalize(version.LessThanOrEqual);
|
||||
|
||||
var rangeExpression = Normalize(string.IsNullOrWhiteSpace(version.Range)
|
||||
? BuildRangeExpression(version)
|
||||
: version.Range);
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CveConnectorPlugin.SourceName,
|
||||
"affected-range",
|
||||
identifier,
|
||||
recordedAt);
|
||||
|
||||
var semVerPrimitive = TryBuildSemVerPrimitive(
|
||||
version.VersionType,
|
||||
introduced,
|
||||
fixedExclusive,
|
||||
lastInclusive,
|
||||
rangeExpression,
|
||||
out var primitive);
|
||||
|
||||
if (semVerPrimitive)
|
||||
{
|
||||
introduced = primitive!.Introduced ?? introduced;
|
||||
fixedExclusive = primitive.Fixed ?? fixedExclusive;
|
||||
lastInclusive = primitive.LastAffected ?? lastInclusive;
|
||||
}
|
||||
|
||||
if (introduced is null && fixedExclusive is null && lastInclusive is null && rangeExpression is null && primitive is null && vendor.Count == baseVendorExtensions.Count)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var rangeKind = primitive is not null
|
||||
? NormalizedVersionSchemes.SemVer
|
||||
: (string.IsNullOrWhiteSpace(version.VersionType)
|
||||
? "vendor"
|
||||
: version.VersionType!.Trim().ToLowerInvariant());
|
||||
|
||||
var rangePrimitives = primitive is null && vendor.Count == 0
|
||||
? null
|
||||
: new RangePrimitives(
|
||||
primitive,
|
||||
Nevra: null,
|
||||
Evr: null,
|
||||
VendorExtensions: vendor.Count == 0 ? null : vendor);
|
||||
|
||||
return new AffectedVersionRange(
|
||||
rangeKind: rangeKind,
|
||||
introducedVersion: introduced,
|
||||
fixedVersion: fixedExclusive,
|
||||
lastAffectedVersion: lastInclusive,
|
||||
rangeExpression: rangeExpression,
|
||||
provenance: provenance,
|
||||
primitives: rangePrimitives);
|
||||
}
|
||||
|
||||
private static List<AffectedPackageStatus> CreateStatuses(CveAffectedDto affected, DateTimeOffset recordedAt, string identifier)
|
||||
{
|
||||
var statuses = new List<AffectedPackageStatus>();
|
||||
|
||||
void AddStatus(string? status)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
statuses.Add(new AffectedPackageStatus(
|
||||
status,
|
||||
new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected-status", identifier, recordedAt)));
|
||||
}
|
||||
|
||||
AddStatus(affected.DefaultStatus);
|
||||
|
||||
foreach (var version in affected.Versions)
|
||||
{
|
||||
AddStatus(version.Status);
|
||||
}
|
||||
|
||||
return statuses;
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) || value is "*" or "-" ? null : value.Trim();
|
||||
|
||||
private static string? BuildRangeExpression(CveVersionDto version)
|
||||
{
|
||||
var builder = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(version.Version))
|
||||
{
|
||||
builder.Add($"version={version.Version}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(version.LessThan))
|
||||
{
|
||||
builder.Add($"< {version.LessThan}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(version.LessThanOrEqual))
|
||||
{
|
||||
builder.Add($"<= {version.LessThanOrEqual}");
|
||||
}
|
||||
|
||||
if (builder.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return string.Join(", ", builder);
|
||||
}
|
||||
|
||||
private static string BuildNormalizedNote(string cveId, string identifier)
|
||||
{
|
||||
var baseId = string.IsNullOrWhiteSpace(cveId)
|
||||
? "unknown"
|
||||
: cveId.Trim().ToLowerInvariant();
|
||||
return $"cve:{baseId}:{identifier}";
|
||||
}
|
||||
|
||||
private static bool TryBuildSemVerPrimitive(
|
||||
string? versionType,
|
||||
string? introduced,
|
||||
string? fixedExclusive,
|
||||
string? lastInclusive,
|
||||
string? constraintExpression,
|
||||
out SemVerPrimitive? primitive)
|
||||
{
|
||||
primitive = null;
|
||||
|
||||
if (!string.Equals(versionType, "semver", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryNormalizeSemVer(introduced, out var normalizedIntroduced)
|
||||
|| !TryNormalizeSemVer(fixedExclusive, out var normalizedFixed)
|
||||
|| !TryNormalizeSemVer(lastInclusive, out var normalizedLast))
|
||||
{
|
||||
normalizedIntroduced = introduced;
|
||||
normalizedFixed = fixedExclusive;
|
||||
normalizedLast = lastInclusive;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (normalizedIntroduced is null && normalizedFixed is null && normalizedLast is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var introducedInclusive = true;
|
||||
var fixedInclusive = false;
|
||||
var lastInclusiveFlag = true;
|
||||
|
||||
if (normalizedFixed is null && normalizedLast is null && normalizedIntroduced is not null)
|
||||
{
|
||||
// Exact version. Treat as introduced/fixed equality.
|
||||
normalizedFixed = normalizedIntroduced;
|
||||
normalizedLast = normalizedIntroduced;
|
||||
}
|
||||
|
||||
primitive = new SemVerPrimitive(
|
||||
Introduced: normalizedIntroduced,
|
||||
IntroducedInclusive: introducedInclusive,
|
||||
Fixed: normalizedFixed,
|
||||
FixedInclusive: fixedInclusive,
|
||||
LastAffected: normalizedLast,
|
||||
LastAffectedInclusive: lastInclusiveFlag,
|
||||
ConstraintExpression: constraintExpression,
|
||||
ExactValue: normalizedFixed is not null && normalizedIntroduced is not null && normalizedFixed == normalizedIntroduced
|
||||
? normalizedIntroduced
|
||||
: null);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryNormalizeSemVer(string? value, out string? normalized)
|
||||
{
|
||||
normalized = null;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 1)
|
||||
{
|
||||
trimmed = trimmed[1..];
|
||||
}
|
||||
|
||||
if (!NuGetVersion.TryParse(trimmed, out var parsed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
normalized = parsed.ToNormalizedString();
|
||||
return true;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> CreateCvssMetrics(CveRecordDto dto, DateTimeOffset recordedAt, string sourceUri)
|
||||
{
|
||||
if (dto.Metrics.Count == 0)
|
||||
{
|
||||
return Array.Empty<CvssMetric>();
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "cvss", sourceUri, recordedAt);
|
||||
var metrics = new List<CvssMetric>(dto.Metrics.Count);
|
||||
foreach (var metric in dto.Metrics)
|
||||
{
|
||||
if (!CvssMetricNormalizer.TryNormalize(metric.Version, metric.Vector, metric.BaseScore, metric.BaseSeverity, out var normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
metrics.Add(new CvssMetric(
|
||||
normalized.Version,
|
||||
normalized.Vector,
|
||||
normalized.BaseScore,
|
||||
normalized.BaseSeverity,
|
||||
provenance));
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static string? DetermineSeverity(IReadOnlyList<CvssMetric> metrics)
|
||||
{
|
||||
if (metrics.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var level in SeverityOrder)
|
||||
{
|
||||
if (metrics.Any(metric => string.Equals(metric.BaseSeverity, level, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return level;
|
||||
}
|
||||
}
|
||||
|
||||
return metrics
|
||||
.Select(metric => metric.BaseSeverity)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
}
|
||||
105
src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordDto.cs
Normal file
105
src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordDto.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
internal sealed record CveRecordDto
|
||||
{
|
||||
[JsonPropertyName("cveId")]
|
||||
public string CveId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("state")]
|
||||
public string State { get; init; } = "PUBLISHED";
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("aliases")]
|
||||
public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CveReferenceDto> References { get; init; } = Array.Empty<CveReferenceDto>();
|
||||
|
||||
[JsonPropertyName("affected")]
|
||||
public IReadOnlyList<CveAffectedDto> Affected { get; init; } = Array.Empty<CveAffectedDto>();
|
||||
|
||||
[JsonPropertyName("metrics")]
|
||||
public IReadOnlyList<CveCvssMetricDto> Metrics { get; init; } = Array.Empty<CveCvssMetricDto>();
|
||||
}
|
||||
|
||||
internal sealed record CveReferenceDto
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string Url { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("source")]
|
||||
public string? Source { get; init; }
|
||||
|
||||
[JsonPropertyName("tags")]
|
||||
public IReadOnlyList<string> Tags { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
internal sealed record CveAffectedDto
|
||||
{
|
||||
[JsonPropertyName("vendor")]
|
||||
public string? Vendor { get; init; }
|
||||
|
||||
[JsonPropertyName("product")]
|
||||
public string? Product { get; init; }
|
||||
|
||||
[JsonPropertyName("platform")]
|
||||
public string? Platform { get; init; }
|
||||
|
||||
[JsonPropertyName("defaultStatus")]
|
||||
public string? DefaultStatus { get; init; }
|
||||
|
||||
[JsonPropertyName("versions")]
|
||||
public IReadOnlyList<CveVersionDto> Versions { get; init; } = Array.Empty<CveVersionDto>();
|
||||
}
|
||||
|
||||
internal sealed record CveVersionDto
|
||||
{
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; init; }
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; init; }
|
||||
|
||||
[JsonPropertyName("lessThan")]
|
||||
public string? LessThan { get; init; }
|
||||
|
||||
[JsonPropertyName("lessThanOrEqual")]
|
||||
public string? LessThanOrEqual { get; init; }
|
||||
|
||||
[JsonPropertyName("versionType")]
|
||||
public string? VersionType { get; init; }
|
||||
|
||||
[JsonPropertyName("versionRange")]
|
||||
public string? Range { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CveCvssMetricDto
|
||||
{
|
||||
[JsonPropertyName("version")]
|
||||
public string? Version { get; init; }
|
||||
|
||||
[JsonPropertyName("vector")]
|
||||
public string? Vector { get; init; }
|
||||
|
||||
[JsonPropertyName("baseScore")]
|
||||
public double? BaseScore { get; init; }
|
||||
|
||||
[JsonPropertyName("baseSeverity")]
|
||||
public string? BaseSeverity { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.Normalization.Text;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve.Internal;
|
||||
|
||||
internal static class CveRecordParser
|
||||
{
|
||||
public static CveRecordDto Parse(ReadOnlySpan<byte> content)
|
||||
{
|
||||
using var document = JsonDocument.Parse(content.ToArray());
|
||||
var root = document.RootElement;
|
||||
|
||||
var metadata = TryGetProperty(root, "cveMetadata");
|
||||
if (metadata.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
throw new JsonException("cveMetadata section missing.");
|
||||
}
|
||||
|
||||
var containers = TryGetProperty(root, "containers");
|
||||
var cna = TryGetProperty(containers, "cna");
|
||||
|
||||
var cveId = GetString(metadata, "cveId") ?? throw new JsonException("cveMetadata.cveId missing.");
|
||||
var state = GetString(metadata, "state") ?? "PUBLISHED";
|
||||
var published = GetDate(metadata, "datePublished");
|
||||
var modified = GetDate(metadata, "dateUpdated") ?? GetDate(metadata, "dateReserved");
|
||||
|
||||
var description = ParseDescription(cna);
|
||||
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
cveId,
|
||||
};
|
||||
foreach (var alias in ParseAliases(cna))
|
||||
{
|
||||
aliases.Add(alias);
|
||||
}
|
||||
|
||||
var references = ParseReferences(cna);
|
||||
var affected = ParseAffected(cna);
|
||||
var metrics = ParseMetrics(cna);
|
||||
|
||||
return new CveRecordDto
|
||||
{
|
||||
CveId = cveId,
|
||||
Title = GetString(cna, "title") ?? cveId,
|
||||
Summary = description.Text,
|
||||
Language = description.Language,
|
||||
State = state,
|
||||
Published = published,
|
||||
Modified = modified,
|
||||
Aliases = aliases.ToArray(),
|
||||
References = references,
|
||||
Affected = affected,
|
||||
Metrics = metrics,
|
||||
};
|
||||
}
|
||||
|
||||
private static NormalizedDescription ParseDescription(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>());
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty("descriptions", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>());
|
||||
}
|
||||
|
||||
var items = new List<LocalizedText>(descriptions.GetArrayLength());
|
||||
foreach (var entry in descriptions.EnumerateArray())
|
||||
{
|
||||
if (entry.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var text = GetString(entry, "value");
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var lang = GetString(entry, "lang");
|
||||
items.Add(new LocalizedText(text, lang));
|
||||
}
|
||||
|
||||
return DescriptionNormalizer.Normalize(items);
|
||||
}
|
||||
|
||||
private static IEnumerable<string> ParseAliases(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty("aliases", out var aliases) || aliases.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var alias in aliases.EnumerateArray())
|
||||
{
|
||||
if (alias.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var value = alias.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
yield return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CveReferenceDto> ParseReferences(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return Array.Empty<CveReferenceDto>();
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return Array.Empty<CveReferenceDto>();
|
||||
}
|
||||
|
||||
var list = new List<CveReferenceDto>(references.GetArrayLength());
|
||||
foreach (var reference in references.EnumerateArray())
|
||||
{
|
||||
if (reference.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var url = GetString(reference, "url");
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var tags = Array.Empty<string>();
|
||||
if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
tags = tagsElement
|
||||
.EnumerateArray()
|
||||
.Where(static t => t.ValueKind == JsonValueKind.String)
|
||||
.Select(static t => t.GetString()!)
|
||||
.Where(static v => !string.IsNullOrWhiteSpace(v))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
var source = GetString(reference, "name") ?? GetString(reference, "source");
|
||||
list.Add(new CveReferenceDto
|
||||
{
|
||||
Url = url,
|
||||
Source = source,
|
||||
Tags = tags,
|
||||
});
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CveAffectedDto> ParseAffected(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return Array.Empty<CveAffectedDto>();
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty("affected", out var affected) || affected.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return Array.Empty<CveAffectedDto>();
|
||||
}
|
||||
|
||||
var list = new List<CveAffectedDto>(affected.GetArrayLength());
|
||||
foreach (var item in affected.EnumerateArray())
|
||||
{
|
||||
if (item.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var versions = new List<CveVersionDto>();
|
||||
if (item.TryGetProperty("versions", out var versionsElement) && versionsElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var versionEntry in versionsElement.EnumerateArray())
|
||||
{
|
||||
if (versionEntry.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
versions.Add(new CveVersionDto
|
||||
{
|
||||
Status = GetString(versionEntry, "status"),
|
||||
Version = GetString(versionEntry, "version"),
|
||||
LessThan = GetString(versionEntry, "lessThan"),
|
||||
LessThanOrEqual = GetString(versionEntry, "lessThanOrEqual"),
|
||||
VersionType = GetString(versionEntry, "versionType"),
|
||||
Range = GetString(versionEntry, "versionRange"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
list.Add(new CveAffectedDto
|
||||
{
|
||||
Vendor = GetString(item, "vendor") ?? GetString(item, "vendorName"),
|
||||
Product = GetString(item, "product") ?? GetString(item, "productName"),
|
||||
Platform = GetString(item, "platform"),
|
||||
DefaultStatus = GetString(item, "defaultStatus"),
|
||||
Versions = versions,
|
||||
});
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CveCvssMetricDto> ParseMetrics(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return Array.Empty<CveCvssMetricDto>();
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Array)
|
||||
{
|
||||
return Array.Empty<CveCvssMetricDto>();
|
||||
}
|
||||
|
||||
var list = new List<CveCvssMetricDto>(metrics.GetArrayLength());
|
||||
foreach (var metric in metrics.EnumerateArray())
|
||||
{
|
||||
if (metric.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (metric.TryGetProperty("cvssV4_0", out var cvss40) && cvss40.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
list.Add(ParseCvss(cvss40, "4.0"));
|
||||
}
|
||||
else if (metric.TryGetProperty("cvssV3_1", out var cvss31) && cvss31.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
list.Add(ParseCvss(cvss31, "3.1"));
|
||||
}
|
||||
else if (metric.TryGetProperty("cvssV3", out var cvss3) && cvss3.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
list.Add(ParseCvss(cvss3, "3.0"));
|
||||
}
|
||||
else if (metric.TryGetProperty("cvssV2", out var cvss2) && cvss2.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
list.Add(ParseCvss(cvss2, "2.0"));
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static CveCvssMetricDto ParseCvss(JsonElement element, string fallbackVersion)
|
||||
{
|
||||
var version = GetString(element, "version") ?? fallbackVersion;
|
||||
var vector = GetString(element, "vectorString") ?? GetString(element, "vector");
|
||||
var baseScore = GetDouble(element, "baseScore");
|
||||
var severity = GetString(element, "baseSeverity") ?? GetString(element, "severity");
|
||||
|
||||
return new CveCvssMetricDto
|
||||
{
|
||||
Version = version,
|
||||
Vector = vector,
|
||||
BaseScore = baseScore,
|
||||
BaseSeverity = severity,
|
||||
};
|
||||
}
|
||||
|
||||
private static JsonElement TryGetProperty(JsonElement element, string propertyName)
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var property))
|
||||
{
|
||||
return property;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
|
||||
private static string? GetString(JsonElement element, string propertyName)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty(propertyName, out var property))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return property.ValueKind switch
|
||||
{
|
||||
JsonValueKind.String => property.GetString(),
|
||||
JsonValueKind.Number when property.TryGetDouble(out var number) => number.ToString(CultureInfo.InvariantCulture),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTimeOffset? GetDate(JsonElement element, string propertyName)
|
||||
{
|
||||
var value = GetString(element, propertyName);
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
? parsed.ToUniversalTime()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static double? GetDouble(JsonElement element, string propertyName)
|
||||
{
|
||||
if (element.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!element.TryGetProperty(propertyName, out var property))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (property.ValueKind == JsonValueKind.Number && property.TryGetDouble(out var number))
|
||||
{
|
||||
return number;
|
||||
}
|
||||
|
||||
if (property.ValueKind == JsonValueKind.String && double.TryParse(property.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
43
src/StellaOps.Concelier.Connector.Cve/Jobs.cs
Normal file
43
src/StellaOps.Concelier.Connector.Cve/Jobs.cs
Normal file
@@ -0,0 +1,43 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Cve;
|
||||
|
||||
internal static class CveJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cve:fetch";
|
||||
public const string Parse = "source:cve:parse";
|
||||
public const string Map = "source:cve:map";
|
||||
}
|
||||
|
||||
internal sealed class CveFetchJob : IJob
|
||||
{
|
||||
private readonly CveConnector _connector;
|
||||
|
||||
public CveFetchJob(CveConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CveParseJob : IJob
|
||||
{
|
||||
private readonly CveConnector _connector;
|
||||
|
||||
public CveParseJob(CveConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class CveMapJob : IJob
|
||||
{
|
||||
private readonly CveConnector _connector;
|
||||
|
||||
public CveMapJob(CveConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
12
src/StellaOps.Concelier.Connector.Cve/TASKS.md
Normal file
12
src/StellaOps.Concelier.Connector.Cve/TASKS.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|Define CVE data source + contract|BE-Conn-CVE|Research|**DONE (2025-10-10)** – Connector targets the CVE Services JSON 5 API with authenticated windowed queries documented in `CveOptions` (`CVE-API-*` headers, pagination semantics, failure backoff).|
|
||||
|Fetch/cursor implementation|BE-Conn-CVE|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Time-window + page-aware cursor with SourceFetchService fetching list/detail pairs, resumable state persisted via `CveCursor`.|
|
||||
|DTOs & parser|BE-Conn-CVE|Source.Common|**DONE (2025-10-10)** – `CveRecordParser` and DTOs capture aliases, references, metrics, vendor ranges; sanitises text and timestamps.|
|
||||
|Canonical mapping & range primitives|BE-Conn-CVE|Models|**DONE (2025-10-10)** – `CveMapper` emits canonical advisories, vendor range primitives, SemVer/range statuses, references, CVSS normalization.<br>2025-10-11 research trail: confirm subsequent MR adds `NormalizedVersions` shaped like `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"nvd:CVE-2025-XXXX"}]` so storage provenance joins continue to work.|
|
||||
|Deterministic tests & fixtures|QA|Testing|**DONE (2025-10-10)** – Added `StellaOps.Concelier.Connector.Cve.Tests` harness with canned fixtures + snapshot regression covering fetch/parse/map.|
|
||||
|Observability & docs|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`cve.fetch.*`, etc.) wired; options/usage documented via `CveServiceCollectionExtensions`.|
|
||||
|Operator rollout playbook|BE-Conn-CVE, Ops|Docs|**DONE (2025-10-12)** – Refreshed `docs/ops/concelier-cve-kev-operations.md` with credential checklist, smoke book, PromQL guardrails, and linked Grafana pack (`docs/ops/concelier-cve-kev-grafana-dashboard.json`).|
|
||||
|Live smoke & monitoring|QA, BE-Conn-CVE|WebService, Observability|**DONE (2025-10-15)** – Executed connector harness smoke using CVE Services sample window (CVE-2024-0001), confirmed fetch/parse/map telemetry (`cve.fetch.*`, `cve.map.success`) all incremented once, and archived the summary log + Grafana import guidance in `docs/ops/concelier-cve-kev-operations.md` (“Staging smoke 2025-10-15”).|
|
||||
|FEEDCONN-CVE-02-003 Normalized versions rollout|BE-Conn-CVE|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Confirmed SemVer primitives map to normalized rules with `cve:{cveId}:{identifier}` notes and refreshed snapshots; `dotnet test src/StellaOps.Concelier.Connector.Cve.Tests` passes on net10 preview.|
|
||||
Reference in New Issue
Block a user