feat: Add CVSS receipt management endpoints and related functionality
- Introduced new API endpoints for creating, retrieving, amending, and listing CVSS receipts. - Updated IPolicyEngineClient interface to include methods for CVSS receipt operations. - Implemented PolicyEngineClient to handle CVSS receipt requests. - Enhanced Program.cs to map new CVSS receipt routes with appropriate authorization. - Added necessary models and contracts for CVSS receipt requests and responses. - Integrated Postgres document store for managing CVSS receipts and related data. - Updated database schema with new migrations for source documents and payload storage. - Refactored existing components to support new CVSS functionality.
This commit is contained in:
StellaOps Bot
@@ -1,325 +1,325 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Jvn.Configuration;
|
||||
using StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.JpFlags;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn;
|
||||
|
||||
public sealed class JvnConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly MyJvnClient _client;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly IJpFlagStore _jpFlagStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JvnOptions _options;
|
||||
private readonly ILogger<JvnConnector> _logger;
|
||||
|
||||
public JvnConnector(
|
||||
MyJvnClient client,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
IJpFlagStore jpFlagStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<JvnOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<JvnConnector> logger)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_jpFlagStore = jpFlagStore ?? throw new ArgumentNullException(nameof(jpFlagStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => JvnConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var windowEnd = now;
|
||||
var defaultWindowStart = windowEnd - _options.WindowSize;
|
||||
|
||||
var windowStart = cursor.LastCompletedWindowEnd.HasValue
|
||||
? cursor.LastCompletedWindowEnd.Value - _options.WindowOverlap
|
||||
: defaultWindowStart;
|
||||
|
||||
if (windowStart < defaultWindowStart)
|
||||
{
|
||||
windowStart = defaultWindowStart;
|
||||
}
|
||||
|
||||
if (windowStart >= windowEnd)
|
||||
{
|
||||
windowStart = windowEnd - TimeSpan.FromHours(1);
|
||||
}
|
||||
|
||||
_logger.LogInformation("JVN fetch window {WindowStart:o} - {WindowEnd:o}", windowStart, windowEnd);
|
||||
|
||||
IReadOnlyList<JvnOverviewItem> overviewItems;
|
||||
try
|
||||
{
|
||||
overviewItems = await _client.GetOverviewAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to retrieve JVN overview between {Start:o} and {End:o}", windowStart, windowEnd);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
_logger.LogInformation("JVN overview returned {Count} items", overviewItems.Count);
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
|
||||
foreach (var item in overviewItems)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var detailUri = _client.BuildDetailUri(item.VulnerabilityId);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["jvn.vulnId"] = item.VulnerabilityId,
|
||||
["jvn.detailUrl"] = detailUri.ToString(),
|
||||
};
|
||||
|
||||
if (item.DateFirstPublished.HasValue)
|
||||
{
|
||||
metadata["jvn.firstPublished"] = item.DateFirstPublished.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (item.DateLastUpdated.HasValue)
|
||||
{
|
||||
metadata["jvn.lastUpdated"] = item.DateLastUpdated.Value.ToString("O");
|
||||
}
|
||||
|
||||
var result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(JvnOptions.HttpClientName, SourceName, detailUri)
|
||||
{
|
||||
Metadata = metadata
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
if (!result.IsNotModified)
|
||||
{
|
||||
_logger.LogWarning("JVN fetch for {Uri} returned status {Status}", detailUri, result.StatusCode);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogDebug("JVN fetched document {DocumentId}", result.Document.Id);
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithWindow(windowStart, windowEnd)
|
||||
.WithCompletedWindow(windowEnd)
|
||||
.WithPendingDocuments(pendingDocuments);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("JVN parse pending documents: {PendingCount}", cursor.PendingDocuments.Count);
|
||||
Console.WriteLine($"JVN parse pending count: {cursor.PendingDocuments.Count}");
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
_logger.LogDebug("JVN parsing document {DocumentId}", documentId);
|
||||
Console.WriteLine($"JVN parsing document {documentId}");
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
_logger.LogWarning("JVN document {DocumentId} no longer exists; skipping", documentId);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("JVN document {DocumentId} is missing GridFS content; marking as failed", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unable to download raw JVN document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
JvnDetailDto detail;
|
||||
try
|
||||
{
|
||||
detail = JvnDetailParser.Parse(rawBytes, document.Uri);
|
||||
}
|
||||
catch (JvnSchemaValidationException ex)
|
||||
{
|
||||
Console.WriteLine($"JVN schema validation exception: {ex.Message}");
|
||||
_logger.LogWarning(ex, "JVN schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
var sanitizedJson = JsonSerializer.Serialize(detail, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(sanitizedJson);
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
JvnConstants.DtoSchemaVersion,
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
Console.WriteLine($"Added mapping for {documentId}");
|
||||
_logger.LogDebug("JVN parsed document {DocumentId}", documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("JVN map pending mappings: {PendingCount}", cursor.PendingMappings.Count);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping JVN mapping for {DocumentId}: DTO or document missing", documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
JvnDetailDto detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<JvnDetailDto>(dtoJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Deserialized DTO was null.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize JVN DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (advisory, flag) = JvnAdvisoryMapper.Map(detail, document, dto, _timeProvider);
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _jpFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
_logger.LogDebug("JVN mapped document {DocumentId}", documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<JvnCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? JvnCursor.Empty : JvnCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(JvnCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursorDocument = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursorDocument, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Jvn.Configuration;
|
||||
using StellaOps.Concelier.Connector.Jvn.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Concelier.Storage.Mongo.JpFlags;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Jvn;
|
||||
|
||||
public sealed class JvnConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly MyJvnClient _client;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly IJpFlagStore _jpFlagStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JvnOptions _options;
|
||||
private readonly ILogger<JvnConnector> _logger;
|
||||
|
||||
public JvnConnector(
|
||||
MyJvnClient client,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
IJpFlagStore jpFlagStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<JvnOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<JvnConnector> logger)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_jpFlagStore = jpFlagStore ?? throw new ArgumentNullException(nameof(jpFlagStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => JvnConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var windowEnd = now;
|
||||
var defaultWindowStart = windowEnd - _options.WindowSize;
|
||||
|
||||
var windowStart = cursor.LastCompletedWindowEnd.HasValue
|
||||
? cursor.LastCompletedWindowEnd.Value - _options.WindowOverlap
|
||||
: defaultWindowStart;
|
||||
|
||||
if (windowStart < defaultWindowStart)
|
||||
{
|
||||
windowStart = defaultWindowStart;
|
||||
}
|
||||
|
||||
if (windowStart >= windowEnd)
|
||||
{
|
||||
windowStart = windowEnd - TimeSpan.FromHours(1);
|
||||
}
|
||||
|
||||
_logger.LogInformation("JVN fetch window {WindowStart:o} - {WindowEnd:o}", windowStart, windowEnd);
|
||||
|
||||
IReadOnlyList<JvnOverviewItem> overviewItems;
|
||||
try
|
||||
{
|
||||
overviewItems = await _client.GetOverviewAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to retrieve JVN overview between {Start:o} and {End:o}", windowStart, windowEnd);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
_logger.LogInformation("JVN overview returned {Count} items", overviewItems.Count);
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
|
||||
foreach (var item in overviewItems)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var detailUri = _client.BuildDetailUri(item.VulnerabilityId);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["jvn.vulnId"] = item.VulnerabilityId,
|
||||
["jvn.detailUrl"] = detailUri.ToString(),
|
||||
};
|
||||
|
||||
if (item.DateFirstPublished.HasValue)
|
||||
{
|
||||
metadata["jvn.firstPublished"] = item.DateFirstPublished.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (item.DateLastUpdated.HasValue)
|
||||
{
|
||||
metadata["jvn.lastUpdated"] = item.DateLastUpdated.Value.ToString("O");
|
||||
}
|
||||
|
||||
var result = await _fetchService.FetchAsync(
|
||||
new SourceFetchRequest(JvnOptions.HttpClientName, SourceName, detailUri)
|
||||
{
|
||||
Metadata = metadata
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
if (!result.IsNotModified)
|
||||
{
|
||||
_logger.LogWarning("JVN fetch for {Uri} returned status {Status}", detailUri, result.StatusCode);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogDebug("JVN fetched document {DocumentId}", result.Document.Id);
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithWindow(windowStart, windowEnd)
|
||||
.WithCompletedWindow(windowEnd)
|
||||
.WithPendingDocuments(pendingDocuments);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("JVN parse pending documents: {PendingCount}", cursor.PendingDocuments.Count);
|
||||
Console.WriteLine($"JVN parse pending count: {cursor.PendingDocuments.Count}");
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
_logger.LogDebug("JVN parsing document {DocumentId}", documentId);
|
||||
Console.WriteLine($"JVN parsing document {documentId}");
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
_logger.LogWarning("JVN document {DocumentId} no longer exists; skipping", documentId);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.PayloadId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("JVN document {DocumentId} is missing GridFS content; marking as failed", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.PayloadId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unable to download raw JVN document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
JvnDetailDto detail;
|
||||
try
|
||||
{
|
||||
detail = JvnDetailParser.Parse(rawBytes, document.Uri);
|
||||
}
|
||||
catch (JvnSchemaValidationException ex)
|
||||
{
|
||||
Console.WriteLine($"JVN schema validation exception: {ex.Message}");
|
||||
_logger.LogWarning(ex, "JVN schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
var sanitizedJson = JsonSerializer.Serialize(detail, SerializerOptions);
|
||||
var payload = BsonDocument.Parse(sanitizedJson);
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
document.Id,
|
||||
SourceName,
|
||||
JvnConstants.DtoSchemaVersion,
|
||||
payload,
|
||||
_timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
Console.WriteLine($"Added mapping for {documentId}");
|
||||
_logger.LogDebug("JVN parsed document {DocumentId}", documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("JVN map pending mappings: {PendingCount}", cursor.PendingMappings.Count);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping JVN mapping for {DocumentId}: DTO or document missing", documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
JvnDetailDto detail;
|
||||
try
|
||||
{
|
||||
detail = JsonSerializer.Deserialize<JvnDetailDto>(dtoJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Deserialized DTO was null.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize JVN DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (advisory, flag) = JvnAdvisoryMapper.Map(detail, document, dto, _timeProvider);
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _jpFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
_logger.LogDebug("JVN mapped document {DocumentId}", documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<JvnCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? JvnCursor.Empty : JvnCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(JvnCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursorDocument = cursor.ToBsonDocument();
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursorDocument, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user