feat: Add CVSS receipt management endpoints and related functionality
- Introduced new API endpoints for creating, retrieving, amending, and listing CVSS receipts. - Updated IPolicyEngineClient interface to include methods for CVSS receipt operations. - Implemented PolicyEngineClient to handle CVSS receipt requests. - Enhanced Program.cs to map new CVSS receipt routes with appropriate authorization. - Added necessary models and contracts for CVSS receipt requests and responses. - Integrated Postgres document store for managing CVSS receipts and related data. - Updated database schema with new migrations for source documents and payload storage. - Refactored existing components to support new CVSS functionality.
This commit is contained in:
StellaOps Bot
@@ -1,384 +1,384 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly KasperskyFeedClient _feedClient;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly KasperskyOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<KasperskyConnector> _logger;
|
||||
|
||||
public KasperskyConnector(
|
||||
KasperskyFeedClient feedClient,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<KasperskyOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<KasperskyConnector> logger)
|
||||
{
|
||||
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => KasperskyConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var windowStart = cursor.LastPublished.HasValue
|
||||
? cursor.LastPublished.Value - _options.WindowOverlap
|
||||
: now - _options.WindowSize;
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
var cursorState = cursor;
|
||||
var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var page = 1; page <= _options.MaxPagesPerFetch; page++)
|
||||
{
|
||||
IReadOnlyList<KasperskyFeedItem> items;
|
||||
try
|
||||
{
|
||||
items = await _feedClient.GetItemsAsync(page, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load Kaspersky ICS feed page {Page}", page);
|
||||
await _stateRepository.MarkFailureAsync(
|
||||
SourceName,
|
||||
now,
|
||||
TimeSpan.FromMinutes(5),
|
||||
ex.Message,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
if (items.Count == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (item.Published < windowStart)
|
||||
{
|
||||
page = _options.MaxPagesPerFetch + 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["kaspersky.title"] = item.Title,
|
||||
["kaspersky.link"] = item.Link.ToString(),
|
||||
["kaspersky.published"] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Summary))
|
||||
{
|
||||
metadata["kaspersky.summary"] = item.Summary!;
|
||||
}
|
||||
|
||||
var slug = ExtractSlug(item.Link);
|
||||
if (!string.IsNullOrWhiteSpace(slug))
|
||||
{
|
||||
metadata["kaspersky.slug"] = slug;
|
||||
}
|
||||
|
||||
var resourceKey = item.Link.ToString();
|
||||
touchedResources.Add(resourceKey);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, resourceKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchRequest = new SourceFetchRequest(KasperskyOptions.HttpClientName, SourceName, item.Link)
|
||||
{
|
||||
Metadata = metadata,
|
||||
};
|
||||
|
||||
if (cursorState.TryGetFetchMetadata(resourceKey, out var cachedFetch))
|
||||
{
|
||||
fetchRequest = fetchRequest with
|
||||
{
|
||||
ETag = cachedFetch.ETag,
|
||||
LastModified = cachedFetch.LastModified,
|
||||
};
|
||||
}
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to fetch Kaspersky advisory {Link}", item.Link);
|
||||
await _stateRepository.MarkFailureAsync(
|
||||
SourceName,
|
||||
_timeProvider.GetUtcNow(),
|
||||
TimeSpan.FromMinutes(5),
|
||||
ex.Message,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing is not null
|
||||
&& string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal))
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false);
|
||||
cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified);
|
||||
if (item.Published > maxPublished)
|
||||
{
|
||||
maxPublished = item.Published;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified);
|
||||
if (item.Published > maxPublished)
|
||||
{
|
||||
maxPublished = item.Published;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cursorState = cursorState.PruneFetchCache(touchedResources);
|
||||
|
||||
var updatedCursor = cursorState
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Kaspersky document {DocumentId} missing GridFS content", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = document.Metadata ?? new Dictionary<string, string>();
|
||||
var title = metadata.TryGetValue("kaspersky.title", out var titleValue) ? titleValue : document.Uri;
|
||||
var link = metadata.TryGetValue("kaspersky.link", out var linkValue) ? linkValue : document.Uri;
|
||||
var published = metadata.TryGetValue("kaspersky.published", out var publishedValue) && DateTimeOffset.TryParse(publishedValue, out var parsedPublished)
|
||||
? parsedPublished.ToUniversalTime()
|
||||
: document.FetchedAt;
|
||||
var summary = metadata.TryGetValue("kaspersky.summary", out var summaryValue) ? summaryValue : null;
|
||||
var slug = metadata.TryGetValue("kaspersky.slug", out var slugValue) ? slugValue : ExtractSlug(new Uri(link, UriKind.Absolute));
|
||||
var advisoryKey = string.IsNullOrWhiteSpace(slug) ? Guid.NewGuid().ToString("N") : slug;
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed downloading raw Kaspersky document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
var dto = KasperskyAdvisoryParser.Parse(advisoryKey, title, link, published, summary, rawBytes);
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ics.kaspersky/1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping Kaspersky mapping for {DocumentId}: DTO or document missing", documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
KasperskyAdvisoryDto advisoryDto;
|
||||
try
|
||||
{
|
||||
advisoryDto = JsonSerializer.Deserialize<KasperskyAdvisoryDto>(dtoJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Deserialized DTO was null.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize Kaspersky DTO for {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt);
|
||||
var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryKey, dto.ValidatedAt);
|
||||
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
advisoryDto.AdvisoryKey,
|
||||
};
|
||||
foreach (var cve in advisoryDto.CveIds)
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>();
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(
|
||||
advisoryDto.Link,
|
||||
"advisory",
|
||||
"kaspersky-ics",
|
||||
null,
|
||||
new AdvisoryProvenance(SourceName, "reference", advisoryDto.Link, dto.ValidatedAt)));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
_logger.LogWarning("Invalid advisory link {Link} for {AdvisoryKey}", advisoryDto.Link, advisoryDto.AdvisoryKey);
|
||||
}
|
||||
|
||||
foreach (var cve in advisoryDto.CveIds)
|
||||
{
|
||||
var url = $"https://www.cve.org/CVERecord?id={cve}";
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(
|
||||
url,
|
||||
"advisory",
|
||||
cve,
|
||||
null,
|
||||
new AdvisoryProvenance(SourceName, "reference", url, dto.ValidatedAt)));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore malformed
|
||||
}
|
||||
}
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly KasperskyFeedClient _feedClient;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly KasperskyOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<KasperskyConnector> _logger;
|
||||
|
||||
public KasperskyConnector(
|
||||
KasperskyFeedClient feedClient,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<KasperskyOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<KasperskyConnector> logger)
|
||||
{
|
||||
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => KasperskyConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var windowStart = cursor.LastPublished.HasValue
|
||||
? cursor.LastPublished.Value - _options.WindowOverlap
|
||||
: now - _options.WindowSize;
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
var cursorState = cursor;
|
||||
var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var page = 1; page <= _options.MaxPagesPerFetch; page++)
|
||||
{
|
||||
IReadOnlyList<KasperskyFeedItem> items;
|
||||
try
|
||||
{
|
||||
items = await _feedClient.GetItemsAsync(page, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load Kaspersky ICS feed page {Page}", page);
|
||||
await _stateRepository.MarkFailureAsync(
|
||||
SourceName,
|
||||
now,
|
||||
TimeSpan.FromMinutes(5),
|
||||
ex.Message,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
if (items.Count == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (item.Published < windowStart)
|
||||
{
|
||||
page = _options.MaxPagesPerFetch + 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["kaspersky.title"] = item.Title,
|
||||
["kaspersky.link"] = item.Link.ToString(),
|
||||
["kaspersky.published"] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Summary))
|
||||
{
|
||||
metadata["kaspersky.summary"] = item.Summary!;
|
||||
}
|
||||
|
||||
var slug = ExtractSlug(item.Link);
|
||||
if (!string.IsNullOrWhiteSpace(slug))
|
||||
{
|
||||
metadata["kaspersky.slug"] = slug;
|
||||
}
|
||||
|
||||
var resourceKey = item.Link.ToString();
|
||||
touchedResources.Add(resourceKey);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, resourceKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var fetchRequest = new SourceFetchRequest(KasperskyOptions.HttpClientName, SourceName, item.Link)
|
||||
{
|
||||
Metadata = metadata,
|
||||
};
|
||||
|
||||
if (cursorState.TryGetFetchMetadata(resourceKey, out var cachedFetch))
|
||||
{
|
||||
fetchRequest = fetchRequest with
|
||||
{
|
||||
ETag = cachedFetch.ETag,
|
||||
LastModified = cachedFetch.LastModified,
|
||||
};
|
||||
}
|
||||
|
||||
SourceFetchResult result;
|
||||
try
|
||||
{
|
||||
result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to fetch Kaspersky advisory {Link}", item.Link);
|
||||
await _stateRepository.MarkFailureAsync(
|
||||
SourceName,
|
||||
_timeProvider.GetUtcNow(),
|
||||
TimeSpan.FromMinutes(5),
|
||||
ex.Message,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existing is not null
|
||||
&& string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal))
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false);
|
||||
cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified);
|
||||
if (item.Published > maxPublished)
|
||||
{
|
||||
maxPublished = item.Published;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified);
|
||||
if (item.Published > maxPublished)
|
||||
{
|
||||
maxPublished = item.Published;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cursorState = cursorState.PruneFetchCache(touchedResources);
|
||||
|
||||
var updatedCursor = cursorState
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.PayloadId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("Kaspersky document {DocumentId} missing GridFS content", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = document.Metadata ?? new Dictionary<string, string>();
|
||||
var title = metadata.TryGetValue("kaspersky.title", out var titleValue) ? titleValue : document.Uri;
|
||||
var link = metadata.TryGetValue("kaspersky.link", out var linkValue) ? linkValue : document.Uri;
|
||||
var published = metadata.TryGetValue("kaspersky.published", out var publishedValue) && DateTimeOffset.TryParse(publishedValue, out var parsedPublished)
|
||||
? parsedPublished.ToUniversalTime()
|
||||
: document.FetchedAt;
|
||||
var summary = metadata.TryGetValue("kaspersky.summary", out var summaryValue) ? summaryValue : null;
|
||||
var slug = metadata.TryGetValue("kaspersky.slug", out var slugValue) ? slugValue : ExtractSlug(new Uri(link, UriKind.Absolute));
|
||||
var advisoryKey = string.IsNullOrWhiteSpace(slug) ? Guid.NewGuid().ToString("N") : slug;
|
||||
|
||||
byte[] rawBytes;
|
||||
try
|
||||
{
|
||||
rawBytes = await _rawDocumentStorage.DownloadAsync(document.PayloadId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed downloading raw Kaspersky document {DocumentId}", document.Id);
|
||||
throw;
|
||||
}
|
||||
|
||||
var dto = KasperskyAdvisoryParser.Parse(advisoryKey, title, link, published, summary, rawBytes);
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ics.kaspersky/1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dto is null || document is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping Kaspersky mapping for {DocumentId}: DTO or document missing", documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
KasperskyAdvisoryDto advisoryDto;
|
||||
try
|
||||
{
|
||||
advisoryDto = JsonSerializer.Deserialize<KasperskyAdvisoryDto>(dtoJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Deserialized DTO was null.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to deserialize Kaspersky DTO for {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt);
|
||||
var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryKey, dto.ValidatedAt);
|
||||
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
advisoryDto.AdvisoryKey,
|
||||
};
|
||||
foreach (var cve in advisoryDto.CveIds)
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>();
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(
|
||||
advisoryDto.Link,
|
||||
"advisory",
|
||||
"kaspersky-ics",
|
||||
null,
|
||||
new AdvisoryProvenance(SourceName, "reference", advisoryDto.Link, dto.ValidatedAt)));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
_logger.LogWarning("Invalid advisory link {Link} for {AdvisoryKey}", advisoryDto.Link, advisoryDto.AdvisoryKey);
|
||||
}
|
||||
|
||||
foreach (var cve in advisoryDto.CveIds)
|
||||
{
|
||||
var url = $"https://www.cve.org/CVERecord?id={cve}";
|
||||
try
|
||||
{
|
||||
references.Add(new AdvisoryReference(
|
||||
url,
|
||||
"advisory",
|
||||
cve,
|
||||
null,
|
||||
new AdvisoryProvenance(SourceName, "reference", url, dto.ValidatedAt)));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// ignore malformed
|
||||
}
|
||||
}
|
||||
|
||||
var affectedPackages = new List<AffectedPackage>();
|
||||
foreach (var vendor in advisoryDto.VendorNames)
|
||||
{
|
||||
@@ -413,52 +413,52 @@ public sealed class KasperskyConnector : IFeedConnector
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: provenance));
|
||||
}
|
||||
|
||||
var advisory = new Advisory(
|
||||
advisoryDto.AdvisoryKey,
|
||||
advisoryDto.Title,
|
||||
advisoryDto.Summary ?? advisoryDto.Content,
|
||||
language: "en",
|
||||
published: advisoryDto.Published,
|
||||
modified: advisoryDto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mappingProvenance });
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<KasperskyCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? KasperskyCursor.Empty : KasperskyCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(KasperskyCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string? ExtractSlug(Uri link)
|
||||
{
|
||||
var segments = link.Segments;
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var last = segments[^1].Trim('/');
|
||||
return string.IsNullOrWhiteSpace(last) && segments.Length > 1 ? segments[^2].Trim('/') : last;
|
||||
}
|
||||
}
|
||||
|
||||
var advisory = new Advisory(
|
||||
advisoryDto.AdvisoryKey,
|
||||
advisoryDto.Title,
|
||||
advisoryDto.Summary ?? advisoryDto.Content,
|
||||
language: "en",
|
||||
published: advisoryDto.Published,
|
||||
modified: advisoryDto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: affectedPackages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { fetchProvenance, mappingProvenance });
|
||||
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<KasperskyCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? KasperskyCursor.Empty : KasperskyCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private async Task UpdateCursorAsync(KasperskyCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string? ExtractSlug(Uri link)
|
||||
{
|
||||
var segments = link.Segments;
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var last = segments[^1].Trim('/');
|
||||
return string.IsNullOrWhiteSpace(last) && segments.Length > 1 ? segments[^2].Trim('/') : last;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user