up
This commit is contained in:
		
							
								
								
									
										601
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										601
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,601 @@ | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using System.Security.Cryptography; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using MongoDB.Driver; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| public sealed class CiscoConnector : IFeedConnector | ||||
| { | ||||
|     private const string DtoSchemaVersion = "cisco.dto.v1"; | ||||
|  | ||||
|     private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|     }; | ||||
|  | ||||
|     private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|     }; | ||||
|  | ||||
|     private static readonly IReadOnlyDictionary<string, string> DefaultHeaders = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|     { | ||||
|         ["content-type"] = "application/json", | ||||
|     }; | ||||
|  | ||||
|     private readonly CiscoOpenVulnClient _openVulnClient; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly CiscoDtoFactory _dtoFactory; | ||||
|     private readonly CiscoDiagnostics _diagnostics; | ||||
|     private readonly IOptions<CiscoOptions> _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<CiscoConnector> _logger; | ||||
|  | ||||
|     public CiscoConnector( | ||||
|         CiscoOpenVulnClient openVulnClient, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         CiscoDtoFactory dtoFactory, | ||||
|         CiscoDiagnostics diagnostics, | ||||
|         IOptions<CiscoOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<CiscoConnector> logger) | ||||
|     { | ||||
|         _openVulnClient = openVulnClient ?? throw new ArgumentNullException(nameof(openVulnClient)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _dtoFactory = dtoFactory ?? throw new ArgumentNullException(nameof(dtoFactory)); | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _options = options ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => VndrCiscoConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var options = _options.Value; | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         var latestModified = cursor.LastModified; | ||||
|         var latestAdvisoryId = cursor.LastAdvisoryId; | ||||
|  | ||||
|         var startDate = DetermineStartDate(cursor, now, options); | ||||
|         var endDate = DateOnly.FromDateTime(now.UtcDateTime.Date); | ||||
|  | ||||
|         var added = 0; | ||||
|         var pagesFetched = 0; | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             for (var date = startDate; date <= endDate; date = date.AddDays(1)) | ||||
|             { | ||||
|                 cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|                 for (var pageIndex = 1; pageIndex <= options.MaxPagesPerFetch; pageIndex++) | ||||
|                 { | ||||
|                     cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|                     var page = await _openVulnClient.FetchAsync(date, pageIndex, cancellationToken).ConfigureAwait(false); | ||||
|                     pagesFetched++; | ||||
|  | ||||
|                     if (page is null || page.Advisories.Count == 0) | ||||
|                     { | ||||
|                         break; | ||||
|                     } | ||||
|  | ||||
|                     foreach (var advisory in page.Advisories | ||||
|                                  .OrderBy(static a => a.LastUpdated ?? DateTimeOffset.MinValue) | ||||
|                                  .ThenBy(static a => a.AdvisoryId, StringComparer.OrdinalIgnoreCase)) | ||||
|                     { | ||||
|                         cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|                         if (!ShouldProcess(advisory, latestModified, latestAdvisoryId)) | ||||
|                         { | ||||
|                             continue; | ||||
|                         } | ||||
|  | ||||
|                         var documentUri = BuildDocumentUri(advisory.AdvisoryId); | ||||
|                         var payload = advisory.GetRawBytes(); | ||||
|                         var sha = ComputeSha256(payload); | ||||
|  | ||||
|                         var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); | ||||
|                         if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)) | ||||
|                         { | ||||
|                             _diagnostics.FetchUnchanged(); | ||||
|                             continue; | ||||
|                         } | ||||
|  | ||||
|                         ObjectId gridFsId; | ||||
|                         try | ||||
|                         { | ||||
|                             gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", cancellationToken).ConfigureAwait(false); | ||||
|                         } | ||||
|                         catch (MongoWriteException ex) | ||||
|                         { | ||||
|                             _diagnostics.FetchFailure(); | ||||
|                             _logger.LogError(ex, "Failed to upload Cisco advisory {AdvisoryId} to GridFS", advisory.AdvisoryId); | ||||
|                             throw; | ||||
|                         } | ||||
|  | ||||
|                         var recordId = existing?.Id ?? Guid.NewGuid(); | ||||
|                         var record = new DocumentRecord( | ||||
|                             recordId, | ||||
|                             SourceName, | ||||
|                             documentUri, | ||||
|                             now, | ||||
|                             sha, | ||||
|                             DocumentStatuses.PendingParse, | ||||
|                             "application/json", | ||||
|                             DefaultHeaders, | ||||
|                             BuildMetadata(advisory), | ||||
|                             Etag: null, | ||||
|                             LastModified: advisory.LastUpdated ?? advisory.FirstPublished ?? now, | ||||
|                             GridFsId: gridFsId, | ||||
|                             ExpiresAt: null); | ||||
|  | ||||
|                         var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); | ||||
|                         pendingDocuments.Add(upserted.Id); | ||||
|                         pendingMappings.Remove(upserted.Id); | ||||
|                         added++; | ||||
|                         _diagnostics.FetchDocument(); | ||||
|  | ||||
|                         if (advisory.LastUpdated.HasValue) | ||||
|                         { | ||||
|                             latestModified = advisory.LastUpdated; | ||||
|                             latestAdvisoryId = advisory.AdvisoryId; | ||||
|                         } | ||||
|  | ||||
|                         if (added >= options.MaxAdvisoriesPerFetch) | ||||
|                         { | ||||
|                             break; | ||||
|                         } | ||||
|                     } | ||||
|  | ||||
|                     if (added >= options.MaxAdvisoriesPerFetch) | ||||
|                     { | ||||
|                         break; | ||||
|                     } | ||||
|  | ||||
|                     if (!page.HasMore) | ||||
|                     { | ||||
|                         break; | ||||
|                     } | ||||
|  | ||||
|                     await DelayAsync(options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|  | ||||
|                 if (added >= options.MaxAdvisoriesPerFetch) | ||||
|                 { | ||||
|                     break; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             var updatedCursor = cursor | ||||
|                 .WithPendingDocuments(pendingDocuments) | ||||
|                 .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|             if (latestModified.HasValue) | ||||
|             { | ||||
|                 updatedCursor = updatedCursor.WithCheckpoint(latestModified.Value, latestAdvisoryId ?? string.Empty); | ||||
|             } | ||||
|  | ||||
|             await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|             _logger.LogInformation( | ||||
|                 "Cisco fetch completed startDate={StartDate} pages={PagesFetched} added={Added} lastUpdated={LastUpdated} lastAdvisoryId={LastAdvisoryId}", | ||||
|                 startDate, | ||||
|                 pagesFetched, | ||||
|                 added, | ||||
|                 latestModified, | ||||
|                 latestAdvisoryId); | ||||
|         } | ||||
|         catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or MongoException) | ||||
|         { | ||||
|             _diagnostics.FetchFailure(); | ||||
|             _logger.LogError(ex, "Cisco fetch failed"); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToList(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|         var parsed = 0; | ||||
|         var failures = 0; | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning("Cisco document {DocumentId} missing GridFS payload", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             byte[] payload; | ||||
|             try | ||||
|             { | ||||
|                 payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogError(ex, "Cisco unable to download raw document {DocumentId}", documentId); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             CiscoRawAdvisory? raw; | ||||
|             try | ||||
|             { | ||||
|                 raw = JsonSerializer.Deserialize<CiscoRawAdvisory>(payload, RawSerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning(ex, "Cisco failed to deserialize raw document {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (raw is null) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning("Cisco raw document {DocumentId} produced null payload", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             CiscoAdvisoryDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = await _dtoFactory.CreateAsync(raw, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning(ex, "Cisco failed to build DTO for {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions); | ||||
|                 var dtoBson = BsonDocument.Parse(dtoJson); | ||||
|                 var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, _timeProvider.GetUtcNow()); | ||||
|                 await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 if (!pendingMappings.Contains(documentId)) | ||||
|                 { | ||||
|                     pendingMappings.Add(documentId); | ||||
|                 } | ||||
|  | ||||
|                 _diagnostics.ParseSuccess(); | ||||
|                 parsed++; | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogError(ex, "Cisco failed to persist DTO for {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingDocuments.Remove(documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (parsed > 0 || failures > 0) | ||||
|         { | ||||
|             _logger.LogInformation( | ||||
|                 "Cisco parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}", | ||||
|                 parsed, | ||||
|                 failures, | ||||
|                 pendingDocuments.Count, | ||||
|                 pendingMappings.Count); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|         var mapped = 0; | ||||
|         var failures = 0; | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.MapFailure(); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (dtoRecord is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(); | ||||
|                 _logger.LogWarning("Cisco document {DocumentId} missing DTO payload", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             CiscoAdvisoryDto? dto; | ||||
|             try | ||||
|             { | ||||
|                 var json = dtoRecord.Payload.ToJson(); | ||||
|                 dto = JsonSerializer.Deserialize<CiscoAdvisoryDto>(json, DtoSerializerOptions); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(); | ||||
|                 _logger.LogWarning(ex, "Cisco failed to deserialize DTO for document {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (dto is null) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(); | ||||
|                 _logger.LogWarning("Cisco DTO for document {DocumentId} evaluated to null", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var advisory = CiscoMapper.Map(dto, document, dtoRecord); | ||||
|                 await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 _diagnostics.MapSuccess(); | ||||
|                 _diagnostics.MapAffected(advisory.AffectedPackages.Length); | ||||
|                 mapped++; | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.MapFailure(); | ||||
|                 _logger.LogError(ex, "Cisco mapping failed for document {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 failures++; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (mapped > 0 || failures > 0) | ||||
|         { | ||||
|             _logger.LogInformation( | ||||
|                 "Cisco map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}", | ||||
|                 mapped, | ||||
|                 failures, | ||||
|                 pendingMappings.Count); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static string ComputeSha256(byte[] payload) | ||||
|     { | ||||
|         Span<byte> hash = stackalloc byte[32]; | ||||
|         SHA256.HashData(payload, hash); | ||||
|         return Convert.ToHexString(hash).ToLowerInvariant(); | ||||
|     } | ||||
|  | ||||
|     private static bool ShouldProcess(CiscoAdvisoryItem advisory, DateTimeOffset? checkpoint, string? checkpointId) | ||||
|     { | ||||
|         if (checkpoint is null || advisory.LastUpdated is null) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         var comparison = advisory.LastUpdated.Value.CompareTo(checkpoint.Value); | ||||
|         if (comparison > 0) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         if (comparison < 0) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(checkpointId)) | ||||
|         { | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return string.Compare(advisory.AdvisoryId, checkpointId, StringComparison.OrdinalIgnoreCase) > 0; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, string> BuildMetadata(CiscoAdvisoryItem advisory) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ["cisco.advisoryId"] = advisory.AdvisoryId, | ||||
|         }; | ||||
|  | ||||
|         if (advisory.LastUpdated.HasValue) | ||||
|         { | ||||
|             metadata["cisco.lastUpdated"] = advisory.LastUpdated.Value.ToString("O", CultureInfo.InvariantCulture); | ||||
|         } | ||||
|  | ||||
|         if (advisory.FirstPublished.HasValue) | ||||
|         { | ||||
|             metadata["cisco.firstPublished"] = advisory.FirstPublished.Value.ToString("O", CultureInfo.InvariantCulture); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(advisory.Severity)) | ||||
|         { | ||||
|             metadata["cisco.severity"] = advisory.Severity!; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(advisory.CsafUrl)) | ||||
|         { | ||||
|             metadata["cisco.csafUrl"] = advisory.CsafUrl!; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(advisory.CvrfUrl)) | ||||
|         { | ||||
|             metadata["cisco.cvrfUrl"] = advisory.CvrfUrl!; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(advisory.PublicationUrl)) | ||||
|         { | ||||
|             metadata["cisco.publicationUrl"] = advisory.PublicationUrl!; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(advisory.CvssBaseScore)) | ||||
|         { | ||||
|             metadata["cisco.cvssBaseScore"] = advisory.CvssBaseScore!; | ||||
|         } | ||||
|  | ||||
|         if (advisory.Cves.Count > 0) | ||||
|         { | ||||
|             metadata["cisco.cves"] = string.Join(",", advisory.Cves); | ||||
|         } | ||||
|  | ||||
|         if (advisory.BugIds.Count > 0) | ||||
|         { | ||||
|             metadata["cisco.bugIds"] = string.Join(",", advisory.BugIds); | ||||
|         } | ||||
|  | ||||
|         if (advisory.ProductNames.Count > 0) | ||||
|         { | ||||
|             metadata["cisco.productNames"] = string.Join(";", advisory.ProductNames.Take(10)); | ||||
|         } | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
|  | ||||
|     private static DateOnly DetermineStartDate(CiscoCursor cursor, DateTimeOffset now, CiscoOptions options) | ||||
|     { | ||||
|         if (cursor.LastModified.HasValue) | ||||
|         { | ||||
|             return DateOnly.FromDateTime(cursor.LastModified.Value.UtcDateTime.Date); | ||||
|         } | ||||
|  | ||||
|         var baseline = now - options.InitialBackfillWindow; | ||||
|         return DateOnly.FromDateTime(baseline.UtcDateTime.Date); | ||||
|     } | ||||
|  | ||||
|     private string BuildDocumentUri(string advisoryId) | ||||
|     { | ||||
|         var baseUri = _options.Value.BaseUri; | ||||
|         var relative = $"advisories/{Uri.EscapeDataString(advisoryId)}"; | ||||
|         return new Uri(baseUri, relative).ToString(); | ||||
|     } | ||||
|  | ||||
|     private async Task<CiscoCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? CiscoCursor.Empty : CiscoCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(CiscoCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBson(); | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private static Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (delay <= TimeSpan.Zero) | ||||
|         { | ||||
|             return Task.CompletedTask; | ||||
|         } | ||||
|  | ||||
|         return Task.Delay(delay, cancellationToken); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| public sealed class CiscoDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "feedser:sources:cisco"; | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddCiscoConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         services.AddTransient<CiscoFetchJob>(); | ||||
|         services.AddTransient<CiscoParseJob>(); | ||||
|         services.AddTransient<CiscoMapJob>(); | ||||
|  | ||||
|         services.PostConfigure<JobSchedulerOptions>(options => | ||||
|         { | ||||
|             EnsureJob(options, CiscoJobKinds.Fetch, typeof(CiscoFetchJob)); | ||||
|             EnsureJob(options, CiscoJobKinds.Parse, typeof(CiscoParseJob)); | ||||
|             EnsureJob(options, CiscoJobKinds.Map, typeof(CiscoMapJob)); | ||||
|         }); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
|  | ||||
|     private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) | ||||
|     { | ||||
|         if (options.Definitions.ContainsKey(kind)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         options.Definitions[kind] = new JobDefinition( | ||||
|             kind, | ||||
|             jobType, | ||||
|             options.DefaultTimeout, | ||||
|             options.DefaultLeaseDuration, | ||||
|             CronExpression: null, | ||||
|             Enabled: true); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,68 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Common.Http; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| public static class CiscoServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddCiscoConnector(this IServiceCollection services, Action<CiscoOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<CiscoOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static opts => opts.Validate()) | ||||
|             .ValidateOnStart(); | ||||
|  | ||||
|         services.TryAddSingleton<TimeProvider>(_ => TimeProvider.System); | ||||
|         services.AddSingleton<CiscoDiagnostics>(); | ||||
|         services.AddSingleton<CiscoAccessTokenProvider>(); | ||||
|         services.AddTransient<CiscoOAuthMessageHandler>(); | ||||
|  | ||||
|         services.AddHttpClient(CiscoOptions.AuthHttpClientName) | ||||
|             .ConfigureHttpClient((sp, client) => | ||||
|             { | ||||
|                 var options = sp.GetRequiredService<IOptions<CiscoOptions>>().Value; | ||||
|                 client.Timeout = options.RequestTimeout; | ||||
|                 client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Feedser.Cisco/1.0"); | ||||
|                 client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); | ||||
|                 if (options.TokenEndpoint is not null) | ||||
|                 { | ||||
|                     client.BaseAddress = new Uri(options.TokenEndpoint.GetLeftPart(UriPartial.Authority)); | ||||
|                 } | ||||
|             }); | ||||
|  | ||||
|         services.AddSourceHttpClient(CiscoOptions.HttpClientName, static (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<CiscoOptions>>().Value; | ||||
|             clientOptions.Timeout = options.RequestTimeout; | ||||
|             clientOptions.UserAgent = "StellaOps.Feedser.Cisco/1.0"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.BaseUri.Host); | ||||
|             clientOptions.AllowedHosts.Add("sec.cloudapps.cisco.com"); | ||||
|             clientOptions.AllowedHosts.Add("www.cisco.com"); | ||||
|             clientOptions.MaxAttempts = 5; | ||||
|             clientOptions.BaseDelay = TimeSpan.FromSeconds(2); | ||||
|         }).AddHttpMessageHandler<CiscoOAuthMessageHandler>(); | ||||
|  | ||||
|         services.AddSingleton<CiscoOpenVulnClient>(sp => | ||||
|         { | ||||
|             var fetchService = sp.GetRequiredService<SourceFetchService>(); | ||||
|             var optionsMonitor = sp.GetRequiredService<IOptionsMonitor<CiscoOptions>>(); | ||||
|             var logger = sp.GetRequiredService<Microsoft.Extensions.Logging.ILogger<CiscoOpenVulnClient>>(); | ||||
|             return new CiscoOpenVulnClient(fetchService, optionsMonitor, logger, VndrCiscoConnectorPlugin.SourceName); | ||||
|         }); | ||||
|  | ||||
|         services.AddSingleton<ICiscoCsafClient, CiscoCsafClient>(); | ||||
|         services.AddSingleton<CiscoDtoFactory>(); | ||||
|         services.AddTransient<CiscoConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -1,29 +0,0 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| public sealed class VndrCiscoConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public string Name => "vndr-cisco"; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => true; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); | ||||
|  | ||||
|     private sealed class StubConnector : IFeedConnector | ||||
|     { | ||||
|         public StubConnector(string sourceName) => SourceName = sourceName; | ||||
|  | ||||
|         public string SourceName { get; } | ||||
|  | ||||
|         public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|     } | ||||
| } | ||||
|  | ||||
| @@ -0,0 +1,124 @@ | ||||
| using System.Globalization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
|  | ||||
| public sealed class CiscoOptions | ||||
| { | ||||
|     public const string HttpClientName = "feedser.source.vndr.cisco"; | ||||
|     public const string AuthHttpClientName = "feedser.source.vndr.cisco.auth"; | ||||
|  | ||||
|     public Uri BaseUri { get; set; } = new("https://api.cisco.com/security/advisories/v2/", UriKind.Absolute); | ||||
|  | ||||
|     public Uri TokenEndpoint { get; set; } = new("https://id.cisco.com/oauth2/default/v1/token", UriKind.Absolute); | ||||
|  | ||||
|     public string ClientId { get; set; } = string.Empty; | ||||
|  | ||||
|     public string ClientSecret { get; set; } = string.Empty; | ||||
|  | ||||
|     public int PageSize { get; set; } = 100; | ||||
|  | ||||
|     public int MaxPagesPerFetch { get; set; } = 5; | ||||
|  | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 200; | ||||
|  | ||||
|     public TimeSpan InitialBackfillWindow { get; set; } = TimeSpan.FromDays(30); | ||||
|  | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); | ||||
|  | ||||
|     public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); | ||||
|  | ||||
|     public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     public TimeSpan TokenRefreshSkew { get; set; } = TimeSpan.FromMinutes(1); | ||||
|  | ||||
|     public string LastModifiedPathTemplate { get; set; } = "advisories/lastmodified/{0}"; | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (BaseUri is null || !BaseUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco BaseUri must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (TokenEndpoint is null || !TokenEndpoint.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco TokenEndpoint must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ClientId)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco clientId must be configured."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ClientSecret)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco clientSecret must be configured."); | ||||
|         } | ||||
|  | ||||
|         if (PageSize is < 1 or > 100) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco PageSize must be between 1 and 100."); | ||||
|         } | ||||
|  | ||||
|         if (MaxPagesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco MaxPagesPerFetch must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco MaxAdvisoriesPerFetch must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfillWindow <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco InitialBackfillWindow must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco RequestDelay cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (RequestTimeout <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco RequestTimeout must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (FailureBackoff <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco FailureBackoff must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (TokenRefreshSkew < TimeSpan.FromSeconds(5)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco TokenRefreshSkew must be at least 5 seconds."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(LastModifiedPathTemplate)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco LastModifiedPathTemplate must be configured."); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public Uri BuildLastModifiedUri(DateOnly date, int pageIndex, int pageSize) | ||||
|     { | ||||
|         if (pageIndex < 1) | ||||
|         { | ||||
|             throw new ArgumentOutOfRangeException(nameof(pageIndex), pageIndex, "Page index must be >= 1."); | ||||
|         } | ||||
|  | ||||
|         if (pageSize is < 1 or > 100) | ||||
|         { | ||||
|             throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, "Page size must be between 1 and 100."); | ||||
|         } | ||||
|  | ||||
|         var path = string.Format(CultureInfo.InvariantCulture, LastModifiedPathTemplate, date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); | ||||
|         var builder = new UriBuilder(BaseUri); | ||||
|         var basePath = builder.Path.TrimEnd('/'); | ||||
|         builder.Path = $"{basePath}/{path}".Replace("//", "/", StringComparison.Ordinal); | ||||
|         var query = $"pageIndex={pageIndex.ToString(CultureInfo.InvariantCulture)}&pageSize={pageSize.ToString(CultureInfo.InvariantCulture)}"; | ||||
|         builder.Query = string.IsNullOrEmpty(builder.Query) ? query : builder.Query.TrimStart('?') + "&" + query; | ||||
|         return builder.Uri; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,145 @@ | ||||
| using System.Net.Http.Headers; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| internal sealed class CiscoAccessTokenProvider : IDisposable | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|     }; | ||||
|  | ||||
|     private readonly IHttpClientFactory _httpClientFactory; | ||||
|     private readonly IOptionsMonitor<CiscoOptions> _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<CiscoAccessTokenProvider> _logger; | ||||
|     private readonly SemaphoreSlim _refreshLock = new(1, 1); | ||||
|  | ||||
|     private volatile AccessToken? _cached; | ||||
|     private bool _disposed; | ||||
|  | ||||
|     public CiscoAccessTokenProvider( | ||||
|         IHttpClientFactory httpClientFactory, | ||||
|         IOptionsMonitor<CiscoOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<CiscoAccessTokenProvider> logger) | ||||
|     { | ||||
|         _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); | ||||
|         _options = options ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<string> GetTokenAsync(CancellationToken cancellationToken) | ||||
|         => await GetTokenInternalAsync(forceRefresh: false, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|     public void Invalidate() | ||||
|         => _cached = null; | ||||
|  | ||||
|     private async Task<string> GetTokenInternalAsync(bool forceRefresh, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ThrowIfDisposed(); | ||||
|  | ||||
|         var options = _options.CurrentValue; | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var cached = _cached; | ||||
|         if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) | ||||
|         { | ||||
|             return cached.Value; | ||||
|         } | ||||
|  | ||||
|         await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); | ||||
|         try | ||||
|         { | ||||
|             cached = _cached; | ||||
|             now = _timeProvider.GetUtcNow(); | ||||
|             if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) | ||||
|             { | ||||
|                 return cached.Value; | ||||
|             } | ||||
|  | ||||
|             var fresh = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); | ||||
|             _cached = fresh; | ||||
|             return fresh.Value; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             _refreshLock.Release(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private async Task<AccessToken> RequestTokenAsync(CiscoOptions options, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var client = _httpClientFactory.CreateClient(CiscoOptions.AuthHttpClientName); | ||||
|         client.Timeout = options.RequestTimeout; | ||||
|  | ||||
|         using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); | ||||
|         request.Headers.Accept.Clear(); | ||||
|         request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); | ||||
|  | ||||
|         var content = new FormUrlEncodedContent(new Dictionary<string, string> | ||||
|         { | ||||
|             ["grant_type"] = "client_credentials", | ||||
|             ["client_id"] = options.ClientId, | ||||
|             ["client_secret"] = options.ClientSecret, | ||||
|         }); | ||||
|  | ||||
|         request.Content = content; | ||||
|  | ||||
|         using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         if (!response.IsSuccessStatusCode) | ||||
|         { | ||||
|             var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); | ||||
|             var message = $"Cisco OAuth token request failed with status {(int)response.StatusCode} {response.StatusCode}."; | ||||
|             _logger.LogError("Cisco openVuln token request failed: {Message}; response={Preview}", message, preview); | ||||
|             throw new HttpRequestException(message); | ||||
|         } | ||||
|  | ||||
|         await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var payload = await JsonSerializer.DeserializeAsync<TokenResponse>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|         if (payload is null || string.IsNullOrWhiteSpace(payload.AccessToken)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco OAuth token response did not include an access token."); | ||||
|         } | ||||
|  | ||||
|         var expiresIn = payload.ExpiresIn > 0 ? TimeSpan.FromSeconds(payload.ExpiresIn) : TimeSpan.FromHours(1); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var expiresAt = now + expiresIn; | ||||
|         _logger.LogInformation("Cisco openVuln token issued; expires in {ExpiresIn}", expiresIn); | ||||
|         return new AccessToken(payload.AccessToken, expiresAt); | ||||
|     } | ||||
|  | ||||
|     public async Task<string> RefreshAsync(CancellationToken cancellationToken) | ||||
|         => await GetTokenInternalAsync(forceRefresh: true, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|     private void ThrowIfDisposed() | ||||
|     { | ||||
|         if (_disposed) | ||||
|         { | ||||
|             throw new ObjectDisposedException(nameof(CiscoAccessTokenProvider)); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void Dispose() | ||||
|     { | ||||
|         if (_disposed) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         _refreshLock.Dispose(); | ||||
|         _disposed = true; | ||||
|     } | ||||
|  | ||||
|     private sealed record AccessToken(string Value, DateTimeOffset ExpiresAt); | ||||
|  | ||||
|     private sealed record TokenResponse( | ||||
|         [property: JsonPropertyName("access_token")] string AccessToken, | ||||
|         [property: JsonPropertyName("expires_in")] int ExpiresIn, | ||||
|         [property: JsonPropertyName("token_type")] string? TokenType); | ||||
| } | ||||
| @@ -0,0 +1,25 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public sealed record CiscoAdvisoryDto( | ||||
|     string AdvisoryId, | ||||
|     string Title, | ||||
|     string? Summary, | ||||
|     string? Severity, | ||||
|     DateTimeOffset? Published, | ||||
|     DateTimeOffset? Updated, | ||||
|     string? PublicationUrl, | ||||
|     string? CsafUrl, | ||||
|     string? CvrfUrl, | ||||
|     double? CvssBaseScore, | ||||
|     IReadOnlyList<string> Cves, | ||||
|     IReadOnlyList<string> BugIds, | ||||
|     IReadOnlyList<CiscoAffectedProductDto> Products); | ||||
|  | ||||
| public sealed record CiscoAffectedProductDto( | ||||
|     string Name, | ||||
|     string? ProductId, | ||||
|     string? Version, | ||||
|     IReadOnlyCollection<string> Statuses); | ||||
| @@ -0,0 +1,64 @@ | ||||
| using System; | ||||
| using System.IO; | ||||
| using System.Net.Http; | ||||
| using System.Text; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public interface ICiscoCsafClient | ||||
| { | ||||
|     Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken); | ||||
| } | ||||
|  | ||||
| public class CiscoCsafClient : ICiscoCsafClient | ||||
| { | ||||
|     private static readonly string[] AcceptHeaders = { "application/json", "application/csaf+json", "application/vnd.cisco.csaf+json" }; | ||||
|  | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly ILogger<CiscoCsafClient> _logger; | ||||
|  | ||||
|     public CiscoCsafClient(SourceFetchService fetchService, ILogger<CiscoCsafClient> logger) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public virtual async Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(url)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||
|         { | ||||
|             _logger.LogWarning("Cisco CSAF URL '{Url}' is not a valid absolute URI.", url); | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var request = new SourceFetchRequest(CiscoOptions.HttpClientName, VndrCiscoConnectorPlugin.SourceName, uri) | ||||
|             { | ||||
|                 AcceptHeaders = AcceptHeaders, | ||||
|             }; | ||||
|  | ||||
|             var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|             if (!result.IsSuccess || result.Content is null) | ||||
|             { | ||||
|                 _logger.LogWarning("Cisco CSAF download returned status {Status} for {Url}", result.StatusCode, url); | ||||
|                 return null; | ||||
|             } | ||||
|  | ||||
|             return System.Text.Encoding.UTF8.GetString(result.Content); | ||||
|         } | ||||
|         catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException or InvalidOperationException) | ||||
|         { | ||||
|             _logger.LogWarning(ex, "Cisco CSAF download failed for {Url}", url); | ||||
|             return null; | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,9 @@ | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| internal sealed record CiscoCsafData( | ||||
|     IReadOnlyDictionary<string, CiscoCsafProduct> Products, | ||||
|     IReadOnlyDictionary<string, IReadOnlyCollection<string>> ProductStatuses); | ||||
|  | ||||
| internal sealed record CiscoCsafProduct(string ProductId, string Name); | ||||
| @@ -0,0 +1,123 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text.Json; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| internal static class CiscoCsafParser | ||||
| { | ||||
|     public static CiscoCsafData Parse(string content) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(content)) | ||||
|         { | ||||
|             return new CiscoCsafData( | ||||
|                 Products: new Dictionary<string, CiscoCsafProduct>(0, StringComparer.OrdinalIgnoreCase), | ||||
|                 ProductStatuses: new Dictionary<string, IReadOnlyCollection<string>>(0, StringComparer.OrdinalIgnoreCase)); | ||||
|         } | ||||
|  | ||||
|         using var document = JsonDocument.Parse(content); | ||||
|         var root = document.RootElement; | ||||
|  | ||||
|         var products = ParseProducts(root); | ||||
|         var statuses = ParseStatuses(root); | ||||
|  | ||||
|         return new CiscoCsafData(products, statuses); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, CiscoCsafProduct> ParseProducts(JsonElement root) | ||||
|     { | ||||
|         var dictionary = new Dictionary<string, CiscoCsafProduct>(StringComparer.OrdinalIgnoreCase); | ||||
|         if (!root.TryGetProperty("product_tree", out var productTree)) | ||||
|         { | ||||
|             return dictionary; | ||||
|         } | ||||
|  | ||||
|         if (productTree.TryGetProperty("full_product_names", out var fullProductNames) | ||||
|             && fullProductNames.ValueKind == JsonValueKind.Array) | ||||
|         { | ||||
|             foreach (var entry in fullProductNames.EnumerateArray()) | ||||
|             { | ||||
|                 var productId = entry.TryGetProperty("product_id", out var idElement) && idElement.ValueKind == JsonValueKind.String | ||||
|                     ? idElement.GetString() | ||||
|                     : null; | ||||
|  | ||||
|                 if (string.IsNullOrWhiteSpace(productId)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String | ||||
|                     ? nameElement.GetString() | ||||
|                     : null; | ||||
|  | ||||
|                 if (string.IsNullOrWhiteSpace(name)) | ||||
|                 { | ||||
|                     name = productId; | ||||
|                 } | ||||
|  | ||||
|                 dictionary[productId] = new CiscoCsafProduct(productId, name); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return dictionary; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ParseStatuses(JsonElement root) | ||||
|     { | ||||
|         var map = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) | ||||
|             || vulnerabilities.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return map.ToDictionary( | ||||
|                 static kvp => kvp.Key, | ||||
|                 static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(), | ||||
|                 StringComparer.OrdinalIgnoreCase); | ||||
|         } | ||||
|  | ||||
|         foreach (var vulnerability in vulnerabilities.EnumerateArray()) | ||||
|         { | ||||
|             if (!vulnerability.TryGetProperty("product_status", out var productStatus) | ||||
|                 || productStatus.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             foreach (var property in productStatus.EnumerateObject()) | ||||
|             { | ||||
|                 var statusLabel = property.Name; | ||||
|                 if (property.Value.ValueKind != JsonValueKind.Array) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 foreach (var productIdElement in property.Value.EnumerateArray()) | ||||
|                 { | ||||
|                     if (productIdElement.ValueKind != JsonValueKind.String) | ||||
|                     { | ||||
|                         continue; | ||||
|                     } | ||||
|  | ||||
|                     var productId = productIdElement.GetString(); | ||||
|                     if (string.IsNullOrWhiteSpace(productId)) | ||||
|                     { | ||||
|                         continue; | ||||
|                     } | ||||
|  | ||||
|                     if (!map.TryGetValue(productId, out var set)) | ||||
|                     { | ||||
|                         set = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|                         map[productId] = set; | ||||
|                     } | ||||
|  | ||||
|                     set.Add(statusLabel); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return map.ToDictionary( | ||||
|             static kvp => kvp.Key, | ||||
|             static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(), | ||||
|             StringComparer.OrdinalIgnoreCase); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										101
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCursor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										101
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCursor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,101 @@ | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| internal sealed record CiscoCursor( | ||||
|     DateTimeOffset? LastModified, | ||||
|     string? LastAdvisoryId, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>(); | ||||
|  | ||||
|     public static CiscoCursor Empty { get; } = new(null, null, EmptyGuidCollection, EmptyGuidCollection); | ||||
|  | ||||
|     public BsonDocument ToBson() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastModified.HasValue) | ||||
|         { | ||||
|             document["lastModified"] = LastModified.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(LastAdvisoryId)) | ||||
|         { | ||||
|             document["lastAdvisoryId"] = LastAdvisoryId; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static CiscoCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset? lastModified = null; | ||||
|         if (document.TryGetValue("lastModified", out var lastModifiedValue)) | ||||
|         { | ||||
|             lastModified = lastModifiedValue.BsonType switch | ||||
|             { | ||||
|                 BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), | ||||
|                 BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|                 _ => null, | ||||
|             }; | ||||
|         } | ||||
|  | ||||
|         string? lastAdvisoryId = null; | ||||
|         if (document.TryGetValue("lastAdvisoryId", out var idValue) && idValue.BsonType == BsonType.String) | ||||
|         { | ||||
|             var value = idValue.AsString.Trim(); | ||||
|             if (value.Length > 0) | ||||
|             { | ||||
|                 lastAdvisoryId = value; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|  | ||||
|         return new CiscoCursor(lastModified, lastAdvisoryId, pendingDocuments, pendingMappings); | ||||
|     } | ||||
|  | ||||
|     public CiscoCursor WithCheckpoint(DateTimeOffset lastModified, string advisoryId) | ||||
|         => this with | ||||
|         { | ||||
|             LastModified = lastModified.ToUniversalTime(), | ||||
|             LastAdvisoryId = string.IsNullOrWhiteSpace(advisoryId) ? null : advisoryId.Trim(), | ||||
|         }; | ||||
|  | ||||
|     public CiscoCursor WithPendingDocuments(IEnumerable<Guid>? documents) | ||||
|         => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidCollection }; | ||||
|  | ||||
|     public CiscoCursor WithPendingMappings(IEnumerable<Guid>? mappings) | ||||
|         => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidCollection }; | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key) | ||||
|     { | ||||
|         if (!document.TryGetValue(key, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidCollection; | ||||
|         } | ||||
|  | ||||
|         var results = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 results.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,82 @@ | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public sealed class CiscoDiagnostics : IDisposable | ||||
| { | ||||
|     public const string MeterName = "StellaOps.Feedser.Source.Vndr.Cisco"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _fetchDocuments; | ||||
|     private readonly Counter<long> _fetchFailures; | ||||
|     private readonly Counter<long> _fetchUnchanged; | ||||
|     private readonly Counter<long> _parseSuccess; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Counter<long> _mapSuccess; | ||||
|     private readonly Counter<long> _mapFailures; | ||||
|     private readonly Histogram<long> _mapAffected; | ||||
|  | ||||
|     public CiscoDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _fetchDocuments = _meter.CreateCounter<long>( | ||||
|             name: "cisco.fetch.documents", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco advisories fetched."); | ||||
|         _fetchFailures = _meter.CreateCounter<long>( | ||||
|             name: "cisco.fetch.failures", | ||||
|             unit: "operations", | ||||
|             description: "Number of Cisco fetch failures."); | ||||
|         _fetchUnchanged = _meter.CreateCounter<long>( | ||||
|             name: "cisco.fetch.unchanged", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco advisories skipped because they were unchanged."); | ||||
|         _parseSuccess = _meter.CreateCounter<long>( | ||||
|             name: "cisco.parse.success", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco documents parsed successfully."); | ||||
|         _parseFailures = _meter.CreateCounter<long>( | ||||
|             name: "cisco.parse.failures", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco documents that failed to parse."); | ||||
|         _mapSuccess = _meter.CreateCounter<long>( | ||||
|             name: "cisco.map.success", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco advisories mapped successfully."); | ||||
|         _mapFailures = _meter.CreateCounter<long>( | ||||
|             name: "cisco.map.failures", | ||||
|             unit: "documents", | ||||
|             description: "Number of Cisco advisories that failed to map to canonical form."); | ||||
|         _mapAffected = _meter.CreateHistogram<long>( | ||||
|             name: "cisco.map.affected.packages", | ||||
|             unit: "packages", | ||||
|             description: "Distribution of affected package counts emitted per Cisco advisory."); | ||||
|     } | ||||
|  | ||||
|     public Meter Meter => _meter; | ||||
|  | ||||
|     public void FetchDocument() => _fetchDocuments.Add(1); | ||||
|  | ||||
|     public void FetchFailure() => _fetchFailures.Add(1); | ||||
|  | ||||
|     public void FetchUnchanged() => _fetchUnchanged.Add(1); | ||||
|  | ||||
|     public void ParseSuccess() => _parseSuccess.Add(1); | ||||
|  | ||||
|     public void ParseFailure() => _parseFailures.Add(1); | ||||
|  | ||||
|     public void MapSuccess() => _mapSuccess.Add(1); | ||||
|  | ||||
|     public void MapFailure() => _mapFailures.Add(1); | ||||
|  | ||||
|     public void MapAffected(int count) | ||||
|     { | ||||
|         if (count >= 0) | ||||
|         { | ||||
|             _mapAffected.Record(count); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public void Dispose() => _meter.Dispose(); | ||||
| } | ||||
| @@ -0,0 +1,190 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using System.Text.Json; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Feedser.Models; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public class CiscoDtoFactory | ||||
| { | ||||
|     private readonly ICiscoCsafClient _csafClient; | ||||
|     private readonly ILogger<CiscoDtoFactory> _logger; | ||||
|  | ||||
|     public CiscoDtoFactory(ICiscoCsafClient csafClient, ILogger<CiscoDtoFactory> logger) | ||||
|     { | ||||
|         _csafClient = csafClient ?? throw new ArgumentNullException(nameof(csafClient)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<CiscoAdvisoryDto> CreateAsync(CiscoRawAdvisory raw, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(raw); | ||||
|  | ||||
|         var advisoryId = raw.AdvisoryId?.Trim(); | ||||
|         if (string.IsNullOrWhiteSpace(advisoryId)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Cisco advisory is missing advisoryId."); | ||||
|         } | ||||
|  | ||||
|         var title = string.IsNullOrWhiteSpace(raw.AdvisoryTitle) ? advisoryId : raw.AdvisoryTitle!.Trim(); | ||||
|         var severity = SeverityNormalization.Normalize(raw.Sir); | ||||
|         var published = ParseDate(raw.FirstPublished); | ||||
|         var updated = ParseDate(raw.LastUpdated); | ||||
|  | ||||
|         CiscoCsafData? csafData = null; | ||||
|         if (!string.IsNullOrWhiteSpace(raw.CsafUrl)) | ||||
|         { | ||||
|             var csafContent = await _csafClient.TryFetchAsync(raw.CsafUrl, cancellationToken).ConfigureAwait(false); | ||||
|             if (!string.IsNullOrWhiteSpace(csafContent)) | ||||
|             { | ||||
|                 try | ||||
|                 { | ||||
|                     csafData = CiscoCsafParser.Parse(csafContent!); | ||||
|                 } | ||||
|                 catch (JsonException ex) | ||||
|                 { | ||||
|                     _logger.LogWarning(ex, "Cisco CSAF payload parsing failed for {AdvisoryId}", advisoryId); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var products = BuildProducts(raw, csafData); | ||||
|         var cves = NormalizeList(raw.Cves); | ||||
|         var bugIds = NormalizeList(raw.BugIds); | ||||
|         var cvss = ParseDouble(raw.CvssBaseScore); | ||||
|  | ||||
|         return new CiscoAdvisoryDto( | ||||
|             AdvisoryId: advisoryId, | ||||
|             Title: title, | ||||
|             Summary: string.IsNullOrWhiteSpace(raw.Summary) ? null : raw.Summary!.Trim(), | ||||
|             Severity: severity, | ||||
|             Published: published, | ||||
|             Updated: updated, | ||||
|             PublicationUrl: NormalizeUrl(raw.PublicationUrl), | ||||
|             CsafUrl: NormalizeUrl(raw.CsafUrl), | ||||
|             CvrfUrl: NormalizeUrl(raw.CvrfUrl), | ||||
|             CvssBaseScore: cvss, | ||||
|             Cves: cves, | ||||
|             BugIds: bugIds, | ||||
|             Products: products); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<CiscoAffectedProductDto> BuildProducts(CiscoRawAdvisory raw, CiscoCsafData? csafData) | ||||
|     { | ||||
|         var map = new Dictionary<string, CiscoAffectedProductDto>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         if (csafData is not null) | ||||
|         { | ||||
|             foreach (var entry in csafData.ProductStatuses) | ||||
|             { | ||||
|                 var productId = entry.Key; | ||||
|                 var name = csafData.Products.TryGetValue(productId, out var product) | ||||
|                     ? product.Name | ||||
|                     : productId; | ||||
|  | ||||
|                 var statuses = NormalizeStatuses(entry.Value); | ||||
|                 map[name] = new CiscoAffectedProductDto( | ||||
|                     Name: name, | ||||
|                     ProductId: productId, | ||||
|                     Version: raw.Version?.Trim(), | ||||
|                     Statuses: statuses); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var rawProducts = NormalizeList(raw.ProductNames); | ||||
|         foreach (var productName in rawProducts) | ||||
|         { | ||||
|             if (map.ContainsKey(productName)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             map[productName] = new CiscoAffectedProductDto( | ||||
|                 Name: productName, | ||||
|                 ProductId: null, | ||||
|                 Version: raw.Version?.Trim(), | ||||
|                 Statuses: new[] { AffectedPackageStatusCatalog.KnownAffected }); | ||||
|         } | ||||
|  | ||||
|         return map.Count == 0 | ||||
|             ? Array.Empty<CiscoAffectedProductDto>() | ||||
|             : map.Values | ||||
|                 .OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase) | ||||
|                 .ThenBy(static p => p.ProductId, StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<string> NormalizeStatuses(IEnumerable<string> statuses) | ||||
|     { | ||||
|         var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var status in statuses) | ||||
|         { | ||||
|             if (AffectedPackageStatusCatalog.TryNormalize(status, out var normalized)) | ||||
|             { | ||||
|                 set.Add(normalized); | ||||
|             } | ||||
|             else if (!string.IsNullOrWhiteSpace(status)) | ||||
|             { | ||||
|                 set.Add(status.Trim().ToLowerInvariant()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (set.Count == 0) | ||||
|         { | ||||
|             set.Add(AffectedPackageStatusCatalog.KnownAffected); | ||||
|         } | ||||
|  | ||||
|         return set; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> NormalizeList(IEnumerable<string>? items) | ||||
|     { | ||||
|         if (items is null) | ||||
|         { | ||||
|             return Array.Empty<string>(); | ||||
|         } | ||||
|  | ||||
|         var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var item in items) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(item)) | ||||
|             { | ||||
|                 set.Add(item.Trim()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return set.Count == 0 ? Array.Empty<string>() : set.ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static double? ParseDouble(string? value) | ||||
|         => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) | ||||
|             ? parsed | ||||
|             : null; | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) | ||||
|         { | ||||
|             return parsed.ToUniversalTime(); | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizeUrl(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return Uri.TryCreate(value.Trim(), UriKind.Absolute, out var uri) ? uri.ToString() : null; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										263
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoMapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										263
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoMapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,263 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common.Packages; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public static class CiscoMapper | ||||
| { | ||||
|     public static Advisory Map(CiscoAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|         ArgumentNullException.ThrowIfNull(dtoRecord); | ||||
|  | ||||
|         var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); | ||||
|         var fetchProvenance = new AdvisoryProvenance( | ||||
|             VndrCiscoConnectorPlugin.SourceName, | ||||
|             "document", | ||||
|             document.Uri, | ||||
|             document.FetchedAt.ToUniversalTime()); | ||||
|  | ||||
|         var mapProvenance = new AdvisoryProvenance( | ||||
|             VndrCiscoConnectorPlugin.SourceName, | ||||
|             "map", | ||||
|             dto.AdvisoryId, | ||||
|             recordedAt); | ||||
|  | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var affected = BuildAffectedPackages(dto, recordedAt); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey: dto.AdvisoryId, | ||||
|             title: dto.Title, | ||||
|             summary: dto.Summary, | ||||
|             language: "en", | ||||
|             published: dto.Published, | ||||
|             modified: dto.Updated, | ||||
|             severity: dto.Severity, | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             references: references, | ||||
|             affectedPackages: affected, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { fetchProvenance, mapProvenance }); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> BuildAliases(CiscoAdvisoryDto dto) | ||||
|     { | ||||
|         var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             dto.AdvisoryId, | ||||
|         }; | ||||
|  | ||||
|         foreach (var cve in dto.Cves) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(cve)) | ||||
|             { | ||||
|                 set.Add(cve.Trim()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         foreach (var bugId in dto.BugIds) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(bugId)) | ||||
|             { | ||||
|                 set.Add(bugId.Trim()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (dto.PublicationUrl is not null) | ||||
|         { | ||||
|             set.Add(dto.PublicationUrl); | ||||
|         } | ||||
|  | ||||
|         return set.Count == 0 | ||||
|             ? Array.Empty<string>() | ||||
|             : set.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryReference> BuildReferences(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var list = new List<AdvisoryReference>(3); | ||||
|         AddReference(list, dto.PublicationUrl, "publication", recordedAt); | ||||
|         AddReference(list, dto.CvrfUrl, "cvrf", recordedAt); | ||||
|         AddReference(list, dto.CsafUrl, "csaf", recordedAt); | ||||
|  | ||||
|         return list.Count == 0 | ||||
|             ? Array.Empty<AdvisoryReference>() | ||||
|             : list.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase).ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static void AddReference(ICollection<AdvisoryReference> references, string? url, string kind, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(url)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             VndrCiscoConnectorPlugin.SourceName, | ||||
|             $"reference:{kind}", | ||||
|             uri.ToString(), | ||||
|             recordedAt); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             references.Add(new AdvisoryReference( | ||||
|                 url: uri.ToString(), | ||||
|                 kind: kind, | ||||
|                 sourceTag: null, | ||||
|                 summary: null, | ||||
|                 provenance: provenance)); | ||||
|         } | ||||
|         catch (ArgumentException) | ||||
|         { | ||||
|             // ignore invalid URLs | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Products.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Products.Count); | ||||
|         foreach (var product in dto.Products) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(product.Name)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var range = BuildVersionRange(product, recordedAt); | ||||
|             var statuses = BuildStatuses(product, recordedAt); | ||||
|             var provenance = new[] | ||||
|             { | ||||
|                 new AdvisoryProvenance( | ||||
|                     VndrCiscoConnectorPlugin.SourceName, | ||||
|                     "affected", | ||||
|                     product.ProductId ?? product.Name, | ||||
|                     recordedAt), | ||||
|             }; | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 type: AffectedPackageTypes.Vendor, | ||||
|                 identifier: product.Name, | ||||
|                 platform: null, | ||||
|                 versionRanges: range is null ? Array.Empty<AffectedVersionRange>() : new[] { range }, | ||||
|                 statuses: statuses, | ||||
|                 provenance: provenance, | ||||
|                 normalizedVersions: Array.Empty<NormalizedVersionRule>())); | ||||
|         } | ||||
|  | ||||
|         return packages.Count == 0 | ||||
|             ? Array.Empty<AffectedPackage>() | ||||
|             : packages.OrderBy(static p => p.Identifier, StringComparer.OrdinalIgnoreCase).ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static AffectedVersionRange? BuildVersionRange(CiscoAffectedProductDto product, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(product.Version)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var version = product.Version.Trim(); | ||||
|         RangePrimitives? primitives = null; | ||||
|         string rangeKind = "vendor"; | ||||
|         string? rangeExpression = version; | ||||
|  | ||||
|         if (PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) | ||||
|         { | ||||
|             var semver = new SemVerPrimitive( | ||||
|                 Introduced: null, | ||||
|                 IntroducedInclusive: true, | ||||
|                 Fixed: null, | ||||
|                 FixedInclusive: false, | ||||
|                 LastAffected: null, | ||||
|                 LastAffectedInclusive: true, | ||||
|                 ConstraintExpression: null, | ||||
|                 ExactValue: normalized); | ||||
|  | ||||
|             primitives = new RangePrimitives(semver, null, null, BuildVendorExtensions(product)); | ||||
|             rangeKind = "semver"; | ||||
|             rangeExpression = normalized; | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             primitives = new RangePrimitives(null, null, null, BuildVendorExtensions(product, includeVersion: true)); | ||||
|         } | ||||
|  | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             VndrCiscoConnectorPlugin.SourceName, | ||||
|             "range", | ||||
|             product.ProductId ?? product.Name, | ||||
|             recordedAt); | ||||
|  | ||||
|         return new AffectedVersionRange( | ||||
|             rangeKind: rangeKind, | ||||
|             introducedVersion: null, | ||||
|             fixedVersion: null, | ||||
|             lastAffectedVersion: null, | ||||
|             rangeExpression: rangeExpression, | ||||
|             provenance: provenance, | ||||
|             primitives: primitives); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, string>? BuildVendorExtensions(CiscoAffectedProductDto product, bool includeVersion = false) | ||||
|     { | ||||
|         var dictionary = new Dictionary<string, string>(StringComparer.Ordinal); | ||||
|         if (!string.IsNullOrWhiteSpace(product.ProductId)) | ||||
|         { | ||||
|             dictionary["cisco.productId"] = product.ProductId!; | ||||
|         } | ||||
|  | ||||
|         if (includeVersion && !string.IsNullOrWhiteSpace(product.Version)) | ||||
|         { | ||||
|             dictionary["cisco.version.raw"] = product.Version!; | ||||
|         } | ||||
|  | ||||
|         return dictionary.Count == 0 ? null : dictionary; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(CiscoAffectedProductDto product, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (product.Statuses is null || product.Statuses.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackageStatus>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<AffectedPackageStatus>(product.Statuses.Count); | ||||
|         foreach (var status in product.Statuses) | ||||
|         { | ||||
|             if (!AffectedPackageStatusCatalog.TryNormalize(status, out var normalized) | ||||
|                 || string.IsNullOrWhiteSpace(normalized)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 VndrCiscoConnectorPlugin.SourceName, | ||||
|                 "status", | ||||
|                 product.ProductId ?? product.Name, | ||||
|                 recordedAt); | ||||
|  | ||||
|             list.Add(new AffectedPackageStatus(normalized, provenance)); | ||||
|         } | ||||
|  | ||||
|         return list.Count == 0 ? Array.Empty<AffectedPackageStatus>() : list; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,101 @@ | ||||
| using System.IO; | ||||
| using System.Net; | ||||
| using System.Net.Http.Headers; | ||||
| using Microsoft.Extensions.Logging; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| internal sealed class CiscoOAuthMessageHandler : DelegatingHandler | ||||
| { | ||||
|     private readonly CiscoAccessTokenProvider _tokenProvider; | ||||
|     private readonly ILogger<CiscoOAuthMessageHandler> _logger; | ||||
|  | ||||
|     public CiscoOAuthMessageHandler( | ||||
|         CiscoAccessTokenProvider tokenProvider, | ||||
|         ILogger<CiscoOAuthMessageHandler> logger) | ||||
|     { | ||||
|         _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(request); | ||||
|  | ||||
|         HttpRequestMessage? retryTemplate = null; | ||||
|         try | ||||
|         { | ||||
|             retryTemplate = await CloneRequestAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (IOException) | ||||
|         { | ||||
|             // Unable to buffer content; retry will fail if needed. | ||||
|             retryTemplate = null; | ||||
|         } | ||||
|  | ||||
|         request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); | ||||
|         var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         if (response.StatusCode != HttpStatusCode.Unauthorized) | ||||
|         { | ||||
|             return response; | ||||
|         } | ||||
|  | ||||
|         response.Dispose(); | ||||
|         _logger.LogWarning("Cisco openVuln request returned 401 Unauthorized; refreshing access token."); | ||||
|         await _tokenProvider.RefreshAsync(cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (retryTemplate is null) | ||||
|         { | ||||
|             _tokenProvider.Invalidate(); | ||||
|             throw new HttpRequestException("Cisco openVuln request returned 401 Unauthorized and could not be retried."); | ||||
|         } | ||||
|  | ||||
|         retryTemplate.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var retryResponse = await base.SendAsync(retryTemplate, cancellationToken).ConfigureAwait(false); | ||||
|             if (retryResponse.StatusCode == HttpStatusCode.Unauthorized) | ||||
|             { | ||||
|                 _tokenProvider.Invalidate(); | ||||
|             } | ||||
|  | ||||
|             return retryResponse; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             retryTemplate.Dispose(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static async Task<HttpRequestMessage?> CloneRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var clone = new HttpRequestMessage(request.Method, request.RequestUri) | ||||
|         { | ||||
|             Version = request.Version, | ||||
|             VersionPolicy = request.VersionPolicy, | ||||
|         }; | ||||
|  | ||||
|         foreach (var header in request.Headers) | ||||
|         { | ||||
|             clone.Headers.TryAddWithoutValidation(header.Key, header.Value); | ||||
|         } | ||||
|  | ||||
|         if (request.Content is not null) | ||||
|         { | ||||
|             using var memory = new MemoryStream(); | ||||
|             await request.Content.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); | ||||
|             memory.Position = 0; | ||||
|             var buffer = memory.ToArray(); | ||||
|             var contentClone = new ByteArrayContent(buffer); | ||||
|             foreach (var header in request.Content.Headers) | ||||
|             { | ||||
|                 contentClone.Headers.TryAddWithoutValidation(header.Key, header.Value); | ||||
|             } | ||||
|  | ||||
|             clone.Content = contentClone; | ||||
|         } | ||||
|  | ||||
|         return clone; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,196 @@ | ||||
| using System.Diagnostics.CodeAnalysis; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using System.Text.Json.Serialization; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public sealed class CiscoOpenVulnClient | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         NumberHandling = JsonNumberHandling.AllowReadingFromString, | ||||
|     }; | ||||
|  | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly IOptionsMonitor<CiscoOptions> _options; | ||||
|     private readonly ILogger<CiscoOpenVulnClient> _logger; | ||||
|     private readonly string _sourceName; | ||||
|  | ||||
|     public CiscoOpenVulnClient( | ||||
|         SourceFetchService fetchService, | ||||
|         IOptionsMonitor<CiscoOptions> options, | ||||
|         ILogger<CiscoOpenVulnClient> logger, | ||||
|         string sourceName) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _options = options ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|         _sourceName = sourceName ?? throw new ArgumentNullException(nameof(sourceName)); | ||||
|     } | ||||
|  | ||||
|     internal async Task<CiscoAdvisoryPage?> FetchAsync(DateOnly date, int pageIndex, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var options = _options.CurrentValue; | ||||
|         var requestUri = options.BuildLastModifiedUri(date, pageIndex, options.PageSize); | ||||
|         var request = new SourceFetchRequest(CiscoOptions.HttpClientName, _sourceName, requestUri) | ||||
|         { | ||||
|             AcceptHeaders = new[] { "application/json" }, | ||||
|             TimeoutOverride = options.RequestTimeout, | ||||
|         }; | ||||
|  | ||||
|         var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         if (!result.IsSuccess || result.Content is null) | ||||
|         { | ||||
|             _logger.LogDebug("Cisco openVuln request returned empty payload for {Uri} (status {Status})", requestUri, result.StatusCode); | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return CiscoAdvisoryPage.Parse(result.Content); | ||||
|     } | ||||
| } | ||||
|  | ||||
| internal sealed record CiscoAdvisoryPage( | ||||
|     IReadOnlyList<CiscoAdvisoryItem> Advisories, | ||||
|     CiscoPagination Pagination) | ||||
| { | ||||
|     public bool HasMore => Pagination.PageIndex < Pagination.TotalPages; | ||||
|  | ||||
|     public static CiscoAdvisoryPage Parse(byte[] content) | ||||
|     { | ||||
|         using var document = JsonDocument.Parse(content); | ||||
|         var root = document.RootElement; | ||||
|         var advisories = new List<CiscoAdvisoryItem>(); | ||||
|  | ||||
|         if (root.TryGetProperty("advisories", out var advisoriesElement) && advisoriesElement.ValueKind == JsonValueKind.Array) | ||||
|         { | ||||
|             foreach (var advisory in advisoriesElement.EnumerateArray()) | ||||
|             { | ||||
|                 if (!TryCreateItem(advisory, out var item)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 advisories.Add(item); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var pagination = CiscoPagination.FromJson(root.TryGetProperty("pagination", out var paginationElement) ? paginationElement : default); | ||||
|         return new CiscoAdvisoryPage(advisories, pagination); | ||||
|     } | ||||
|  | ||||
|     private static bool TryCreateItem(JsonElement advisory, [NotNullWhen(true)] out CiscoAdvisoryItem? item) | ||||
|     { | ||||
|         var rawJson = advisory.GetRawText(); | ||||
|         var advisoryId = GetString(advisory, "advisoryId"); | ||||
|         if (string.IsNullOrWhiteSpace(advisoryId)) | ||||
|         { | ||||
|             item = null; | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var lastUpdated = ParseDate(GetString(advisory, "lastUpdated")); | ||||
|         var firstPublished = ParseDate(GetString(advisory, "firstPublished")); | ||||
|         var severity = GetString(advisory, "sir"); | ||||
|         var publicationUrl = GetString(advisory, "publicationUrl"); | ||||
|         var csafUrl = GetString(advisory, "csafUrl"); | ||||
|         var cvrfUrl = GetString(advisory, "cvrfUrl"); | ||||
|         var cvss = GetString(advisory, "cvssBaseScore"); | ||||
|  | ||||
|         var cves = ReadStringArray(advisory, "cves"); | ||||
|         var bugIds = ReadStringArray(advisory, "bugIDs"); | ||||
|         var productNames = ReadStringArray(advisory, "productNames"); | ||||
|  | ||||
|         item = new CiscoAdvisoryItem( | ||||
|             advisoryId, | ||||
|             lastUpdated, | ||||
|             firstPublished, | ||||
|             severity, | ||||
|             publicationUrl, | ||||
|             csafUrl, | ||||
|             cvrfUrl, | ||||
|             cvss, | ||||
|             cves, | ||||
|             bugIds, | ||||
|             productNames, | ||||
|             rawJson); | ||||
|         return true; | ||||
|     } | ||||
|  | ||||
|     private static string? GetString(JsonElement element, string propertyName) | ||||
|         => element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String | ||||
|             ? value.GetString() | ||||
|             : null; | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (DateTimeOffset.TryParse(value, out var parsed)) | ||||
|         { | ||||
|             return parsed.ToUniversalTime(); | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> ReadStringArray(JsonElement element, string property) | ||||
|     { | ||||
|         if (!element.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<string>(); | ||||
|         } | ||||
|  | ||||
|         var results = new List<string>(); | ||||
|         foreach (var child in value.EnumerateArray()) | ||||
|         { | ||||
|             if (child.ValueKind == JsonValueKind.String) | ||||
|             { | ||||
|                 var text = child.GetString(); | ||||
|                 if (!string.IsNullOrWhiteSpace(text)) | ||||
|                 { | ||||
|                     results.Add(text.Trim()); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
| } | ||||
|  | ||||
| internal sealed record CiscoAdvisoryItem( | ||||
|     string AdvisoryId, | ||||
|     DateTimeOffset? LastUpdated, | ||||
|     DateTimeOffset? FirstPublished, | ||||
|     string? Severity, | ||||
|     string? PublicationUrl, | ||||
|     string? CsafUrl, | ||||
|     string? CvrfUrl, | ||||
|     string? CvssBaseScore, | ||||
|     IReadOnlyList<string> Cves, | ||||
|     IReadOnlyList<string> BugIds, | ||||
|     IReadOnlyList<string> ProductNames, | ||||
|     string RawJson) | ||||
| { | ||||
|     public byte[] GetRawBytes() => Encoding.UTF8.GetBytes(RawJson); | ||||
| } | ||||
|  | ||||
| internal sealed record CiscoPagination(int PageIndex, int PageSize, int TotalPages, int TotalRecords) | ||||
| { | ||||
|     public static CiscoPagination FromJson(JsonElement element) | ||||
|     { | ||||
|         var pageIndex = element.TryGetProperty("pageIndex", out var index) && index.TryGetInt32(out var parsedIndex) ? parsedIndex : 1; | ||||
|         var pageSize = element.TryGetProperty("pageSize", out var size) && size.TryGetInt32(out var parsedSize) ? parsedSize : 0; | ||||
|         var totalPages = element.TryGetProperty("totalPages", out var pages) && pages.TryGetInt32(out var parsedPages) ? parsedPages : pageIndex; | ||||
|         var totalRecords = element.TryGetProperty("totalRecords", out var records) && records.TryGetInt32(out var parsedRecords) ? parsedRecords : 0; | ||||
|         return new CiscoPagination(pageIndex, pageSize, totalPages, totalRecords); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,64 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Text.Json.Serialization; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; | ||||
|  | ||||
| public class CiscoRawAdvisory | ||||
| { | ||||
|     [JsonPropertyName("advisoryId")] | ||||
|     public string? AdvisoryId { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("advisoryTitle")] | ||||
|     public string? AdvisoryTitle { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("publicationUrl")] | ||||
|     public string? PublicationUrl { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cvrfUrl")] | ||||
|     public string? CvrfUrl { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("csafUrl")] | ||||
|     public string? CsafUrl { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("summary")] | ||||
|     public string? Summary { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("sir")] | ||||
|     public string? Sir { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("firstPublished")] | ||||
|     public string? FirstPublished { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("lastUpdated")] | ||||
|     public string? LastUpdated { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("productNames")] | ||||
|     public List<string>? ProductNames { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("version")] | ||||
|     public string? Version { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("iosRelease")] | ||||
|     public string? IosRelease { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cves")] | ||||
|     public List<string>? Cves { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("bugIDs")] | ||||
|     public List<string>? BugIds { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cvssBaseScore")] | ||||
|     public string? CvssBaseScore { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cvssTemporalScore")] | ||||
|     public string? CvssTemporalScore { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cvssEnvironmentalScore")] | ||||
|     public string? CvssEnvironmentalScore { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("cvssBaseScoreVersion2")] | ||||
|     public string? CvssBaseScoreV2 { get; set; } | ||||
|  | ||||
|     [JsonPropertyName("status")] | ||||
|     public string? Status { get; set; } | ||||
| } | ||||
							
								
								
									
										46
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										46
									
								
								src/StellaOps.Feedser.Source.Vndr.Cisco/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,46 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| internal static class CiscoJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:vndr-cisco:fetch"; | ||||
|     public const string Parse = "source:vndr-cisco:parse"; | ||||
|     public const string Map = "source:vndr-cisco:map"; | ||||
| } | ||||
|  | ||||
| internal sealed class CiscoFetchJob : IJob | ||||
| { | ||||
|     private readonly CiscoConnector _connector; | ||||
|  | ||||
|     public CiscoFetchJob(CiscoConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class CiscoParseJob : IJob | ||||
| { | ||||
|     private readonly CiscoConnector _connector; | ||||
|  | ||||
|     public CiscoParseJob(CiscoConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.ParseAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class CiscoMapJob : IJob | ||||
| { | ||||
|     private readonly CiscoConnector _connector; | ||||
|  | ||||
|     public CiscoMapJob(CiscoConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.MapAsync(context.Services, cancellationToken); | ||||
| } | ||||
| @@ -6,11 +6,11 @@ | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|  | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|   | ||||
| @@ -2,10 +2,10 @@ | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |FEEDCONN-CISCO-02-001 Confirm Cisco PSIRT data source|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Selected openVuln REST API (`https://apix.cisco.com/security/advisories/v2/…`) as primary (structured JSON, CSAF/CVRF links) with RSS as fallback. Documented OAuth2 client-credentials flow (`cloudsso.cisco.com/as/token.oauth2`), baseline quotas (5 req/s, 30 req/min, 5 000 req/day), and pagination contract (`pageIndex`, `pageSize≤100`) in `docs/feedser-connector-research-20251011.md`.| | ||||
| |FEEDCONN-CISCO-02-002 Fetch pipeline & state persistence|BE-Conn-Cisco|Source.Common, Storage.Mongo|**TODO** – Implement fetch job using shared OAuth token cache, honor `Retry-After` on 429, and persist raw advisory payloads + CSAF links. Cursor strategy: `lastUpdated` + advisory ID, with incremental filters (`/lastmodified/{YYYY-MM-DD}` or `/year/{YYYY}` + paging).| | ||||
| |FEEDCONN-CISCO-02-003 Parser & DTO implementation|BE-Conn-Cisco|Source.Common|**TODO** – Map openVuln JSON fields (`advisoryId`, `advisoryTitle`, `cves`, `bugIDs`, `sir`, `productNames`, `version`, `cvssBaseScore`, `publicationUrl`, `cvrfUrl`, `csafUrl`). Normalize severity (SIR→Feedser severity), expand product list into affected packages, ingest CSAF where present to derive range primitives.| | ||||
| |FEEDCONN-CISCO-02-004 Canonical mapping & range primitives|BE-Conn-Cisco|Models|**TODO** – Map advisories into canonical records with aliases, references, range primitives (SemVer/IOS/ASA versions). Sync scheme decisions and deadlines via `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: baseline array `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"cisco:psirt:advisory-id"}]`; if IOS-specific comparer is required, capture sample payload and raise Models issue before introducing a new `scheme`.| | ||||
| |FEEDCONN-CISCO-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add fetch/parse/map regression tests; support `UPDATE_CISCO_FIXTURES=1`.| | ||||
| |FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector usage, update backlog when ready.| | ||||
| |FEEDCONN-CISCO-02-002 Fetch pipeline & state persistence|BE-Conn-Cisco|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Fetch job now streams openVuln pages with OAuth bearer handler, honours 429 `Retry-After`, persists per-advisory JSON + metadata into GridFS, and updates cursor (`lastModified`, advisory ID, pending docs).| | ||||
| |FEEDCONN-CISCO-02-003 Parser & DTO implementation|BE-Conn-Cisco|Source.Common|**DONE (2025-10-14)** – DTO factory normalizes SIR, folds CSAF product statuses, and persists `cisco.dto.v1` payloads (see `CiscoDtoFactory`).| | ||||
| |FEEDCONN-CISCO-02-004 Canonical mapping & range primitives|BE-Conn-Cisco|Models|**DONE (2025-10-14)** – `CiscoMapper` emits canonical advisories with vendor + SemVer primitives, provenance, and status tags.| | ||||
| |FEEDCONN-CISCO-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added unit tests (`StellaOps.Feedser.Source.Vndr.Cisco.Tests`) exercising DTO/mapper pipelines; `dotnet test` validated.| | ||||
| |FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Cisco diagnostics counters exposed and ops runbook updated with telemetry guidance (`docs/ops/feedser-cisco-operations.md`).| | ||||
| |FEEDCONN-CISCO-02-007 API selection decision memo|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Drafted decision matrix: openVuln (structured/delta filters, OAuth throttle) vs RSS (delayed/minimal metadata). Pending OAuth onboarding (`FEEDCONN-CISCO-02-008`) before final recommendation circulated.| | ||||
| |FEEDCONN-CISCO-02-008 OAuth client provisioning|Ops, BE-Conn-Cisco|Ops|**TODO** – Register openVuln application, capture client credential rotation steps, throttle limits, and Offline Kit secret distribution guidance.| | ||||
| |FEEDCONN-CISCO-02-008 OAuth client provisioning|Ops, BE-Conn-Cisco|Ops|**DONE (2025-10-14)** – `docs/ops/feedser-cisco-operations.md` documents OAuth provisioning/rotation, quotas, and Offline Kit distribution guidance.| | ||||
|   | ||||
| @@ -0,0 +1,21 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Vndr.Cisco; | ||||
|  | ||||
| public sealed class VndrCiscoConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "vndr-cisco"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) | ||||
|         => services.GetService<CiscoConnector>() is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return services.GetRequiredService<CiscoConnector>(); | ||||
|     } | ||||
| } | ||||
		Reference in New Issue
	
	Block a user