using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text.Json; using Microsoft.Extensions.Logging; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; namespace StellaOps.Excititor.Connectors.Cisco.CSAF; public sealed class CiscoCsafConnector : VexConnectorBase { private static readonly VexConnectorDescriptor DescriptorInstance = new( id: "excititor:cisco", kind: VexProviderKind.Vendor, displayName: "Cisco CSAF") { Tags = ImmutableArray.Create("cisco", "csaf"), }; private readonly CiscoProviderMetadataLoader _metadataLoader; private readonly IHttpClientFactory _httpClientFactory; private readonly IVexConnectorStateRepository _stateRepository; private readonly IEnumerable> _validators; private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); private CiscoConnectorOptions? _options; private CiscoProviderMetadataResult? _providerMetadata; public CiscoCsafConnector( CiscoProviderMetadataLoader metadataLoader, IHttpClientFactory httpClientFactory, IVexConnectorStateRepository stateRepository, IEnumerable>? validators, ILogger logger, TimeProvider timeProvider) : base(DescriptorInstance, logger, timeProvider) { _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); _validators = validators ?? Array.Empty>(); } public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) { _options = VexConnectorOptionsBinder.Bind( Descriptor, settings, validators: _validators); _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary { ["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length, ["fromOffline"] = _providerMetadata.FromOfflineSnapshot, }); } public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(context); if (_options is null) { throw new InvalidOperationException("Connector must be validated before fetch operations."); } if (_providerMetadata is null) { _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); } var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); var digestList = new List(knownDigests); var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; var latestTimestamp = state?.LastUpdated ?? since; var stateChanged = false; var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); foreach (var directory in _providerMetadata.Provider.BaseUris) { await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false)) { var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue; if (published <= since) { continue; } using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); contentResponse.EnsureSuccessStatusCode(); var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); var rawDocument = CreateRawDocument( VexDocumentFormat.Csaf, advisory.DocumentUri, payload, BuildMetadata(builder => builder .Add("cisco.csaf.advisoryId", advisory.Id) .Add("cisco.csaf.revision", advisory.Revision) .Add("cisco.csaf.published", advisory.Published?.ToString("O")) .Add("cisco.csaf.modified", advisory.LastModified?.ToString("O")) .Add("cisco.csaf.sha256", advisory.Sha256))); if (!digestSet.Add(rawDocument.Digest)) { continue; } await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); digestList.Add(rawDocument.Digest); stateChanged = true; if (published > latestTimestamp) { latestTimestamp = published; } yield return rawDocument; } } if (stateChanged) { var baseState = state ?? new VexConnectorState( Descriptor.Id, null, ImmutableArray.Empty, ImmutableDictionary.Empty, null, 0, null, null); var newState = baseState with { LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, DocumentDigests = digestList.ToImmutableArray(), }; await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); } } public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) => throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing."); private async IAsyncEnumerable EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken) { var nextUri = BuildIndexUri(directory, null); while (nextUri is not null) { using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var page = JsonSerializer.Deserialize(json, _serializerOptions); if (page?.Advisories is null) { yield break; } foreach (var advisory in page.Advisories) { if (string.IsNullOrWhiteSpace(advisory.Url)) { continue; } if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri)) { continue; } if (!documentUri.IsAbsoluteUri) { documentUri = new Uri(directory, documentUri); } yield return new CiscoAdvisoryEntry( advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(), documentUri, advisory.Revision, advisory.Published, advisory.LastModified, advisory.Sha256); } nextUri = ResolveNextUri(directory, page.Next); } } private static Uri BuildIndexUri(Uri directory, string? relative) { if (string.IsNullOrWhiteSpace(relative)) { var baseText = directory.ToString(); if (!baseText.EndsWith('/')) { baseText += "/"; } return new Uri(new Uri(baseText, UriKind.Absolute), "index.json"); } if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute)) { return absolute; } var baseTextRelative = directory.ToString(); if (!baseTextRelative.EndsWith('/')) { baseTextRelative += "/"; } return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative); } private static Uri? ResolveNextUri(Uri directory, string? next) { if (string.IsNullOrWhiteSpace(next)) { return null; } return BuildIndexUri(directory, next); } private sealed record CiscoAdvisoryIndex { public List? Advisories { get; init; } public string? Next { get; init; } } private sealed record CiscoAdvisory { public string? Id { get; init; } public string? Url { get; init; } public string? Revision { get; init; } public DateTimeOffset? Published { get; init; } public DateTimeOffset? LastModified { get; init; } public string? Sha256 { get; init; } } private sealed record CiscoAdvisoryEntry( string Id, Uri DocumentUri, string? Revision, DateTimeOffset? Published, DateTimeOffset? LastModified, string? Sha256); }