git.stella-ops.org/src/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs

using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;

namespace StellaOps.Excititor.Connectors.Cisco.CSAF;

public sealed class CiscoCsafConnector : VexConnectorBase
{
    private static readonly VexConnectorDescriptor DescriptorInstance = new(
        id: "excititor:cisco",
        kind: VexProviderKind.Vendor,
        displayName: "Cisco CSAF")
    {
        Tags = ImmutableArray.Create("cisco", "csaf"),
    };

    private readonly CiscoProviderMetadataLoader _metadataLoader;
    private readonly IHttpClientFactory _httpClientFactory;
    private readonly IVexConnectorStateRepository _stateRepository;
    private readonly IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>> _validators;
    private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);

    private CiscoConnectorOptions? _options;
    private CiscoProviderMetadataResult? _providerMetadata;

    public CiscoCsafConnector(
        CiscoProviderMetadataLoader metadataLoader,
        IHttpClientFactory httpClientFactory,
        IVexConnectorStateRepository stateRepository,
        IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>>? validators,
        ILogger<CiscoCsafConnector> logger,
        TimeProvider timeProvider)
        : base(DescriptorInstance, logger, timeProvider)
    {
        _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
        _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
        _validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<CiscoConnectorOptions>>();
    }

    public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
    {
        _options = VexConnectorOptionsBinder.Bind(
            Descriptor,
            settings,
            validators: _validators);

        _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
        LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary<string, object?>
        {
            ["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length,
            ["fromOffline"] = _providerMetadata.FromOfflineSnapshot,
        });
    }

    public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(context);

        if (_options is null)
        {
            throw new InvalidOperationException("Connector must be validated before fetch operations.");
        }

        if (_providerMetadata is null)
        {
            _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
        }

        var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
        var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
        var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
        var digestList = new List<string>(knownDigests);
        var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
        var latestTimestamp = state?.LastUpdated ?? since;
        var stateChanged = false;

        var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
        foreach (var directory in _providerMetadata.Provider.BaseUris)
        {
            await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false))
            {
                var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue;
                if (published <= since)
                {
                    continue;
                }

                using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
                contentResponse.EnsureSuccessStatusCode();
                var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);

                var rawDocument = CreateRawDocument(
                    VexDocumentFormat.Csaf,
                    advisory.DocumentUri,
                    payload,
                    BuildMetadata(builder => builder
                        .Add("cisco.csaf.advisoryId", advisory.Id)
                        .Add("cisco.csaf.revision", advisory.Revision)
                        .Add("cisco.csaf.published", advisory.Published?.ToString("O"))
                        .Add("cisco.csaf.modified", advisory.LastModified?.ToString("O"))
                        .Add("cisco.csaf.sha256", advisory.Sha256)));

                if (!digestSet.Add(rawDocument.Digest))
                {
                    continue;
                }

                await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
                digestList.Add(rawDocument.Digest);
                stateChanged = true;
                if (published > latestTimestamp)
                {
                    latestTimestamp = published;
                }

                yield return rawDocument;
            }
        }

        if (stateChanged)
        {
            var baseState = state ?? new VexConnectorState(
                Descriptor.Id,
                null,
                ImmutableArray<string>.Empty,
                ImmutableDictionary<string, string>.Empty,
                null,
                0,
                null,
                null);
            var newState = baseState with
            {
                LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
                DocumentDigests = digestList.ToImmutableArray(),
            };
            await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
        }
    }

    public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
        => throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing.");

    private async IAsyncEnumerable<CiscoAdvisoryEntry> EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        var nextUri = BuildIndexUri(directory, null);
        while (nextUri is not null)
        {
            using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
            response.EnsureSuccessStatusCode();
            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
            var page = JsonSerializer.Deserialize<CiscoAdvisoryIndex>(json, _serializerOptions);
            if (page?.Advisories is null)
            {
                yield break;
            }

            foreach (var advisory in page.Advisories)
            {
                if (string.IsNullOrWhiteSpace(advisory.Url))
                {
                    continue;
                }

                if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri))
                {
                    continue;
                }

                if (!documentUri.IsAbsoluteUri)
                {
                    documentUri = new Uri(directory, documentUri);
                }

                yield return new CiscoAdvisoryEntry(
                    advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(),
                    documentUri,
                    advisory.Revision,
                    advisory.Published,
                    advisory.LastModified,
                    advisory.Sha256);
            }

            nextUri = ResolveNextUri(directory, page.Next);
        }
    }

    private static Uri BuildIndexUri(Uri directory, string? relative)
    {
        if (string.IsNullOrWhiteSpace(relative))
        {
            var baseText = directory.ToString();
            if (!baseText.EndsWith('/'))
            {
                baseText += "/";
            }

            return new Uri(new Uri(baseText, UriKind.Absolute), "index.json");
        }

        if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute))
        {
            return absolute;
        }

        var baseTextRelative = directory.ToString();
        if (!baseTextRelative.EndsWith('/'))
        {
            baseTextRelative += "/";
        }

        return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative);
    }

    private static Uri? ResolveNextUri(Uri directory, string? next)
    {
        if (string.IsNullOrWhiteSpace(next))
        {
            return null;
        }

        return BuildIndexUri(directory, next);
    }

    private sealed record CiscoAdvisoryIndex
    {
        public List<CiscoAdvisory>? Advisories { get; init; }
        public string? Next { get; init; }
    }

    private sealed record CiscoAdvisory
    {
        public string? Id { get; init; }
        public string? Url { get; init; }
        public string? Revision { get; init; }
        public DateTimeOffset? Published { get; init; }
        public DateTimeOffset? LastModified { get; init; }
        public string? Sha256 { get; init; }
    }

    private sealed record CiscoAdvisoryEntry(
        string Id,
        Uri DocumentUri,
        string? Revision,
        DateTimeOffset? Published,
        DateTimeOffset? LastModified,
        string? Sha256);
}