using System.Collections.Immutable;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using AngleSharp.Html.Parser;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
using StellaOps.Feedser.Source.Ru.Nkcki.Internal;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Plugin;

namespace StellaOps.Feedser.Source.Ru.Nkcki;

public sealed class RuNkckiConnector : IFeedConnector
{
    private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
    {
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
        WriteIndented = false,
    };

    private static readonly string[] ListingAcceptHeaders =
    {
        "text/html",
        "application/xhtml+xml;q=0.9",
        "text/plain;q=0.1",
    };

    private static readonly string[] BulletinAcceptHeaders =
    {
        "application/zip",
        "application/octet-stream",
        "application/x-zip-compressed",
    };

    private readonly SourceFetchService _fetchService;
    private readonly RawDocumentStorage _rawDocumentStorage;
    private readonly IDocumentStore _documentStore;
    private readonly IDtoStore _dtoStore;
    private readonly IAdvisoryStore _advisoryStore;
    private readonly ISourceStateRepository _stateRepository;
    private readonly RuNkckiOptions _options;
    private readonly TimeProvider _timeProvider;
    private readonly RuNkckiDiagnostics _diagnostics;
    private readonly ILogger<RuNkckiConnector> _logger;
    private readonly string _cacheDirectory;

    private readonly HtmlParser _htmlParser = new();

    public RuNkckiConnector(
        SourceFetchService fetchService,
        RawDocumentStorage rawDocumentStorage,
        IDocumentStore documentStore,
        IDtoStore dtoStore,
        IAdvisoryStore advisoryStore,
        ISourceStateRepository stateRepository,
        IOptions<RuNkckiOptions> options,
        RuNkckiDiagnostics diagnostics,
        TimeProvider? timeProvider,
        ILogger<RuNkckiConnector> logger)
    {
        _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
        _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
        _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
        _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
        _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
        _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
        _options.Validate();
        _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
        _timeProvider = timeProvider ?? TimeProvider.System;
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory);
        EnsureCacheDirectory();
    }

    public string SourceName => RuNkckiConnectorPlugin.SourceName;

    public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        var pendingDocuments = cursor.PendingDocuments.ToHashSet();
        var pendingMappings = cursor.PendingMappings.ToHashSet();
        var knownBulletins = cursor.KnownBulletins.ToHashSet(StringComparer.OrdinalIgnoreCase);
        var now = _timeProvider.GetUtcNow();
        var processed = 0;

        if (ShouldUseListingCache(cursor, now))
        {
            _logger.LogDebug(
                "NKCKI listing fetch skipped (cache duration {CacheDuration:c}); processing cached bulletins only",
                _options.ListingCacheDuration);

            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
            await UpdateCursorAsync(cursor
                .WithPendingDocuments(pendingDocuments)
                .WithPendingMappings(pendingMappings)
                .WithKnownBulletins(NormalizeBulletins(knownBulletins))
                .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
            return;
        }

        ListingFetchSummary listingSummary;
        try
        {
            listingSummary = await LoadListingAsync(cancellationToken).ConfigureAwait(false);
        }
        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
        {
            _logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins");
            _diagnostics.ListingFetchFailure(ex.Message);
            await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);

            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
            await UpdateCursorAsync(cursor
                .WithPendingDocuments(pendingDocuments)
                .WithPendingMappings(pendingMappings)
                .WithKnownBulletins(NormalizeBulletins(knownBulletins))
                .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
            return;
        }

        var uniqueAttachments = listingSummary.Attachments
            .GroupBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
            .Select(static group => group.First())
            .OrderBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
            .ToList();

        var newAttachments = uniqueAttachments
            .Where(attachment => !knownBulletins.Contains(attachment.Id))
            .Take(_options.MaxBulletinsPerFetch)
            .ToList();

        _diagnostics.ListingFetchSuccess(listingSummary.PagesVisited, uniqueAttachments.Count, newAttachments.Count);

        if (newAttachments.Count == 0)
        {
            _logger.LogDebug("NKCKI listing contained no new bulletin attachments");
            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
            await UpdateCursorAsync(cursor
                .WithPendingDocuments(pendingDocuments)
                .WithPendingMappings(pendingMappings)
                .WithKnownBulletins(NormalizeBulletins(knownBulletins))
                .WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
            return;
        }

        var downloaded = 0;
        var cachedUsed = 0;
        var failures = 0;

        foreach (var attachment in newAttachments)
        {
            cancellationToken.ThrowIfCancellationRequested();

            try
            {
                var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, attachment.Uri)
                {
                    AcceptHeaders = BulletinAcceptHeaders,
                    TimeoutOverride = _options.RequestTimeout,
                };

                var attachmentResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
                if (!attachmentResult.IsSuccess || attachmentResult.Content is null)
                {
                    if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
                    {
                        _diagnostics.BulletinFetchCached();
                        cachedUsed++;
                        _logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode);
                        processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                        knownBulletins.Add(attachment.Id);
                    }
                    else
                    {
                        _diagnostics.BulletinFetchFailure(attachmentResult.StatusCode.ToString());
                        failures++;
                        _logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode);
                    }

                    continue;
                }

                _diagnostics.BulletinFetchSuccess();
                downloaded++;
                TryWriteCachedBulletin(attachment.Id, attachmentResult.Content);
                processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                knownBulletins.Add(attachment.Id);
            }
            catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
            {
                if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
                {
                    _diagnostics.BulletinFetchCached();
                    cachedUsed++;
                    _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id);
                    processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                    knownBulletins.Add(attachment.Id);
                }
                else
                {
                    _diagnostics.BulletinFetchFailure(ex.Message);
                    failures++;
                    _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id);
                    await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
                    throw;
                }
            }

            if (processed >= _options.MaxVulnerabilitiesPerFetch)
            {
                break;
            }

            if (_options.RequestDelay > TimeSpan.Zero)
            {
                try
                {
                    await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
                }
                catch (TaskCanceledException)
                {
                    break;
                }
            }
        }

        if (processed < _options.MaxVulnerabilitiesPerFetch)
        {
            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
        }

        var normalizedBulletins = NormalizeBulletins(knownBulletins);

        var updatedCursor = cursor
            .WithPendingDocuments(pendingDocuments)
            .WithPendingMappings(pendingMappings)
            .WithKnownBulletins(normalizedBulletins)
            .WithLastListingFetch(now);

        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);

        _logger.LogInformation(
            "NKCKI fetch complete: new bulletins {Downloaded}, cached bulletins {Cached}, failures {Failures}, processed entries {Processed}, pending documents {PendingDocuments}, pending mappings {PendingMappings}",
            downloaded,
            cachedUsed,
            failures,
            processed,
            pendingDocuments.Count,
            pendingMappings.Count);
    }

    public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        if (cursor.PendingDocuments.Count == 0)
        {
            return;
        }

        var pendingDocuments = cursor.PendingDocuments.ToList();
        var pendingMappings = cursor.PendingMappings.ToList();

        foreach (var documentId in cursor.PendingDocuments)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
            if (document is null)
            {
                pendingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                continue;
            }

            if (!document.GridFsId.HasValue)
            {
                _logger.LogWarning("NKCKI document {DocumentId} missing GridFS payload", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                continue;
            }

            byte[] payload;
            try
            {
                payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "NKCKI unable to download raw document {DocumentId}", documentId);
                throw;
            }

            RuNkckiVulnerabilityDto? dto;
            try
            {
                dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(payload, SerializerOptions);
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "NKCKI failed to deserialize document {DocumentId}", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                continue;
            }

            if (dto is null)
            {
                _logger.LogWarning("NKCKI document {DocumentId} produced null DTO", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                continue;
            }

            var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
            var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow());
            await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
            await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);

            pendingDocuments.Remove(documentId);
            if (!pendingMappings.Contains(documentId))
            {
                pendingMappings.Add(documentId);
            }
        }

        var updatedCursor = cursor
            .WithPendingDocuments(pendingDocuments)
            .WithPendingMappings(pendingMappings);

        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
    }

    public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        if (cursor.PendingMappings.Count == 0)
        {
            return;
        }

        var pendingMappings = cursor.PendingMappings.ToList();

        foreach (var documentId in cursor.PendingMappings)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
            if (document is null)
            {
                pendingMappings.Remove(documentId);
                continue;
            }

            var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
            if (dtoRecord is null)
            {
                _logger.LogWarning("NKCKI document {DocumentId} missing DTO payload", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingMappings.Remove(documentId);
                continue;
            }

            RuNkckiVulnerabilityDto dto;
            try
            {
                dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "NKCKI failed to deserialize DTO for document {DocumentId}", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingMappings.Remove(documentId);
                continue;
            }

            try
            {
                var advisory = RuNkckiMapper.Map(dto, document, dtoRecord.ValidatedAt);
                await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
                pendingMappings.Remove(documentId);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "NKCKI mapping failed for document {DocumentId}", documentId);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                pendingMappings.Remove(documentId);
            }
        }

        var updatedCursor = cursor.WithPendingMappings(pendingMappings);
        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
    }

    private async Task<int> ProcessCachedBulletinsAsync(
        HashSet<Guid> pendingDocuments,
        HashSet<Guid> pendingMappings,
        HashSet<string> knownBulletins,
        DateTimeOffset now,
        int processed,
        CancellationToken cancellationToken)
    {
        if (!Directory.Exists(_cacheDirectory))
        {
            return processed;
        }

        var updated = processed;
        var cacheFiles = Directory
            .EnumerateFiles(_cacheDirectory, "*.json.zip", SearchOption.TopDirectoryOnly)
            .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
            .ToList();

        foreach (var filePath in cacheFiles)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var bulletinId = ExtractBulletinIdFromCachePath(filePath);
            if (string.IsNullOrWhiteSpace(bulletinId) || knownBulletins.Contains(bulletinId))
            {
                continue;
            }

            byte[] content;
            try
            {
                content = File.ReadAllBytes(filePath);
            }
            catch (Exception ex)
            {
                _logger.LogDebug(ex, "NKCKI failed to read cached bulletin at {CachePath}", filePath);
                continue;
            }

            _diagnostics.BulletinFetchCached();
            updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
            knownBulletins.Add(bulletinId);

            if (updated >= _options.MaxVulnerabilitiesPerFetch)
            {
                break;
            }
        }

        return updated;
    }

    private async Task<int> ProcessBulletinEntriesAsync(
        byte[] content,
        string bulletinId,
        HashSet<Guid> pendingDocuments,
        HashSet<Guid> pendingMappings,
        DateTimeOffset now,
        int processed,
        CancellationToken cancellationToken)
    {
        if (content.Length == 0)
        {
            return processed;
        }

        var updated = processed;
        using var archiveStream = new MemoryStream(content, writable: false);
        using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false);

        foreach (var entry in archive.Entries.OrderBy(static e => e.FullName, StringComparer.OrdinalIgnoreCase))
        {
            cancellationToken.ThrowIfCancellationRequested();

            if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
            {
                continue;
            }

            using var entryStream = entry.Open();
            using var buffer = new MemoryStream();
            await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);

            if (buffer.Length == 0)
            {
                continue;
            }

            buffer.Position = 0;

            using var document = await JsonDocument.ParseAsync(buffer, cancellationToken: cancellationToken).ConfigureAwait(false);
            updated = await ProcessBulletinJsonElementAsync(document.RootElement, entry.FullName, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);

            if (updated >= _options.MaxVulnerabilitiesPerFetch)
            {
                break;
            }
        }

        var delta = updated - processed;
        if (delta > 0)
        {
            _diagnostics.EntriesProcessed(delta);
        }

        return updated;
    }

    private async Task<int> ProcessBulletinJsonElementAsync(
        JsonElement element,
        string entryName,
        string bulletinId,
        HashSet<Guid> pendingDocuments,
        HashSet<Guid> pendingMappings,
        DateTimeOffset now,
        int processed,
        CancellationToken cancellationToken)
    {
        var updated = processed;

        switch (element.ValueKind)
        {
            case JsonValueKind.Array:
                foreach (var child in element.EnumerateArray())
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    if (updated >= _options.MaxVulnerabilitiesPerFetch)
                    {
                        break;
                    }

                    if (child.ValueKind != JsonValueKind.Object)
                    {
                        continue;
                    }

                    if (await ProcessVulnerabilityObjectAsync(child, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
                    {
                        updated++;
                    }
                }

                break;

            case JsonValueKind.Object:
                if (await ProcessVulnerabilityObjectAsync(element, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
                {
                    updated++;
                }

                break;
        }

        return updated;
    }

    private async Task<bool> ProcessVulnerabilityObjectAsync(
        JsonElement element,
        string entryName,
        string bulletinId,
        HashSet<Guid> pendingDocuments,
        HashSet<Guid> pendingMappings,
        DateTimeOffset now,
        CancellationToken cancellationToken)
    {
        RuNkckiVulnerabilityDto dto;
        try
        {
            dto = RuNkckiJsonParser.Parse(element);
        }
        catch (Exception ex)
        {
            _logger.LogDebug(ex, "NKCKI failed to parse vulnerability in bulletin {BulletinId} entry {Entry}", bulletinId, entryName);
            return false;
        }

        var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions);
        var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
        var documentUri = BuildDocumentUri(dto);

        var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
        if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase))
        {
            return false;
        }

        var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false);

        var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
        {
            ["ru-nkcki.bulletin"] = bulletinId,
            ["ru-nkcki.entry"] = entryName,
        };

        if (!string.IsNullOrWhiteSpace(dto.FstecId))
        {
            metadata["ru-nkcki.fstec_id"] = dto.FstecId!;
        }

        if (!string.IsNullOrWhiteSpace(dto.MitreId))
        {
            metadata["ru-nkcki.mitre_id"] = dto.MitreId!;
        }

        var recordId = existing?.Id ?? Guid.NewGuid();
        var lastModified = dto.DateUpdated ?? dto.DatePublished;
        var record = new DocumentRecord(
            recordId,
            SourceName,
            documentUri,
            now,
            sha,
            DocumentStatuses.PendingParse,
            "application/json",
            Headers: null,
            Metadata: metadata,
            Etag: null,
            LastModified: lastModified,
            GridFsId: gridFsId,
            ExpiresAt: null);

        var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
        pendingDocuments.Add(upserted.Id);
        pendingMappings.Remove(upserted.Id);
        return true;
    }

    private Task<SourceFetchContentResult> FetchListingPageAsync(Uri pageUri, CancellationToken cancellationToken)
    {
        var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, pageUri)
        {
            AcceptHeaders = ListingAcceptHeaders,
            TimeoutOverride = _options.RequestTimeout,
        };

        return _fetchService.FetchContentAsync(request, cancellationToken);
    }

    private async Task<ListingPageResult> ParseListingAsync(Uri pageUri, byte[] content, CancellationToken cancellationToken)
    {
        var html = Encoding.UTF8.GetString(content);
        var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false);
        var attachments = new List<BulletinAttachment>();
        var pagination = new List<Uri>();

        foreach (var anchor in document.QuerySelectorAll("a[href$='.json.zip']"))
        {
            var href = anchor.GetAttribute("href");
            if (string.IsNullOrWhiteSpace(href))
            {
                continue;
            }

            if (!Uri.TryCreate(pageUri, href, out var absoluteUri))
            {
                continue;
            }

            var id = DeriveBulletinId(absoluteUri);
            if (string.IsNullOrWhiteSpace(id))
            {
                continue;
            }

            var title = anchor.GetAttribute("title");
            if (string.IsNullOrWhiteSpace(title))
            {
                title = anchor.TextContent?.Trim();
            }

            attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id));
        }

        foreach (var anchor in document.QuerySelectorAll("a[href]"))
        {
            var href = anchor.GetAttribute("href");
            if (string.IsNullOrWhiteSpace(href))
            {
                continue;
            }

            if (!href.Contains("PAGEN", StringComparison.OrdinalIgnoreCase)
                && !href.Contains("page=", StringComparison.OrdinalIgnoreCase))
            {
                continue;
            }

            if (Uri.TryCreate(pageUri, href, out var absoluteUri))
            {
                pagination.Add(absoluteUri);
            }
        }

        var uniquePagination = pagination
            .DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase)
            .Take(_options.MaxListingPagesPerFetch)
            .ToList();

        return new ListingPageResult(attachments, uniquePagination);
    }

    private static string DeriveBulletinId(Uri uri)
    {
        var fileName = Path.GetFileName(uri.AbsolutePath);
        if (string.IsNullOrWhiteSpace(fileName))
        {
            return Guid.NewGuid().ToString("N");
        }

        if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
        {
            fileName = fileName[..^4];
        }

        if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
        {
            fileName = fileName[..^5];
        }

        return fileName.Replace('_', '-');
    }

    private static string BuildDocumentUri(RuNkckiVulnerabilityDto dto)
    {
        if (!string.IsNullOrWhiteSpace(dto.FstecId))
        {
            var slug = dto.FstecId.Contains(':', StringComparison.Ordinal)
                ? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
                : dto.FstecId;
            return $"https://cert.gov.ru/materialy/uyazvimosti/{slug}";
        }

        if (!string.IsNullOrWhiteSpace(dto.MitreId))
        {
            return $"https://nvd.nist.gov/vuln/detail/{dto.MitreId}";
        }

        return $"https://cert.gov.ru/materialy/uyazvimosti/{Guid.NewGuid():N}";
    }

    private string ResolveCacheDirectory(string? configuredPath)
    {
        if (!string.IsNullOrWhiteSpace(configuredPath))
        {
            return Path.GetFullPath(Path.IsPathRooted(configuredPath)
                ? configuredPath
                : Path.Combine(AppContext.BaseDirectory, configuredPath));
        }

        return Path.Combine(AppContext.BaseDirectory, "cache", RuNkckiConnectorPlugin.SourceName);
    }

    private void EnsureCacheDirectory()
    {
        try
        {
            Directory.CreateDirectory(_cacheDirectory);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex, "NKCKI unable to ensure cache directory {CachePath}", _cacheDirectory);
        }
    }

    private string GetBulletinCachePath(string bulletinId)
    {
        var fileStem = string.IsNullOrWhiteSpace(bulletinId)
            ? Guid.NewGuid().ToString("N")
            : Uri.EscapeDataString(bulletinId);
        return Path.Combine(_cacheDirectory, $"{fileStem}.json.zip");
    }

    private static string ExtractBulletinIdFromCachePath(string path)
    {
        if (string.IsNullOrWhiteSpace(path))
        {
            return string.Empty;
        }

        var fileName = Path.GetFileName(path);
        if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
        {
            fileName = fileName[..^4];
        }

        if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
        {
            fileName = fileName[..^5];
        }

        return Uri.UnescapeDataString(fileName);
    }

    private void TryWriteCachedBulletin(string bulletinId, byte[] content)
    {
        try
        {
            var cachePath = GetBulletinCachePath(bulletinId);
            Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!);
            File.WriteAllBytes(cachePath, content);
        }
        catch (Exception ex)
        {
            _logger.LogDebug(ex, "NKCKI failed to cache bulletin {BulletinId}", bulletinId);
        }
    }

    private bool TryReadCachedBulletin(string bulletinId, out byte[] content)
    {
        var cachePath = GetBulletinCachePath(bulletinId);
        try
        {
            if (File.Exists(cachePath))
            {
                content = File.ReadAllBytes(cachePath);
                return true;
            }
        }
        catch (Exception ex)
        {
            _logger.LogDebug(ex, "NKCKI failed to read cached bulletin {BulletinId}", bulletinId);
        }

        content = Array.Empty<byte>();
        return false;
    }

    private IReadOnlyCollection<string> NormalizeBulletins(IEnumerable<string> bulletins)
    {
        var normalized = (bulletins ?? Enumerable.Empty<string>())
            .Where(static id => !string.IsNullOrWhiteSpace(id))
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
            .ToList();

        if (normalized.Count <= _options.KnownBulletinCapacity)
        {
            return normalized.ToArray();
        }

        var skip = normalized.Count - _options.KnownBulletinCapacity;
        return normalized.Skip(skip).ToArray();
    }

    private async Task<RuNkckiCursor> GetCursorAsync(CancellationToken cancellationToken)
    {
        var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
        return state is null ? RuNkckiCursor.Empty : RuNkckiCursor.FromBson(state.Cursor);
    }

    private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken)
    {
        var document = cursor.ToBsonDocument();
        var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow();
        return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
    }

    private readonly record struct ListingFetchSummary(IReadOnlyList<BulletinAttachment> Attachments, int PagesVisited);

    private readonly record struct ListingPageResult(IReadOnlyList<BulletinAttachment> Attachments, IReadOnlyList<Uri> PaginationLinks);

    private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title);

    private bool ShouldUseListingCache(RuNkckiCursor cursor, DateTimeOffset now)
    {
        if (!cursor.LastListingFetchAt.HasValue)
        {
            return false;
        }

        var age = now - cursor.LastListingFetchAt.Value;
        return age < _options.ListingCacheDuration;
    }

    private async Task<ListingFetchSummary> LoadListingAsync(CancellationToken cancellationToken)
    {
        var attachments = new List<BulletinAttachment>();
        var visited = 0;
        var visitedUris = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        var queue = new Queue<Uri>();
        queue.Enqueue(_options.ListingUri);

        while (queue.Count > 0 && visited < _options.MaxListingPagesPerFetch)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var pageUri = queue.Dequeue();
            if (!visitedUris.Add(pageUri.AbsoluteUri))
            {
                continue;
            }

            _diagnostics.ListingFetchAttempt();

            var listingResult = await FetchListingPageAsync(pageUri, cancellationToken).ConfigureAwait(false);
            if (!listingResult.IsSuccess || listingResult.Content is null)
            {
                _diagnostics.ListingFetchFailure(listingResult.StatusCode.ToString());
                _logger.LogWarning("NKCKI listing page {ListingUri} returned no content (status={Status})", pageUri, listingResult.StatusCode);
                continue;
            }

            visited++;

            var page = await ParseListingAsync(pageUri, listingResult.Content, cancellationToken).ConfigureAwait(false);
            attachments.AddRange(page.Attachments);

            foreach (var link in page.PaginationLinks)
            {
                if (!visitedUris.Contains(link.AbsoluteUri) && queue.Count + visitedUris.Count < _options.MaxListingPagesPerFetch)
                {
                    queue.Enqueue(link);
                }
            }

            if (attachments.Count >= _options.MaxBulletinsPerFetch * 2)
            {
                break;
            }
        }

        return new ListingFetchSummary(attachments, visited);
    }
}
