using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using StellaOps.Feedser.Models;
using StellaOps.Feedser.Source.Common;
using StellaOps.Feedser.Source.Common.Fetch;
using StellaOps.Feedser.Source.Common.Html;
using StellaOps.Feedser.Source.Ics.Cisa.Configuration;
using StellaOps.Feedser.Source.Ics.Cisa.Internal;
using StellaOps.Feedser.Storage.Mongo;
using StellaOps.Feedser.Storage.Mongo.Advisories;
using StellaOps.Feedser.Storage.Mongo.Documents;
using StellaOps.Feedser.Storage.Mongo.Dtos;
using StellaOps.Plugin;

namespace StellaOps.Feedser.Source.Ics.Cisa;

public sealed class IcsCisaConnector : IFeedConnector
{
    private const string SchemaVersion = "ics.cisa.feed.v1";

    private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" };
    private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" };
    private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" };

    private readonly SourceFetchService _fetchService;
    private readonly RawDocumentStorage _rawDocumentStorage;
    private readonly IDocumentStore _documentStore;
    private readonly IDtoStore _dtoStore;
    private readonly IAdvisoryStore _advisoryStore;
    private readonly ISourceStateRepository _stateRepository;
    private readonly IcsCisaOptions _options;
    private readonly IcsCisaFeedParser _parser;
    private readonly IcsCisaDiagnostics _diagnostics;
    private readonly TimeProvider _timeProvider;
    private readonly ILogger<IcsCisaConnector> _logger;
    private readonly HtmlContentSanitizer _htmlSanitizer = new();
    private readonly HtmlParser _htmlParser = new();

    public IcsCisaConnector(
        SourceFetchService fetchService,
        RawDocumentStorage rawDocumentStorage,
        IDocumentStore documentStore,
        IDtoStore dtoStore,
        IAdvisoryStore advisoryStore,
        ISourceStateRepository stateRepository,
        IOptions<IcsCisaOptions> options,
        IcsCisaFeedParser parser,
        IcsCisaDiagnostics diagnostics,
        TimeProvider? timeProvider,
        ILogger<IcsCisaConnector> logger)
    {
        _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
        _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
        _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
        _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
        _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
        _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
        _options = options?.Value ?? throw new ArgumentNullException(nameof(options));
        _options.Validate();
        _parser = parser ?? throw new ArgumentNullException(nameof(parser));
        _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
        _timeProvider = timeProvider ?? TimeProvider.System;
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public string SourceName => IcsCisaConnectorPlugin.SourceName;

    public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        var pendingDocuments = cursor.PendingDocuments.ToHashSet();
        var pendingMappings = cursor.PendingMappings.ToHashSet();
        var now = _timeProvider.GetUtcNow();
        var touched = false;

        foreach (var topic in _options.TopicIds)
        {
            cancellationToken.ThrowIfCancellationRequested();

            _diagnostics.FetchAttempt(topic);
            var topicUri = _options.BuildTopicUri(topic);
            var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, topicUri.ToString(), cancellationToken).ConfigureAwait(false);

            var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, topicUri)
            {
                AcceptHeaders = RssAcceptHeaders,
                Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
                {
                    ["icscisa.topicId"] = topic,
                },
            };

            if (existing is not null)
            {
                request = request with
                {
                    ETag = existing.Etag,
                    LastModified = existing.LastModified,
                };
            }

            SourceFetchResult? result = null;
            var documentsAdded = 0;
            var usedFallback = false;

            try
            {
                result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
            }
            catch (HttpRequestException ex) when (ShouldRetryWithFallback(ex))
            {
                _logger.LogWarning(ex, "Retrying CISA ICS topic {TopicId} via Akamai fallback", topic);
                _diagnostics.FetchFallback(topic);
                usedFallback = true;
                var fallbackRequest = request with
                {
                    AcceptHeaders = RssFallbackAcceptHeaders,
                    Metadata = AppendMetadata(request.Metadata, "icscisa.retry", "akamai"),
                };

                try
                {
                    result = await _fetchService.FetchAsync(fallbackRequest, cancellationToken).ConfigureAwait(false);
                }
                catch (Exception fallbackEx) when (fallbackEx is HttpRequestException or TaskCanceledException)
                {
                    _diagnostics.FetchFailure(topic);
                    _logger.LogError(fallbackEx, "Fallback fetch failed for CISA ICS topic {TopicId}", topic);
                    await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, fallbackEx.Message, cancellationToken).ConfigureAwait(false);
                    throw;
                }
            }
            catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
            {
                _diagnostics.FetchFailure(topic);
                _logger.LogError(ex, "Failed to fetch CISA ICS topic {TopicId}", topic);
                await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
                throw;
            }

            if (result is null)
            {
                _diagnostics.FetchFailure(topic);
                continue;
            }

            if (result.IsNotModified)
            {
                _diagnostics.FetchNotModified(topic);
                _logger.LogDebug("CISA ICS topic {TopicId} not modified", topic);
            }
            else if (result.IsSuccess && result.Document is not null)
            {
                pendingDocuments.Add(result.Document.Id);
                pendingMappings.Remove(result.Document.Id);
                touched = true;
                documentsAdded++;
                _diagnostics.FetchSuccess(topic, 1);
                _logger.LogInformation("Fetched CISA ICS topic {TopicId} document {DocumentId}", topic, result.Document.Id);
            }
            else if (result.IsSuccess)
            {
                _diagnostics.FetchSuccess(topic, 0);
                _logger.LogDebug("CISA ICS topic {TopicId} fetch succeeded without new document (fallback={Fallback})", topic, usedFallback);
            }
            else
            {
                _diagnostics.FetchFailure(topic);
                _logger.LogWarning("CISA ICS topic {TopicId} returned status {StatusCode}", topic, result.StatusCode);
            }

            if (documentsAdded > 0)
            {
                _logger.LogInformation("CISA ICS topic {TopicId} added {DocumentsAdded} document(s) (fallbackUsed={Fallback})", topic, documentsAdded, usedFallback);
            }

            if (_options.RequestDelay > TimeSpan.Zero)
            {
                await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
            }
        }

        if (!touched)
        {
            await UpdateCursorAsync(cursor.WithPendingDocuments(pendingDocuments).WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false);
            return;
        }

        var updatedCursor = cursor
            .WithPendingDocuments(pendingDocuments)
            .WithPendingMappings(pendingMappings);

        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
    }

    public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        if (cursor.PendingDocuments.Count == 0)
        {
            return;
        }

        var remainingDocuments = cursor.PendingDocuments.ToList();
        var pendingMappings = cursor.PendingMappings.ToHashSet();
        DateTimeOffset? latestPublished = cursor.LastPublished;

        foreach (var documentId in cursor.PendingDocuments)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var topicId = "unknown";
            var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
            if (document is null)
            {
                remainingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                _diagnostics.ParseFailure(topicId);
                continue;
            }

            if (document.Metadata is not null && document.Metadata.TryGetValue("icscisa.topicId", out var topicValue))
            {
                topicId = topicValue;
            }

            if (!document.GridFsId.HasValue)
            {
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                remainingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                _diagnostics.ParseFailure(topicId);
                continue;
            }

            byte[] bytes;
            try
            {
                bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to download CISA ICS payload {DocumentId}", document.Id);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                remainingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                _diagnostics.ParseFailure(topicId);
                continue;
            }

            IReadOnlyCollection<IcsCisaAdvisoryDto> advisories;
            try
            {
                using var stream = new MemoryStream(bytes, writable: false);
                var topicUri = Uri.TryCreate(document.Uri, UriKind.Absolute, out var parsed) ? parsed : null;
                advisories = _parser.Parse(stream, string.Equals(topicId, "USDHSCISA_19", StringComparison.OrdinalIgnoreCase), parsed);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to parse CISA ICS feed {DocumentId}", document.Id);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                remainingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                _diagnostics.ParseFailure(topicId);
                continue;
            }

            var advisoryList = advisories.ToList();
            var detailAttempts = 0;
            if (_options.EnableDetailScrape)
            {
                var enriched = new List<IcsCisaAdvisoryDto>(advisoryList.Count);
                foreach (var advisory in advisoryList)
                {
                    if (NeedsDetailFetch(advisory))
                    {
                        detailAttempts++;
                    }
                    var enrichedAdvisory = await EnrichAdvisoryAsync(advisory, cancellationToken).ConfigureAwait(false);
                    enriched.Add(enrichedAdvisory);
                }

                advisoryList = enriched;
            }

            var attachmentTotal = advisoryList.Sum(static advisory => advisory.Attachments is null ? 0 : advisory.Attachments.Count);

            var feedDto = new IcsCisaFeedDto
            {
                TopicId = topicId,
                FeedUri = document.Uri,
                Advisories = advisoryList,
            };

            try
            {
                var json = JsonSerializer.Serialize(feedDto, new JsonSerializerOptions(JsonSerializerDefaults.Web)
                {
                    DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
                    WriteIndented = false,
                });
                var bson = BsonDocument.Parse(json);
                var dtoRecord = new DtoRecord(
                    Guid.NewGuid(),
                    document.Id,
                    SourceName,
                    SchemaVersion,
                    bson,
                    _timeProvider.GetUtcNow());

                await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);

                remainingDocuments.Remove(documentId);
                pendingMappings.Add(document.Id);

                var docPublished = advisoryList.Count > 0 ? advisoryList.Max(a => a.Published) : (DateTimeOffset?)null;
                if (docPublished.HasValue && docPublished > latestPublished)
                {
                    latestPublished = docPublished;
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to persist CISA ICS DTO {DocumentId}", document.Id);
                await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                remainingDocuments.Remove(documentId);
                pendingMappings.Remove(documentId);
                _diagnostics.ParseFailure(topicId);
                continue;
            }

            _diagnostics.ParseSuccess(topicId, advisoryList.Count, attachmentTotal, detailAttempts);
            _logger.LogInformation(
                "CISA ICS parse produced advisories={Advisories} attachments={Attachments} detailAttempts={DetailAttempts} topic={TopicId}",
                advisoryList.Count,
                attachmentTotal,
                detailAttempts,
                topicId);
        }

        var updatedCursor = cursor
            .WithPendingDocuments(remainingDocuments)
            .WithPendingMappings(pendingMappings)
            .WithLastPublished(latestPublished);

        await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
    }

    public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(services);

        var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
        if (cursor.PendingMappings.Count == 0)
        {
            return;
        }

        var pendingMappings = cursor.PendingMappings.ToHashSet();

        foreach (var documentId in cursor.PendingMappings)
        {
            cancellationToken.ThrowIfCancellationRequested();

            var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
            if (dtoRecord is null)
            {
                pendingMappings.Remove(documentId);
                _diagnostics.MapFailure("unknown");
                continue;
            }

            IcsCisaFeedDto? feedDto;
            try
            {
                var json = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
                feedDto = JsonSerializer.Deserialize<IcsCisaFeedDto>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web));
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to deserialize CISA ICS DTO {DtoId}", dtoRecord.Id);
                pendingMappings.Remove(documentId);
                await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                _diagnostics.MapFailure("unknown");
                continue;
            }

            if (feedDto is null)
            {
                pendingMappings.Remove(documentId);
                await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                _diagnostics.MapFailure("unknown");
                continue;
            }

            var allMapped = true;
            var mappedCount = 0;
            foreach (var advisoryDto in feedDto.Advisories)
            {
                try
                {
                    var advisory = MapAdvisory(dtoRecord, feedDto, advisoryDto);
                    await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
                    _diagnostics.MapSuccess(
                        advisoryDto.AdvisoryId,
                        advisory.References.Length,
                        advisory.AffectedPackages.Length,
                        advisory.Aliases.Length);
                    mappedCount++;
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Failed to map CISA ICS advisory {AdvisoryId}", advisoryDto.AdvisoryId);
                    _diagnostics.MapFailure(advisoryDto.AdvisoryId);
                    allMapped = false;
                }
            }

            pendingMappings.Remove(documentId);

            if (!allMapped)
            {
                _logger.LogWarning(
                    "CISA ICS mapping failed for document {DocumentId} (mapped={MappedCount} of {Total})",
                    documentId,
                    mappedCount,
                    feedDto.Advisories.Count);
                await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
                continue;
            }

            await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
            _logger.LogInformation("CISA ICS mapped {MappedCount} advisories from document {DocumentId}", mappedCount, documentId);
        }

        await UpdateCursorAsync(cursor.WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false);
    }

    private Advisory MapAdvisory(DtoRecord dtoRecord, IcsCisaFeedDto feedDto, IcsCisaAdvisoryDto advisoryDto)
    {
        var recordedAt = dtoRecord.ValidatedAt;
        var fetchProvenance = new AdvisoryProvenance(SourceName, "feed", feedDto.FeedUri, recordedAt);
        var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryId, _timeProvider.GetUtcNow());

        var aliases = CombineAliases(advisoryDto);
        var references = BuildReferences(advisoryDto, recordedAt).ToList();
        var mitigationReferences = BuildMitigationReferences(advisoryDto, recordedAt);
        if (mitigationReferences.Count > 0)
        {
            references.AddRange(mitigationReferences);
        }

        var affectedPackages = BuildAffectedPackages(advisoryDto, recordedAt);

        return new Advisory(
            advisoryDto.AdvisoryId,
            advisoryDto.Title,
            advisoryDto.Summary,
            language: "en",
            published: advisoryDto.Published,
            modified: advisoryDto.Updated ?? advisoryDto.Published,
            severity: null,
            exploitKnown: false,
            aliases: aliases,
            references: references,
            affectedPackages: affectedPackages,
            cvssMetrics: Array.Empty<CvssMetric>(),
            provenance: new[] { fetchProvenance, mappingProvenance });
    }

    internal static IReadOnlyCollection<string> CombineAliases(IcsCisaAdvisoryDto advisoryDto)
    {
        var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        if (advisoryDto.Aliases is not null)
        {
            foreach (var alias in advisoryDto.Aliases)
            {
                if (string.IsNullOrWhiteSpace(alias))
                {
                    continue;
                }

                set.Add(alias.Trim());
            }
        }

        if (advisoryDto.CveIds is not null)
        {
            foreach (var cve in advisoryDto.CveIds)
            {
                if (string.IsNullOrWhiteSpace(cve))
                {
                    continue;
                }

                set.Add(cve.Trim());
            }
        }

        return set.Count == 0
            ? Array.Empty<string>()
            : set.OrderBy(static value => value, StringComparer.Ordinal).ToArray();
    }

    internal static IReadOnlyCollection<AdvisoryReference> BuildMitigationReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt)
    {
        if (advisoryDto.Mitigations is null || advisoryDto.Mitigations.Count == 0)
        {
            return Array.Empty<AdvisoryReference>();
        }

        var references = new List<AdvisoryReference>();
        var baseUrl = Validation.LooksLikeHttpUrl(advisoryDto.Link) ? advisoryDto.Link : null;
        var sourceTag = advisoryDto.IsMedical ? "icscisa-medical-mitigation" : "icscisa-mitigation";

        var index = 0;
        foreach (var mitigation in advisoryDto.Mitigations)
        {
            index++;
            if (string.IsNullOrWhiteSpace(mitigation))
            {
                continue;
            }

            var summary = mitigation.Trim();
            var url = baseUrl is not null
                ? $"{baseUrl}#mitigation-{index}"
                : $"icscisa:mitigation:{advisoryDto.AdvisoryId}:{index}";

            references.Add(new AdvisoryReference(
                url,
                kind: "mitigation",
                sourceTag: sourceTag,
                summary: summary,
                provenance: new AdvisoryProvenance("ics-cisa", "mitigation", url, recordedAt)));
        }

        return references.Count == 0 ? Array.Empty<AdvisoryReference>() : references;
    }

    internal static IReadOnlyCollection<AdvisoryReference> BuildReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt)
    {
        var references = new List<AdvisoryReference>();
        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        if (advisoryDto.Attachments is { Count: > 0 })
        {
            foreach (var attachment in advisoryDto.Attachments)
            {
                if (attachment is null || !Validation.LooksLikeHttpUrl(attachment.Url))
                {
                    continue;
                }

                var url = attachment.Url;
                if (!seen.Add(url))
                {
                    continue;
                }

                try
                {
                    references.Add(new AdvisoryReference(
                        url,
                        kind: "attachment",
                        sourceTag: advisoryDto.IsMedical ? "icscisa-medical-attachment" : "icscisa-attachment",
                        summary: attachment.Title,
                        provenance: new AdvisoryProvenance("ics-cisa", "attachment", url, recordedAt)));
                }
                catch (ArgumentException)
                {
                    // ignore invalid URIs
                }
            }
        }

        foreach (var reference in advisoryDto.References ?? Array.Empty<string>())
        {
            if (!Validation.LooksLikeHttpUrl(reference))
            {
                continue;
            }

            if (!seen.Add(reference))
            {
                continue;
            }

            try
            {
                references.Add(new AdvisoryReference(
                    reference,
                    kind: "advisory",
                    sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa",
                    summary: null,
                    provenance: new AdvisoryProvenance("ics-cisa", "reference", reference, recordedAt)));
            }
            catch (ArgumentException)
            {
                // ignore invalid URIs
            }
        }

        if (references.Count == 0 && Validation.LooksLikeHttpUrl(advisoryDto.Link) && seen.Add(advisoryDto.Link))
        {
            references.Add(new AdvisoryReference(
                advisoryDto.Link,
                kind: "advisory",
                sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa",
                summary: null,
                provenance: new AdvisoryProvenance("ics-cisa", "reference", advisoryDto.Link, recordedAt)));
        }

        return references;
    }

    internal static IReadOnlyCollection<AffectedPackage> BuildAffectedPackages(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt)
    {
        var packages = new List<AffectedPackage>();
        var vendors = advisoryDto.Vendors ?? Array.Empty<string>();
        var normalizedVendors = vendors
            .Where(static vendor => !string.IsNullOrWhiteSpace(vendor))
            .Select(static vendor => vendor.Trim())
            .Distinct(StringComparer.OrdinalIgnoreCase)
            .ToArray();

        var parsedProducts = (advisoryDto.Products ?? Array.Empty<string>())
            .Where(static product => !string.IsNullOrWhiteSpace(product))
            .Select(ParseProductInfo)
            .Where(static product => !string.IsNullOrWhiteSpace(product.Name))
            .ToArray();

        if (parsedProducts.Length > 0)
        {
            foreach (var product in parsedProducts)
            {
            }

            foreach (var product in parsedProducts)
            {
                var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt);
                var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
                {
                    ["ics.product"] = product.Name!
                };

                if (!string.IsNullOrWhiteSpace(product.VersionExpression))
                {
                    vendorExtensions["ics.version"] = product.VersionExpression!;
                }

                if (normalizedVendors.Length > 0)
                {
                    vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors);
                }

                var semVer = TryCreateSemVerPrimitive(product.VersionExpression);
                var range = new AffectedVersionRange(
                    rangeKind: "product",
                    introducedVersion: null,
                    fixedVersion: null,
                    lastAffectedVersion: null,
                    rangeExpression: product.VersionExpression,
                    provenance: provenance,
                    primitives: new RangePrimitives(semVer, null, null, vendorExtensions));

                packages.Add(new AffectedPackage(
                    AffectedPackageTypes.IcsVendor,
                    product.Name!,
                    platform: null,
                    versionRanges: new[] { range },
                    statuses: Array.Empty<AffectedPackageStatus>(),
                    provenance: new[] { provenance }));
            }

            return packages;
        }

        if (normalizedVendors.Length == 0)
        {
            return packages;
        }

        foreach (var vendor in normalizedVendors)
        {
            var provenance = new AdvisoryProvenance("ics-cisa", "affected", vendor, recordedAt);
            var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
            {
                ["ics.vendor"] = vendor
            };

            var range = new AffectedVersionRange(
                rangeKind: "vendor",
                introducedVersion: null,
                fixedVersion: null,
                lastAffectedVersion: null,
                rangeExpression: null,
                provenance: provenance,
                primitives: new RangePrimitives(null, null, null, vendorExtensions));

            packages.Add(new AffectedPackage(
                AffectedPackageTypes.IcsVendor,
                vendor,
                platform: null,
                versionRanges: new[] { range },
                statuses: Array.Empty<AffectedPackageStatus>(),
                provenance: new[] { provenance }));
        }

        return packages;
    }


    private static ProductInfo ParseProductInfo(string raw)
    {
        var trimmed = raw?.Trim();
        if (string.IsNullOrWhiteSpace(trimmed))
        {
            return new ProductInfo(null, null);
        }

        if (trimmed.Contains(':', StringComparison.Ordinal))
        {
            var parts = trimmed.Split(':', 2);
            var name = parts[0].Trim();
            var versionSegment = parts[1].Trim();
            return new ProductInfo(
                string.IsNullOrWhiteSpace(name) ? trimmed : name,
                string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment);
        }

        var lastSpace = trimmed.LastIndexOf(' ');
        if (lastSpace > 0)
        {
            var candidateVersion = trimmed[(lastSpace + 1)..].Trim();
            if (Regex.IsMatch(candidateVersion, "^[vV]?[0-9].*"))
            {
                var name = trimmed[..lastSpace].Trim();
                return new ProductInfo(
                    string.IsNullOrWhiteSpace(name) ? trimmed : name,
                    candidateVersion);
            }
        }

        return new ProductInfo(trimmed, null);
    }

    private static SemVerPrimitive? TryCreateSemVerPrimitive(string? versionExpression)
    {
        if (string.IsNullOrWhiteSpace(versionExpression))
        {
            return null;
        }

        var normalized = NormalizeSemVer(versionExpression);
        if (normalized is null)
        {
            var trimmed = versionExpression.Trim();
            if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
            {
                trimmed = trimmed[1..];
            }

            if (Version.TryParse(trimmed, out var parsed))
            {
                normalized = string.Join('.', new[]
                {
                    parsed.Major.ToString(CultureInfo.InvariantCulture),
                    parsed.Minor >= 0 ? parsed.Minor.ToString(CultureInfo.InvariantCulture) : "0",
                    parsed.Build >= 0 ? parsed.Build.ToString(CultureInfo.InvariantCulture) : "0",
                });
            }
        }

        if (normalized is null)
        {
            return null;
        }

        return new SemVerPrimitive(
            null,
            true,
            null,
            true,
            null,
            true,
            null,
            normalized);
    }

    private static string? NormalizeSemVer(string rawVersion)
    {
        var trimmed = rawVersion.Trim();
        if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase))
        {
            trimmed = trimmed[1..];
        }

        if (!Regex.IsMatch(trimmed, @"^[0-9]+(\.[0-9]+){0,2}$"))
        {
            return null;
        }

        var parts = trimmed.Split('.', StringSplitOptions.RemoveEmptyEntries);
        var components = parts.Take(3).ToList();
        while (components.Count < 3)
        {
            components.Add("0");
        }

        return string.Join('.', components);
    }

    private sealed record ProductInfo(string? Name, string? VersionExpression);

    private async Task<IcsCisaAdvisoryDto> EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken)
    {
        if (!NeedsDetailFetch(advisory))
        {
            return advisory;
        }

        if (!Uri.TryCreate(advisory.Link, UriKind.Absolute, out var detailUri))
        {
            return advisory;
        }

        var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, detailUri)
        {
            AcceptHeaders = DetailAcceptHeaders,
            Metadata = AppendMetadata(null, "icscisa.detail", advisory.AdvisoryId),
            TimeoutOverride = _options.DetailRequestTimeout,
        };

        try
        {
            var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
            if (!result.IsSuccess || result.Content is null)
            {
                _diagnostics.DetailFetchFailure(advisory.AdvisoryId);
                return advisory;
            }

            var html = Encoding.UTF8.GetString(result.Content);
            var sanitized = _htmlSanitizer.Sanitize(html, detailUri);
            if (string.IsNullOrWhiteSpace(sanitized))
            {
                _diagnostics.DetailFetchSuccess(advisory.AdvisoryId);
                return advisory with { DetailHtml = sanitized };
            }

            var detailAttachments = _options.CaptureAttachments
                ? ParseAttachmentsFromHtml(sanitized, detailUri)
                : Array.Empty<IcsCisaAttachmentDto>();
            var mergedAttachments = _options.CaptureAttachments
                ? MergeAttachments(advisory.Attachments, detailAttachments)
                : advisory.Attachments;

            var detailMitigations = ParseMitigationsFromHtml(sanitized);
            var mergedMitigations = MergeMitigations(advisory.Mitigations, detailMitigations);

            var detailReferences = ParseReferencesFromHtml(sanitized, detailUri);
            var mergedReferences = MergeReferences(advisory.References, detailReferences);

            var summary = string.IsNullOrWhiteSpace(advisory.Summary)
                ? ExtractFirstSentence(sanitized)
                : advisory.Summary;

            var descriptionHtml = string.IsNullOrWhiteSpace(advisory.DescriptionHtml)
                ? sanitized
                : advisory.DescriptionHtml;

            return advisory with
            {
                DetailHtml = sanitized,
                DescriptionHtml = descriptionHtml,
                Summary = summary,
                References = mergedReferences,
                Attachments = mergedAttachments,
                Mitigations = mergedMitigations,
            };
        }
        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
        {
            _logger.LogWarning(ex, "Failed to fetch detail page for {AdvisoryId}", advisory.AdvisoryId);
            _diagnostics.DetailFetchFailure(advisory.AdvisoryId);
            return advisory;
        }
    }

    private bool NeedsDetailFetch(IcsCisaAdvisoryDto advisory)
    {
        if (!_options.EnableDetailScrape)
        {
            return false;
        }

        if (string.IsNullOrWhiteSpace(advisory.DescriptionHtml))
        {
            return true;
        }

        if (string.IsNullOrWhiteSpace(advisory.Summary))
        {
            return true;
        }

        if (advisory.Mitigations is null || advisory.Mitigations.Count == 0)
        {
            return true;
        }

        if (_options.CaptureAttachments && (advisory.Attachments is null || advisory.Attachments.Count == 0))
        {
            return true;
        }

        return false;
    }

    private IReadOnlyCollection<string> ParseMitigationsFromHtml(string sanitizedHtml)
    {
        if (string.IsNullOrWhiteSpace(sanitizedHtml))
        {
            return Array.Empty<string>();
        }

        try
        {
            var document = _htmlParser.ParseDocument(sanitizedHtml);
            var mitigations = new List<string>();

            foreach (var heading in document.QuerySelectorAll("h1, h2, h3, h4, h5, h6"))
            {
                var headingText = heading.TextContent?.Trim();
                if (!IsMitigationHeading(headingText))
                {
                    continue;
                }

                var node = heading.NextElementSibling;
                while (node is not null && node is not IHtmlHeadingElement)
                {
                    if (node is IHtmlParagraphElement or IHtmlDivElement)
                    {
                        var content = Validation.CollapseWhitespace(node.TextContent);
                        if (!string.IsNullOrWhiteSpace(content))
                        {
                            mitigations.Add(content);
                        }
                    }
                    else if (node is IHtmlElement element && (string.Equals(element.TagName, "UL", StringComparison.OrdinalIgnoreCase) || string.Equals(element.TagName, "OL", StringComparison.OrdinalIgnoreCase)))
                    {
                        foreach (var item in element.Children)
                        {
                            var content = Validation.CollapseWhitespace(item.TextContent);
                            if (!string.IsNullOrWhiteSpace(content))
                            {
                                mitigations.Add(content);
                            }
                        }
                    }

                    node = node.NextElementSibling;
                }
            }

            return mitigations.Count == 0 ? Array.Empty<string>() : mitigations;
        }
        catch
        {
            return Array.Empty<string>();
        }
    }

    private static bool IsMitigationHeading(string? headingText)
    {
        if (string.IsNullOrWhiteSpace(headingText))
        {
            return false;
        }

        return headingText.Contains("mitigation", StringComparison.OrdinalIgnoreCase);
    }

    private IReadOnlyCollection<IcsCisaAttachmentDto> ParseAttachmentsFromHtml(string sanitizedHtml, Uri baseUri)
    {
        if (string.IsNullOrWhiteSpace(sanitizedHtml))
        {
            return Array.Empty<IcsCisaAttachmentDto>();
        }

        try
        {
            var document = _htmlParser.ParseDocument(sanitizedHtml);
            var attachments = new Dictionary<string, IcsCisaAttachmentDto>(StringComparer.OrdinalIgnoreCase);

            foreach (var anchor in document.QuerySelectorAll("a"))
            {
                var href = anchor.GetAttribute("href");
                if (string.IsNullOrWhiteSpace(href))
                {
                    continue;
                }

                if (!Uri.TryCreate(baseUri, href, out var resolved))
                {
                    continue;
                }

                var url = resolved.ToString();
                if (!url.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase) &&
                    !url.Contains("/pdf", StringComparison.OrdinalIgnoreCase))
                {
                    continue;
                }

                attachments[url] = new IcsCisaAttachmentDto
                {
                    Title = anchor.TextContent?.Trim(),
                    Url = url,
                };
            }

            return attachments.Count == 0
                ? Array.Empty<IcsCisaAttachmentDto>()
                : attachments.Values.ToArray();
        }
        catch
        {
            return Array.Empty<IcsCisaAttachmentDto>();
        }
    }

    private IReadOnlyCollection<string> ParseReferencesFromHtml(string sanitizedHtml, Uri baseUri)
    {
        if (string.IsNullOrWhiteSpace(sanitizedHtml))
        {
            return Array.Empty<string>();
        }

        try
        {
            var document = _htmlParser.ParseDocument(sanitizedHtml);
            var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

            foreach (var anchor in document.QuerySelectorAll("a"))
            {
                var href = anchor.GetAttribute("href");
                if (string.IsNullOrWhiteSpace(href))
                {
                    continue;
                }

                if (Uri.TryCreate(baseUri, href, out var resolved) && Validation.LooksLikeHttpUrl(resolved.ToString()))
                {
                    links.Add(resolved.ToString());
                }
            }

            return links.Count == 0 ? Array.Empty<string>() : links.ToArray();
        }
        catch
        {
            return Array.Empty<string>();
        }
    }

    internal static IReadOnlyCollection<string> MergeMitigations(IReadOnlyCollection<string>? existing, IReadOnlyCollection<string> incoming)
    {
        if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0))
        {
            return Array.Empty<string>();
        }

        var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        var merged = new List<string>();

        if (existing is not null)
        {
            foreach (var mitigation in existing)
            {
                var value = mitigation?.Trim();
                if (string.IsNullOrWhiteSpace(value))
                {
                    continue;
                }

                if (set.Add(value))
                {
                    merged.Add(value);
                }
            }
        }

        if (incoming is not null)
        {
            foreach (var mitigation in incoming)
            {
                var value = mitigation?.Trim();
                if (string.IsNullOrWhiteSpace(value))
                {
                    continue;
                }

                if (set.Add(value))
                {
                    merged.Add(value);
                }
            }
        }

        return merged.Count == 0 ? Array.Empty<string>() : merged;
    }

    internal static IReadOnlyCollection<IcsCisaAttachmentDto> MergeAttachments(IReadOnlyCollection<IcsCisaAttachmentDto>? existing, IReadOnlyCollection<IcsCisaAttachmentDto> incoming)
    {
        if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0))
        {
            return Array.Empty<IcsCisaAttachmentDto>();
        }

        var map = new Dictionary<string, IcsCisaAttachmentDto>(StringComparer.OrdinalIgnoreCase);

        if (existing is not null)
        {
            foreach (var attachment in existing)
            {
                if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url))
                {
                    continue;
                }

                map[attachment.Url] = attachment;
            }
        }

        if (incoming is not null)
        {
            foreach (var attachment in incoming)
            {
                if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url))
                {
                    continue;
                }

                if (!map.ContainsKey(attachment.Url) || string.IsNullOrWhiteSpace(map[attachment.Url].Title))
                {
                    map[attachment.Url] = attachment;
                }
            }
        }

        return map.Count == 0 ? Array.Empty<IcsCisaAttachmentDto>() : map.Values.ToArray();
    }

    internal static IReadOnlyCollection<string> MergeReferences(IReadOnlyCollection<string>? existing, IReadOnlyCollection<string> incoming)
    {
        var links = new HashSet<string>(existing ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
        foreach (var link in incoming)
        {
            if (Validation.LooksLikeHttpUrl(link))
            {
                links.Add(link);
            }
        }

        return links.Count == 0 ? Array.Empty<string>() : links.ToArray();
    }

    internal static string? ExtractFirstSentence(string sanitizedHtml)
    {
        if (string.IsNullOrWhiteSpace(sanitizedHtml))
        {
            return null;
        }

        var text = Validation.CollapseWhitespace(sanitizedHtml);
        if (text.Length <= 280)
        {
            return text;
        }

        var terminator = text.IndexOf('.', StringComparison.Ordinal);
        if (terminator <= 0 || terminator > 280)
        {
            return text[..Math.Min(280, text.Length)].Trim();
        }

        return text[..(terminator + 1)].Trim();
    }

    internal static IReadOnlyDictionary<string, string> AppendMetadata(IReadOnlyDictionary<string, string>? metadata, string key, string value)
    {
        var dictionary = new Dictionary<string, string>(StringComparer.Ordinal);

        if (metadata is not null)
        {
            foreach (var pair in metadata)
            {
                dictionary[pair.Key] = pair.Value;
            }
        }

        dictionary[key] = value;
        return dictionary;
    }

    internal static bool ShouldRetryWithFallback(HttpRequestException exception)
    {
        var message = exception.Message ?? string.Empty;
        return message.Contains(" 403", StringComparison.OrdinalIgnoreCase)
            || message.Contains("403", StringComparison.OrdinalIgnoreCase)
            || message.Contains("406", StringComparison.OrdinalIgnoreCase);
    }

    private async Task<IcsCisaCursor> GetCursorAsync(CancellationToken cancellationToken)
    {
        var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
        return state is null ? IcsCisaCursor.Empty : IcsCisaCursor.FromBson(state.Cursor);
    }

    private Task UpdateCursorAsync(IcsCisaCursor cursor, CancellationToken cancellationToken)
        => _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken);
}
