up

src/StellaOps.Feedser.Source.Ru.Nkcki/Configuration/RuNkckiOptions.cs

@@ -38,6 +38,11 @@ public sealed class RuNkckiOptions
     /// </summary>
     public int MaxBulletinsPerFetch { get; set; } = 5;
 
+    /// <summary>
+    /// Maximum number of listing pages visited per fetch cycle.
+    /// </summary>
+    public int MaxListingPagesPerFetch { get; set; } = 3;
+
     /// <summary>
     /// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
     /// </summary>
@@ -99,6 +104,11 @@ public sealed class RuNkckiOptions
             throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
         }
 
+        if (MaxListingPagesPerFetch <= 0)
+        {
+            throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
+        }
+
         if (MaxVulnerabilitiesPerFetch <= 0)
         {
             throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");

src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs

@@ -0,0 +1,115 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Metrics;
+
+namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
+
+/// <summary>
+/// Emits telemetry counters for the NKCKI connector lifecycle.
+/// </summary>
+public sealed class RuNkckiDiagnostics : IDisposable
+{
+    private const string MeterName = "StellaOps.Feedser.Source.Ru.Nkcki";
+    private const string MeterVersion = "1.0.0";
+
+    private readonly Meter _meter;
+    private readonly Counter<long> _listingFetchAttempts;
+    private readonly Counter<long> _listingFetchSuccess;
+    private readonly Counter<long> _listingFetchFailures;
+    private readonly Histogram<long> _listingPagesVisited;
+    private readonly Histogram<long> _listingAttachmentsDiscovered;
+    private readonly Histogram<long> _listingAttachmentsNew;
+    private readonly Counter<long> _bulletinFetchSuccess;
+    private readonly Counter<long> _bulletinFetchCached;
+    private readonly Counter<long> _bulletinFetchFailures;
+    private readonly Histogram<long> _entriesProcessed;
+
+    public RuNkckiDiagnostics()
+    {
+        _meter = new Meter(MeterName, MeterVersion);
+        _listingFetchAttempts = _meter.CreateCounter<long>(
+            "nkcki.listing.fetch.attempts",
+            unit: "operations",
+            description: "Number of listing fetch attempts.");
+        _listingFetchSuccess = _meter.CreateCounter<long>(
+            "nkcki.listing.fetch.success",
+            unit: "operations",
+            description: "Number of successful listing fetches.");
+        _listingFetchFailures = _meter.CreateCounter<long>(
+            "nkcki.listing.fetch.failures",
+            unit: "operations",
+            description: "Number of listing fetch failures.");
+        _listingPagesVisited = _meter.CreateHistogram<long>(
+            "nkcki.listing.pages.visited",
+            unit: "pages",
+            description: "Listing pages visited per fetch cycle.");
+        _listingAttachmentsDiscovered = _meter.CreateHistogram<long>(
+            "nkcki.listing.attachments.discovered",
+            unit: "attachments",
+            description: "Attachments discovered across listing pages.");
+        _listingAttachmentsNew = _meter.CreateHistogram<long>(
+            "nkcki.listing.attachments.new",
+            unit: "attachments",
+            description: "New bulletin attachments enqueued per fetch cycle.");
+        _bulletinFetchSuccess = _meter.CreateCounter<long>(
+            "nkcki.bulletin.fetch.success",
+            unit: "operations",
+            description: "Number of bulletin downloads that succeeded.");
+        _bulletinFetchCached = _meter.CreateCounter<long>(
+            "nkcki.bulletin.fetch.cached",
+            unit: "operations",
+            description: "Number of bulletins served from cache.");
+        _bulletinFetchFailures = _meter.CreateCounter<long>(
+            "nkcki.bulletin.fetch.failures",
+            unit: "operations",
+            description: "Number of bulletin download failures.");
+        _entriesProcessed = _meter.CreateHistogram<long>(
+            "nkcki.entries.processed",
+            unit: "entries",
+            description: "Number of vulnerability entries processed per bulletin.");
+    }
+
+    public void ListingFetchAttempt() => _listingFetchAttempts.Add(1);
+
+    public void ListingFetchSuccess(int pagesVisited, int attachmentsDiscovered, int attachmentsNew)
+    {
+        _listingFetchSuccess.Add(1);
+        if (pagesVisited >= 0)
+        {
+            _listingPagesVisited.Record(pagesVisited);
+        }
+
+        if (attachmentsDiscovered >= 0)
+        {
+            _listingAttachmentsDiscovered.Record(attachmentsDiscovered);
+        }
+
+        if (attachmentsNew >= 0)
+        {
+            _listingAttachmentsNew.Record(attachmentsNew);
+        }
+    }
+
+    public void ListingFetchFailure(string reason)
+        => _listingFetchFailures.Add(1, ReasonTag(reason));
+
+    public void BulletinFetchSuccess() => _bulletinFetchSuccess.Add(1);
+
+    public void BulletinFetchCached() => _bulletinFetchCached.Add(1);
+
+    public void BulletinFetchFailure(string reason)
+        => _bulletinFetchFailures.Add(1, ReasonTag(reason));
+
+    public void EntriesProcessed(int count)
+    {
+        if (count >= 0)
+        {
+            _entriesProcessed.Record(count);
+        }
+    }
+
+    private static KeyValuePair<string, object?> ReasonTag(string reason)
+        => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
+
+    public void Dispose() => _meter.Dispose();
+}

src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiJsonParser.cs

@@ -1,16 +1,47 @@
+using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
-using System.Linq;
 using System.Globalization;
+using System.Linq;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 
 namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
 
 internal static class RuNkckiJsonParser
 {
+    private static readonly Regex ComparatorRegex = new(
+        @"^(?<name>.+?)\s*(?<operator><=|>=|<|>|==|=)\s*(?<version>.+?)$",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    private static readonly Regex RangeRegex = new(
+        @"^(?<name>.+?)\s+(?<start>[\p{L}\p{N}\._-]+)\s*[-–]\s*(?<end>[\p{L}\p{N}\._-]+)$",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    private static readonly Regex QualifierRegex = new(
+        @"^(?<name>.+?)\s+(?<version>[\p{L}\p{N}\._-]+)\s+(?<qualifier>(and\s+earlier|and\s+later|and\s+newer|до\s+и\s+включительно|и\s+ниже|и\s+выше|и\s+старше|и\s+позже))$",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
+
+    private static readonly Regex QualifierInlineRegex = new(
+        @"верс(ии|ия)\s+(?<version>[\p{L}\p{N}\._-]+)\s+(?<qualifier>и\s+ниже|и\s+выше|и\s+старше)",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
+
+    private static readonly Regex VersionWindowRegex = new(
+        @"верс(ии|ия)\s+(?<start>[\p{L}\p{N}\._-]+)\s+по\s+(?<end>[\p{L}\p{N}\._-]+)",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
+
+    private static readonly char[] SoftwareSplitDelimiters = { '\n', ';', '\u2022', '\u2023', '\r' };
+
+    private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase;
+
     public static RuNkckiVulnerabilityDto Parse(JsonElement element)
     {
-        var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec) ? Normalize(fstec.GetString()) : null;
-        var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre) ? Normalize(mitre.GetString()) : null;
+        var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec)
+            ? Normalize(fstec.GetString())
+            : null;
+        var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre)
+            ? Normalize(mitre.GetString())
+            : null;
 
         var datePublished = ParseDate(element.TryGetProperty("date_published", out var published) ? published.GetString() : null);
         var dateUpdated = ParseDate(element.TryGetProperty("date_updated", out var updated) ? updated.GetString() : null);
@@ -22,12 +53,11 @@ internal static class RuNkckiJsonParser
             _ => null,
         } : null;
 
-        var description = Normalize(element.TryGetProperty("description", out var desc) ? desc.GetString() : null);
-        var mitigation = Normalize(element.TryGetProperty("mitigation", out var mitigationElement) ? mitigationElement.GetString() : null);
-        var productCategory = Normalize(element.TryGetProperty("product_category", out var category) ? category.GetString() : null);
-        var impact = Normalize(element.TryGetProperty("impact", out var impactElement) ? impactElement.GetString() : null);
-        var method = Normalize(element.TryGetProperty("method_of_exploitation", out var methodElement) ? methodElement.GetString() : null);
-
+        var description = ReadJoinedString(element, "description");
+        var mitigation = ReadJoinedString(element, "mitigation");
+        var productCategories = ReadStringCollection(element, "product_category");
+        var impact = ReadJoinedString(element, "impact");
+        var method = ReadJoinedString(element, "method_of_exploitation");
         bool? userInteraction = element.TryGetProperty("user_interaction", out var uiElement) ? uiElement.ValueKind switch
         {
             JsonValueKind.True => true,
@@ -35,25 +65,7 @@ internal static class RuNkckiJsonParser
             _ => null,
         } : null;
 
-        string? softwareText = null;
-        bool? softwareHasCpe = null;
-        if (element.TryGetProperty("vulnerable_software", out var softwareElement))
-        {
-            if (softwareElement.TryGetProperty("software_text", out var textElement))
-            {
-                softwareText = Normalize(textElement.GetString()?.Replace('\r', ' '));
-            }
-
-            if (softwareElement.TryGetProperty("cpe", out var cpeElement))
-            {
-                softwareHasCpe = cpeElement.ValueKind switch
-                {
-                    JsonValueKind.True => true,
-                    JsonValueKind.False => false,
-                    _ => null,
-                };
-            }
-        }
+        var (softwareText, softwareHasCpe, softwareEntries) = ParseVulnerableSoftware(element);
 
         RuNkckiCweDto? cweDto = null;
         if (element.TryGetProperty("cwe", out var cweElement))
@@ -71,7 +83,7 @@ internal static class RuNkckiJsonParser
                 }
             }
 
-            var cweDescription = Normalize(cweElement.TryGetProperty("cwe_description", out var descElement) ? descElement.GetString() : null);
+            var cweDescription = ReadJoinedString(cweElement, "cwe_description") ?? Normalize(cweElement.GetString());
             if (number.HasValue || !string.IsNullOrWhiteSpace(cweDescription))
             {
                 cweDto = new RuNkckiCweDto(number, cweDescription);
@@ -91,13 +103,8 @@ internal static class RuNkckiJsonParser
             ? Normalize(vectorV4Element.GetString())
             : null;
 
-        var urls = element.TryGetProperty("urls", out var urlsElement) && urlsElement.ValueKind == JsonValueKind.Array
-            ? urlsElement.EnumerateArray()
-                .Select(static url => Normalize(url.GetString()))
-                .Where(static url => !string.IsNullOrWhiteSpace(url))
-                .Cast<string>()
-                .ToImmutableArray()
-            : ImmutableArray<string>.Empty;
+        var urls = ReadUrls(element);
+        var tags = ReadStringCollection(element, "tags");
 
         return new RuNkckiVulnerabilityDto(
             fstecId,
@@ -108,10 +115,11 @@ internal static class RuNkckiJsonParser
             patchAvailable,
             description,
             cweDto,
-            productCategory,
+            productCategories,
             mitigation,
             softwareText,
             softwareHasCpe,
+            softwareEntries,
             cvssScore,
             cvssVector,
             cvssScoreV4,
@@ -119,7 +127,466 @@ internal static class RuNkckiJsonParser
             impact,
             method,
             userInteraction,
-            urls);
+            urls,
+            tags);
+    }
+
+    private static ImmutableArray<string> ReadUrls(JsonElement element)
+    {
+        if (!element.TryGetProperty("urls", out var urlsElement))
+        {
+            return ImmutableArray<string>.Empty;
+        }
+
+        var collected = new List<string>();
+        CollectUrls(urlsElement, collected);
+        if (collected.Count == 0)
+        {
+            return ImmutableArray<string>.Empty;
+        }
+
+        return collected
+            .Select(Normalize)
+            .Where(static url => !string.IsNullOrWhiteSpace(url))
+            .Select(static url => url!)
+            .Distinct(OrdinalIgnoreCase)
+            .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase)
+            .ToImmutableArray();
+    }
+
+    private static void CollectUrls(JsonElement element, ICollection<string> results)
+    {
+        switch (element.ValueKind)
+        {
+            case JsonValueKind.String:
+                var value = element.GetString();
+                if (!string.IsNullOrWhiteSpace(value))
+                {
+                    results.Add(value);
+                }
+                break;
+            case JsonValueKind.Array:
+                foreach (var child in element.EnumerateArray())
+                {
+                    CollectUrls(child, results);
+                }
+                break;
+            case JsonValueKind.Object:
+                if (element.TryGetProperty("url", out var urlProperty))
+                {
+                    CollectUrls(urlProperty, results);
+                }
+
+                if (element.TryGetProperty("href", out var hrefProperty))
+                {
+                    CollectUrls(hrefProperty, results);
+                }
+
+                foreach (var property in element.EnumerateObject())
+                {
+                    if (property.NameEquals("value") || property.NameEquals("link"))
+                    {
+                        CollectUrls(property.Value, results);
+                    }
+                }
+                break;
+        }
+    }
+
+    private static string? ReadJoinedString(JsonElement element, string property)
+    {
+        if (!element.TryGetProperty(property, out var target))
+        {
+            return null;
+        }
+
+        var values = ReadStringCollection(target);
+        if (!values.IsDefaultOrEmpty)
+        {
+            return string.Join("; ", values);
+        }
+
+        return Normalize(target.ValueKind == JsonValueKind.String ? target.GetString() : target.ToString());
+    }
+
+    private static ImmutableArray<string> ReadStringCollection(JsonElement element, string property)
+    {
+        if (!element.TryGetProperty(property, out var target))
+        {
+            return ImmutableArray<string>.Empty;
+        }
+
+        return ReadStringCollection(target);
+    }
+
+    private static ImmutableArray<string> ReadStringCollection(JsonElement element)
+    {
+        var builder = ImmutableArray.CreateBuilder<string>();
+        CollectStrings(element, builder);
+        return Deduplicate(builder);
+    }
+
+    private static void CollectStrings(JsonElement element, ImmutableArray<string>.Builder builder)
+    {
+        switch (element.ValueKind)
+        {
+            case JsonValueKind.String:
+                AddIfPresent(builder, Normalize(element.GetString()));
+                break;
+            case JsonValueKind.Number:
+                AddIfPresent(builder, Normalize(element.ToString()));
+                break;
+            case JsonValueKind.True:
+                builder.Add("true");
+                break;
+            case JsonValueKind.False:
+                builder.Add("false");
+                break;
+            case JsonValueKind.Array:
+                foreach (var child in element.EnumerateArray())
+                {
+                    CollectStrings(child, builder);
+                }
+                break;
+            case JsonValueKind.Object:
+                foreach (var property in element.EnumerateObject())
+                {
+                    CollectStrings(property.Value, builder);
+                }
+                break;
+        }
+    }
+
+    private static ImmutableArray<string> Deduplicate(ImmutableArray<string>.Builder builder)
+    {
+        if (builder.Count == 0)
+        {
+            return ImmutableArray<string>.Empty;
+        }
+
+        return builder
+            .Where(static value => !string.IsNullOrWhiteSpace(value))
+            .Distinct(OrdinalIgnoreCase)
+            .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
+            .ToImmutableArray();
+    }
+
+    private static void AddIfPresent(ImmutableArray<string>.Builder builder, string? value)
+    {
+        if (!string.IsNullOrWhiteSpace(value))
+        {
+            builder.Add(value!);
+        }
+    }
+
+    private static (string? Text, bool? HasCpe, ImmutableArray<RuNkckiSoftwareEntry> Entries) ParseVulnerableSoftware(JsonElement element)
+    {
+        if (!element.TryGetProperty("vulnerable_software", out var softwareElement))
+        {
+            return (null, null, ImmutableArray<RuNkckiSoftwareEntry>.Empty);
+        }
+
+        string? softwareText = null;
+        if (softwareElement.TryGetProperty("software_text", out var textElement))
+        {
+            softwareText = Normalize(textElement.ValueKind == JsonValueKind.String ? textElement.GetString() : textElement.ToString());
+        }
+
+        bool? softwareHasCpe = null;
+        if (softwareElement.TryGetProperty("cpe", out var cpeElement))
+        {
+            softwareHasCpe = cpeElement.ValueKind switch
+            {
+                JsonValueKind.True => true,
+                JsonValueKind.False => false,
+                _ => softwareHasCpe,
+            };
+        }
+
+        var entries = new List<RuNkckiSoftwareEntry>();
+        if (softwareElement.TryGetProperty("software", out var softwareNodes))
+        {
+            entries.AddRange(ParseSoftwareEntries(softwareNodes));
+        }
+
+        if (entries.Count == 0 && !string.IsNullOrWhiteSpace(softwareText))
+        {
+            entries.AddRange(SplitSoftwareTextIntoEntries(softwareText));
+        }
+
+        if (entries.Count == 0)
+        {
+            foreach (var fallbackProperty in new[] { "items", "aliases", "software_lines" })
+            {
+                if (softwareElement.TryGetProperty(fallbackProperty, out var fallbackNodes))
+                {
+                    entries.AddRange(ParseSoftwareEntries(fallbackNodes));
+                }
+            }
+        }
+
+        if (entries.Count == 0)
+        {
+            return (softwareText, softwareHasCpe, ImmutableArray<RuNkckiSoftwareEntry>.Empty);
+        }
+
+        var grouped = entries
+            .GroupBy(static entry => entry.Identifier, OrdinalIgnoreCase)
+            .Select(static group =>
+            {
+                var evidence = string.Join(
+                    "; ",
+                    group.Select(static entry => entry.Evidence)
+                        .Where(static evidence => !string.IsNullOrWhiteSpace(evidence))
+                        .Distinct(OrdinalIgnoreCase));
+
+                var ranges = group
+                    .SelectMany(static entry => entry.RangeExpressions)
+                    .Where(static range => !string.IsNullOrWhiteSpace(range))
+                    .Distinct(OrdinalIgnoreCase)
+                    .OrderBy(static range => range, StringComparer.OrdinalIgnoreCase)
+                    .ToImmutableArray();
+
+                return new RuNkckiSoftwareEntry(
+                    group.Key,
+                    string.IsNullOrWhiteSpace(evidence) ? group.Key : evidence,
+                    ranges);
+            })
+            .OrderBy(static entry => entry.Identifier, StringComparer.OrdinalIgnoreCase)
+            .ToImmutableArray();
+
+        return (softwareText, softwareHasCpe, grouped);
+    }
+
+    private static IEnumerable<RuNkckiSoftwareEntry> ParseSoftwareEntries(JsonElement element)
+    {
+        switch (element.ValueKind)
+        {
+            case JsonValueKind.Array:
+                foreach (var child in element.EnumerateArray())
+                {
+                    foreach (var entry in ParseSoftwareEntries(child))
+                    {
+                        yield return entry;
+                    }
+                }
+                break;
+            case JsonValueKind.Object:
+                yield return CreateEntryFromObject(element);
+                break;
+            case JsonValueKind.String:
+                foreach (var entry in SplitSoftwareTextIntoEntries(element.GetString() ?? string.Empty))
+                {
+                    yield return entry;
+                }
+                break;
+        }
+    }
+
+    private static RuNkckiSoftwareEntry CreateEntryFromObject(JsonElement element)
+    {
+        var vendor = ReadFirstString(element, "vendor", "manufacturer", "organisation");
+        var name = ReadFirstString(element, "name", "product", "title");
+        var rawVersion = ReadFirstString(element, "version", "versions", "range");
+        var comment = ReadFirstString(element, "comment", "notes", "summary");
+
+        var identifierParts = new List<string>();
+        if (!string.IsNullOrWhiteSpace(vendor))
+        {
+            identifierParts.Add(vendor!);
+        }
+
+        if (!string.IsNullOrWhiteSpace(name))
+        {
+            identifierParts.Add(name!);
+        }
+
+        var identifier = identifierParts.Count > 0
+            ? string.Join(" ", identifierParts)
+            : ReadFirstString(element, "identifier") ?? name ?? rawVersion ?? comment ?? "unknown";
+
+        var evidenceParts = new List<string>(identifierParts);
+        if (!string.IsNullOrWhiteSpace(rawVersion))
+        {
+            evidenceParts.Add(rawVersion!);
+        }
+
+        if (!string.IsNullOrWhiteSpace(comment))
+        {
+            evidenceParts.Add(comment!);
+        }
+
+        var evidence = string.Join(" ", evidenceParts.Where(static part => !string.IsNullOrWhiteSpace(part))).Trim();
+
+        var rangeHints = new List<string?>();
+        if (!string.IsNullOrWhiteSpace(rawVersion))
+        {
+            rangeHints.Add(rawVersion);
+        }
+
+        if (element.TryGetProperty("range", out var rangeElement))
+        {
+            rangeHints.Add(Normalize(rangeElement.ToString()));
+        }
+
+        return CreateSoftwareEntry(identifier!, evidence, rangeHints);
+    }
+
+    private static IEnumerable<RuNkckiSoftwareEntry> SplitSoftwareTextIntoEntries(string text)
+    {
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            yield break;
+        }
+
+        var segments = text.Split(SoftwareSplitDelimiters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        if (segments.Length == 0)
+        {
+            segments = new[] { text };
+        }
+
+        foreach (var segment in segments)
+        {
+            var normalized = Normalize(segment);
+            if (string.IsNullOrWhiteSpace(normalized))
+            {
+                continue;
+            }
+
+            var (identifier, hints) = ExtractIdentifierAndRangeHints(normalized!);
+            yield return CreateSoftwareEntry(identifier, normalized!, hints);
+        }
+    }
+
+    private static RuNkckiSoftwareEntry CreateSoftwareEntry(string identifier, string evidence, IEnumerable<string?> hints)
+    {
+        var normalizedIdentifier = Normalize(identifier) ?? "unknown";
+        var normalizedEvidence = Normalize(evidence) ?? normalizedIdentifier;
+
+        var ranges = hints
+            .Select(NormalizeRangeHint)
+            .Where(static hint => !string.IsNullOrWhiteSpace(hint))
+            .Select(static hint => hint!)
+            .Distinct(OrdinalIgnoreCase)
+            .OrderBy(static hint => hint, StringComparer.OrdinalIgnoreCase)
+            .ToImmutableArray();
+
+        return new RuNkckiSoftwareEntry(normalizedIdentifier, normalizedEvidence!, ranges);
+    }
+
+    private static string? NormalizeRangeHint(string? hint)
+    {
+        if (string.IsNullOrWhiteSpace(hint))
+        {
+            return null;
+        }
+
+        var normalized = Normalize(hint)?
+            .Replace("≤", "<=", StringComparison.Ordinal)
+            .Replace("≥", ">=", StringComparison.Ordinal)
+            .Replace("=>", ">=", StringComparison.Ordinal)
+            .Replace("=<", "<=", StringComparison.Ordinal);
+
+        if (string.IsNullOrWhiteSpace(normalized))
+        {
+            return null;
+        }
+
+        return normalized;
+    }
+
+    private static (string Identifier, IReadOnlyList<string?> RangeHints) ExtractIdentifierAndRangeHints(string value)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return ("unknown", Array.Empty<string>());
+        }
+
+        var comparatorMatch = ComparatorRegex.Match(value);
+        if (comparatorMatch.Success)
+        {
+            var name = Normalize(comparatorMatch.Groups["name"].Value);
+            var version = Normalize(comparatorMatch.Groups["version"].Value);
+            var op = comparatorMatch.Groups["operator"].Value;
+            return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $"{op} {version}" });
+        }
+
+        var rangeMatch = RangeRegex.Match(value);
+        if (rangeMatch.Success)
+        {
+            var name = Normalize(rangeMatch.Groups["name"].Value);
+            var start = Normalize(rangeMatch.Groups["start"].Value);
+            var end = Normalize(rangeMatch.Groups["end"].Value);
+            return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" });
+        }
+
+        var qualifierMatch = QualifierRegex.Match(value);
+        if (qualifierMatch.Success)
+        {
+            var name = Normalize(qualifierMatch.Groups["name"].Value);
+            var version = Normalize(qualifierMatch.Groups["version"].Value);
+            var qualifier = qualifierMatch.Groups["qualifier"].Value.ToLowerInvariant();
+            var hint = qualifier.Contains("ниж") || qualifier.Contains("earlier") || qualifier.Contains("включ")
+                ? $"<= {version}"
+                : $">= {version}";
+            return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint });
+        }
+
+        var inlineQualifierMatch = QualifierInlineRegex.Match(value);
+        if (inlineQualifierMatch.Success)
+        {
+            var version = Normalize(inlineQualifierMatch.Groups["version"].Value);
+            var qualifier = inlineQualifierMatch.Groups["qualifier"].Value.ToLowerInvariant();
+            var hint = qualifier.Contains("ниж") ? $"<= {version}" : $">= {version}";
+            var name = Normalize(QualifierInlineRegex.Replace(value, string.Empty));
+            return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint });
+        }
+
+        var windowMatch = VersionWindowRegex.Match(value);
+        if (windowMatch.Success)
+        {
+            var start = Normalize(windowMatch.Groups["start"].Value);
+            var end = Normalize(windowMatch.Groups["end"].Value);
+            var name = Normalize(VersionWindowRegex.Replace(value, string.Empty));
+            return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" });
+        }
+
+        return (value, Array.Empty<string>());
+    }
+
+    private static string? ReadFirstString(JsonElement element, params string[] names)
+    {
+        foreach (var name in names)
+        {
+            if (element.TryGetProperty(name, out var property))
+            {
+                switch (property.ValueKind)
+                {
+                    case JsonValueKind.String:
+                        {
+                            var normalized = Normalize(property.GetString());
+                            if (!string.IsNullOrWhiteSpace(normalized))
+                            {
+                                return normalized;
+                            }
+
+                            break;
+                        }
+                    case JsonValueKind.Number:
+                        {
+                            var normalized = Normalize(property.ToString());
+                            if (!string.IsNullOrWhiteSpace(normalized))
+                            {
+                                return normalized;
+                            }
+
+                            break;
+                        }
+                }
+            }
+        }
+
+        return null;
     }
 
     private static double? ParseDouble(JsonElement element)
@@ -164,6 +631,16 @@ internal static class RuNkckiJsonParser
             return null;
         }
 
-        return value.Replace('\r', ' ').Replace('\n', ' ').Trim();
+        var normalized = value
+            .Replace('\r', ' ')
+            .Replace('\n', ' ')
+            .Trim();
+
+        while (normalized.Contains("  ", StringComparison.Ordinal))
+        {
+            normalized = normalized.Replace("  ", " ", StringComparison.Ordinal);
+        }
+
+        return normalized.Length == 0 ? null : normalized;
     }
 }

src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs

@@ -4,6 +4,7 @@ using System.Globalization;
 using System.Linq;
 using StellaOps.Feedser.Models;
 using StellaOps.Feedser.Normalization.Cvss;
+using StellaOps.Feedser.Normalization.SemVer;
 using StellaOps.Feedser.Storage.Mongo.Documents;
 
 namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
@@ -80,56 +81,56 @@ internal static class RuNkckiMapper
 
     private static IReadOnlyList<AdvisoryReference> BuildReferences(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
     {
-        var references = new List<AdvisoryReference>
+        var references = new List<AdvisoryReference>();
+        var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        void AddReference(string? url, string kind, string? sourceTag, string? summary)
         {
-            new(document.Uri, "details", "ru-nkcki", summary: null, new AdvisoryProvenance(
+            if (string.IsNullOrWhiteSpace(url))
+            {
+                return;
+            }
+
+            var key = $"{kind}|{url}";
+            if (!seen.Add(key))
+            {
+                return;
+            }
+
+            var provenance = new AdvisoryProvenance(
                 RuNkckiConnectorPlugin.SourceName,
                 "reference",
-                document.Uri,
+                url,
                 recordedAt,
-                new[] { ProvenanceFieldMasks.References }))
-        };
+                new[] { ProvenanceFieldMasks.References });
+
+            references.Add(new AdvisoryReference(url, kind, sourceTag, summary, provenance));
+        }
+
+        AddReference(document.Uri, "details", "ru-nkcki", null);
 
         if (!string.IsNullOrWhiteSpace(dto.FstecId))
         {
             var slug = dto.FstecId!.Contains(':', StringComparison.Ordinal)
                 ? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
                 : dto.FstecId;
-            var bduUrl = $"https://bdu.fstec.ru/vul/{slug}";
-            references.Add(new AdvisoryReference(bduUrl, "details", "bdu", summary: null, new AdvisoryProvenance(
-                RuNkckiConnectorPlugin.SourceName,
-                "reference",
-                bduUrl,
-                recordedAt,
-                new[] { ProvenanceFieldMasks.References })));
+            AddReference($"https://bdu.fstec.ru/vul/{slug}", "details", "bdu", null);
         }
 
         foreach (var url in dto.Urls)
         {
-            if (string.IsNullOrWhiteSpace(url))
-            {
-                continue;
-            }
-
             var kind = url.Contains("cert.gov.ru", StringComparison.OrdinalIgnoreCase) ? "details" : "external";
             var sourceTag = url.Contains("siemens", StringComparison.OrdinalIgnoreCase) ? "vendor" : null;
-            references.Add(new AdvisoryReference(url, kind, sourceTag, summary: null, new AdvisoryProvenance(
-                RuNkckiConnectorPlugin.SourceName,
-                "reference",
-                url,
-                recordedAt,
-                new[] { ProvenanceFieldMasks.References })));
+            AddReference(url, kind, sourceTag, null);
         }
 
         if (dto.Cwe?.Number is int number)
         {
-            var url = $"https://cwe.mitre.org/data/definitions/{number}.html";
-            references.Add(new AdvisoryReference(url, "cwe", "cwe", dto.Cwe.Description, new AdvisoryProvenance(
-                RuNkckiConnectorPlugin.SourceName,
-                "reference",
-                url,
-                recordedAt,
-                new[] { ProvenanceFieldMasks.References })));
+            AddReference(
+                $"https://cwe.mitre.org/data/definitions/{number}.html",
+                "cwe",
+                "cwe",
+                dto.Cwe.Description);
         }
 
         return references;
@@ -137,43 +138,68 @@ internal static class RuNkckiMapper
 
     private static IReadOnlyList<AffectedPackage> BuildPackages(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt)
     {
-        if (string.IsNullOrWhiteSpace(dto.VulnerableSoftwareText))
+        if (!dto.VulnerableSoftwareEntries.IsDefaultOrEmpty && dto.VulnerableSoftwareEntries.Length > 0)
         {
-            return Array.Empty<AffectedPackage>();
+            return CreatePackages(dto.VulnerableSoftwareEntries, dto, recordedAt);
         }
 
-        var identifier = dto.VulnerableSoftwareText!.Replace('\n', ' ').Replace('\r', ' ').Trim();
-        if (identifier.Length == 0)
+        if (!string.IsNullOrWhiteSpace(dto.VulnerableSoftwareText))
         {
-            return Array.Empty<AffectedPackage>();
+            var fallbackEntry = new RuNkckiSoftwareEntry(
+                dto.VulnerableSoftwareText!,
+                dto.VulnerableSoftwareText!,
+                ImmutableArray<string>.Empty);
+            return CreatePackages(new[] { fallbackEntry }, dto, recordedAt);
         }
 
-        var packageProvenance = new AdvisoryProvenance(
-            RuNkckiConnectorPlugin.SourceName,
-            "package",
-            identifier,
-            recordedAt,
-            new[] { ProvenanceFieldMasks.AffectedPackages });
+        return Array.Empty<AffectedPackage>();
+    }
 
-        var status = new AffectedPackageStatus(
-            dto.PatchAvailable == true ? AffectedPackageStatusCatalog.Fixed : AffectedPackageStatusCatalog.Affected,
-            new AdvisoryProvenance(
+    private static IReadOnlyList<AffectedPackage> CreatePackages(IEnumerable<RuNkckiSoftwareEntry> entries, RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt)
+    {
+        var type = DeterminePackageType(dto);
+        var platform = dto.ProductCategories.IsDefaultOrEmpty || dto.ProductCategories.Length == 0
+            ? null
+            : string.Join(", ", dto.ProductCategories);
+
+        var packages = new List<AffectedPackage>();
+
+        foreach (var entry in entries)
+        {
+            if (string.IsNullOrWhiteSpace(entry.Identifier))
+            {
+                continue;
+            }
+
+            var packageProvenance = new AdvisoryProvenance(
                 RuNkckiConnectorPlugin.SourceName,
-                "package-status",
-                dto.PatchAvailable == true ? "patch_available" : "affected",
+                "package",
+                entry.Evidence,
                 recordedAt,
-                new[] { ProvenanceFieldMasks.PackageStatuses }));
+                new[] { ProvenanceFieldMasks.AffectedPackages });
 
-        return new[]
-        {
-            new AffectedPackage(
-                dto.VulnerableSoftwareHasCpe == true ? AffectedPackageTypes.Cpe : AffectedPackageTypes.Vendor,
-                identifier,
-                platform: null,
-                versionRanges: null,
-                statuses: new[] { status },
-                provenance: new[] { packageProvenance })
-        };
+            var status = new AffectedPackageStatus(
+                dto.PatchAvailable == true ? AffectedPackageStatusCatalog.Fixed : AffectedPackageStatusCatalog.Affected,
+                new AdvisoryProvenance(
+                    RuNkckiConnectorPlugin.SourceName,
+                    "package-status",
+                    dto.PatchAvailable == true ? "patch_available" : "affected",
+                    recordedAt,
+                    new[] { ProvenanceFieldMasks.PackageStatuses }));
+
+            var rangeMetadata = BuildRangeMetadata(entry, recordedAt);
+
+            packages.Add(new AffectedPackage(
+                type,
+                entry.Identifier,
+                platform,
+                rangeMetadata.Ranges,
+                new[] { status },
+                new[] { packageProvenance },
+                rangeMetadata.Normalized));
+        }
+
+        return packages;
     }
 
     private static IReadOnlyList<CvssMetric> BuildCvssMetrics(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity)
@@ -194,6 +220,27 @@ internal static class RuNkckiMapper
             severity ??= metric.BaseSeverity;
         }
 
+        if (!string.IsNullOrWhiteSpace(dto.CvssVectorV4))
+        {
+            var vector = dto.CvssVectorV4.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)
+                ? dto.CvssVectorV4
+                : $"CVSS:4.0/{dto.CvssVectorV4}";
+            var score = dto.CvssScoreV4.HasValue
+                ? Math.Round(dto.CvssScoreV4.Value, 1, MidpointRounding.AwayFromZero)
+                : 0.0;
+            var severityV4 = DetermineCvss4Severity(score);
+
+            var provenance = new AdvisoryProvenance(
+                RuNkckiConnectorPlugin.SourceName,
+                "cvss",
+                vector,
+                recordedAt,
+                new[] { ProvenanceFieldMasks.CvssMetrics });
+
+            metrics.Add(new CvssMetric("4.0", vector, score, severityV4, provenance));
+            severity ??= severityV4;
+        }
+
         return metrics;
     }
 
@@ -295,4 +342,104 @@ internal static class RuNkckiMapper
 
         return false;
     }
+
+    private static string DeterminePackageType(RuNkckiVulnerabilityDto dto)
+    {
+        if (dto.VulnerableSoftwareHasCpe == true)
+        {
+            return AffectedPackageTypes.Cpe;
+        }
+
+        if (!dto.ProductCategories.IsDefault && dto.ProductCategories.Any(static category =>
+                category.Contains("ics", StringComparison.OrdinalIgnoreCase)
+                || category.Contains("scada", StringComparison.OrdinalIgnoreCase)))
+        {
+            return AffectedPackageTypes.IcsVendor;
+        }
+
+        return AffectedPackageTypes.Vendor;
+    }
+
+    private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) BuildRangeMetadata(
+        RuNkckiSoftwareEntry entry,
+        DateTimeOffset recordedAt)
+    {
+        if (entry.RangeExpressions.IsDefaultOrEmpty || entry.RangeExpressions.Length == 0)
+        {
+            return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>());
+        }
+
+        var ranges = new List<AffectedVersionRange>();
+        var normalized = new List<NormalizedVersionRule>();
+        var dedupe = new HashSet<string>(StringComparer.Ordinal);
+
+        foreach (var expression in entry.RangeExpressions)
+        {
+            if (string.IsNullOrWhiteSpace(expression))
+            {
+                continue;
+            }
+
+            var results = SemVerRangeRuleBuilder.Build(expression, provenanceNote: entry.Evidence);
+            if (results.Count == 0)
+            {
+                continue;
+            }
+
+            foreach (var result in results)
+            {
+                var key = $"{result.Primitive.Introduced}|{result.Primitive.Fixed}|{result.Primitive.LastAffected}|{result.Expression}";
+                if (!dedupe.Add(key))
+                {
+                    continue;
+                }
+
+                var provenance = new AdvisoryProvenance(
+                    RuNkckiConnectorPlugin.SourceName,
+                    "package-range",
+                    entry.Evidence,
+                    recordedAt,
+                    new[] { ProvenanceFieldMasks.VersionRanges });
+
+                ranges.Add(new AffectedVersionRange(
+                    NormalizedVersionSchemes.SemVer,
+                    result.Primitive.Introduced,
+                    result.Primitive.Fixed,
+                    result.Primitive.LastAffected,
+                    result.Expression,
+                    provenance,
+                    new RangePrimitives(result.Primitive, null, null, null)));
+
+                normalized.Add(result.NormalizedRule);
+            }
+        }
+
+        return (
+            ranges.Count == 0 ? Array.Empty<AffectedVersionRange>() : ranges,
+            normalized.Count == 0 ? Array.Empty<NormalizedVersionRule>() : normalized);
+    }
+    private static string DetermineCvss4Severity(double score)
+    {
+        if (score <= 0.0)
+        {
+            return "none";
+        }
+
+        if (score < 4.0)
+        {
+            return "low";
+        }
+
+        if (score < 7.0)
+        {
+            return "medium";
+        }
+
+        if (score < 9.0)
+        {
+            return "high";
+        }
+
+        return "critical";
+    }
 }

src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs

@@ -12,10 +12,11 @@ internal sealed record RuNkckiVulnerabilityDto(
     bool? PatchAvailable,
     string? Description,
     RuNkckiCweDto? Cwe,
-    string? ProductCategory,
+    ImmutableArray<string> ProductCategories,
     string? Mitigation,
     string? VulnerableSoftwareText,
     bool? VulnerableSoftwareHasCpe,
+    ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
     double? CvssScore,
     string? CvssVector,
     double? CvssScoreV4,
@@ -23,7 +24,8 @@ internal sealed record RuNkckiVulnerabilityDto(
     string? Impact,
     string? MethodOfExploitation,
     bool? UserInteraction,
-    ImmutableArray<string> Urls)
+    ImmutableArray<string> Urls,
+    ImmutableArray<string> Tags)
 {
     [JsonIgnore]
     public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
@@ -34,3 +36,5 @@ internal sealed record RuNkckiVulnerabilityDto(
 }
 
 internal sealed record RuNkckiCweDto(int? Number, string? Description);
+
+internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);

src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs

@@ -55,6 +55,7 @@ public sealed class RuNkckiConnector : IFeedConnector
     private readonly ISourceStateRepository _stateRepository;
     private readonly RuNkckiOptions _options;
     private readonly TimeProvider _timeProvider;
+    private readonly RuNkckiDiagnostics _diagnostics;
     private readonly ILogger<RuNkckiConnector> _logger;
     private readonly string _cacheDirectory;
 
@@ -68,6 +69,7 @@ public sealed class RuNkckiConnector : IFeedConnector
         IAdvisoryStore advisoryStore,
         ISourceStateRepository stateRepository,
         IOptions<RuNkckiOptions> options,
+        RuNkckiDiagnostics diagnostics,
         TimeProvider? timeProvider,
         ILogger<RuNkckiConnector> logger)
     {
@@ -79,6 +81,7 @@ public sealed class RuNkckiConnector : IFeedConnector
         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
         _options.Validate();
+        _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
         _timeProvider = timeProvider ?? TimeProvider.System;
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory);
@@ -98,28 +101,12 @@ public sealed class RuNkckiConnector : IFeedConnector
         var now = _timeProvider.GetUtcNow();
         var processed = 0;
 
-        IReadOnlyList<BulletinAttachment> attachments = Array.Empty<BulletinAttachment>();
-
-        try
+        if (ShouldUseListingCache(cursor, now))
         {
-            var listingResult = await FetchListingAsync(cancellationToken).ConfigureAwait(false);
-            if (!listingResult.IsSuccess || listingResult.Content is null)
-            {
-                _logger.LogWarning("NKCKI listing fetch returned no content (status={Status})", listingResult.StatusCode);
-                processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
-                await UpdateCursorAsync(cursor
-                    .WithPendingDocuments(pendingDocuments)
-                    .WithPendingMappings(pendingMappings)
-                    .WithKnownBulletins(NormalizeBulletins(knownBulletins))
-                    .WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
-                return;
-            }
+            _logger.LogDebug(
+                "NKCKI listing fetch skipped (cache duration {CacheDuration:c}); processing cached bulletins only",
+                _options.ListingCacheDuration);
 
-            attachments = await ParseListingAsync(listingResult.Content, cancellationToken).ConfigureAwait(false);
-        }
-        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
-        {
-            _logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins");
             processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
             await UpdateCursorAsync(cursor
                 .WithPendingDocuments(pendingDocuments)
@@ -129,9 +116,42 @@ public sealed class RuNkckiConnector : IFeedConnector
             return;
         }
 
-        if (attachments.Count == 0)
+        ListingFetchSummary listingSummary;
+        try
         {
-            _logger.LogDebug("NKCKI listing contained no bulletin attachments");
+            listingSummary = await LoadListingAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
+        {
+            _logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins");
+            _diagnostics.ListingFetchFailure(ex.Message);
+            await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
+
+            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
+            await UpdateCursorAsync(cursor
+                .WithPendingDocuments(pendingDocuments)
+                .WithPendingMappings(pendingMappings)
+                .WithKnownBulletins(NormalizeBulletins(knownBulletins))
+                .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        var uniqueAttachments = listingSummary.Attachments
+            .GroupBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
+            .Select(static group => group.First())
+            .OrderBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        var newAttachments = uniqueAttachments
+            .Where(attachment => !knownBulletins.Contains(attachment.Id))
+            .Take(_options.MaxBulletinsPerFetch)
+            .ToList();
+
+        _diagnostics.ListingFetchSuccess(listingSummary.PagesVisited, uniqueAttachments.Count, newAttachments.Count);
+
+        if (newAttachments.Count == 0)
+        {
+            _logger.LogDebug("NKCKI listing contained no new bulletin attachments");
             processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
             await UpdateCursorAsync(cursor
                 .WithPendingDocuments(pendingDocuments)
@@ -141,20 +161,9 @@ public sealed class RuNkckiConnector : IFeedConnector
             return;
         }
 
-        var newAttachments = attachments
-            .Where(attachment => !knownBulletins.Contains(attachment.Id))
-            .Take(_options.MaxBulletinsPerFetch)
-            .ToList();
-
-        if (newAttachments.Count == 0)
-        {
-            await UpdateCursorAsync(cursor
-                .WithPendingDocuments(pendingDocuments)
-                .WithPendingMappings(pendingMappings)
-                .WithKnownBulletins(NormalizeBulletins(knownBulletins))
-                .WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
-            return;
-        }
+        var downloaded = 0;
+        var cachedUsed = 0;
+        var failures = 0;
 
         foreach (var attachment in newAttachments)
         {
@@ -173,18 +182,24 @@ public sealed class RuNkckiConnector : IFeedConnector
                 {
                     if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
                     {
+                        _diagnostics.BulletinFetchCached();
+                        cachedUsed++;
                         _logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode);
                         processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                         knownBulletins.Add(attachment.Id);
                     }
                     else
                     {
+                        _diagnostics.BulletinFetchFailure(attachmentResult.StatusCode.ToString());
+                        failures++;
                         _logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode);
                     }
 
                     continue;
                 }
 
+                _diagnostics.BulletinFetchSuccess();
+                downloaded++;
                 TryWriteCachedBulletin(attachment.Id, attachmentResult.Content);
                 processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                 knownBulletins.Add(attachment.Id);
@@ -193,12 +208,16 @@ public sealed class RuNkckiConnector : IFeedConnector
             {
                 if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
                 {
+                    _diagnostics.BulletinFetchCached();
+                    cachedUsed++;
                     _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id);
                     processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
                     knownBulletins.Add(attachment.Id);
                 }
                 else
                 {
+                    _diagnostics.BulletinFetchFailure(ex.Message);
+                    failures++;
                     _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id);
                     await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
                     throw;
@@ -223,6 +242,11 @@ public sealed class RuNkckiConnector : IFeedConnector
             }
         }
 
+        if (processed < _options.MaxVulnerabilitiesPerFetch)
+        {
+            processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
+        }
+
         var normalizedBulletins = NormalizeBulletins(knownBulletins);
 
         var updatedCursor = cursor
@@ -232,6 +256,15 @@ public sealed class RuNkckiConnector : IFeedConnector
             .WithLastListingFetch(now);
 
         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
+
+        _logger.LogInformation(
+            "NKCKI fetch complete: new bulletins {Downloaded}, cached bulletins {Cached}, failures {Failures}, processed entries {Processed}, pending documents {PendingDocuments}, pending mappings {PendingMappings}",
+            downloaded,
+            cachedUsed,
+            failures,
+            processed,
+            pendingDocuments.Count,
+            pendingMappings.Count);
     }
 
     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
@@ -425,6 +458,7 @@ public sealed class RuNkckiConnector : IFeedConnector
                 continue;
             }
 
+            _diagnostics.BulletinFetchCached();
             updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
             knownBulletins.Add(bulletinId);
 
@@ -484,6 +518,12 @@ public sealed class RuNkckiConnector : IFeedConnector
             }
         }
 
+        var delta = updated - processed;
+        if (delta > 0)
+        {
+            _diagnostics.EntriesProcessed(delta);
+        }
+
         return updated;
     }
 
@@ -607,34 +647,25 @@ public sealed class RuNkckiConnector : IFeedConnector
         return true;
     }
 
-    private async Task<SourceFetchContentResult> FetchListingAsync(CancellationToken cancellationToken)
+    private Task<SourceFetchContentResult> FetchListingPageAsync(Uri pageUri, CancellationToken cancellationToken)
     {
-        try
+        var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, pageUri)
         {
-            var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, _options.ListingUri)
-            {
-                AcceptHeaders = ListingAcceptHeaders,
-                TimeoutOverride = _options.RequestTimeout,
-            };
+            AcceptHeaders = ListingAcceptHeaders,
+            TimeoutOverride = _options.RequestTimeout,
+        };
 
-            return await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
-        }
-        catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
-        {
-            _logger.LogError(ex, "NKCKI listing fetch failed for {ListingUri}", _options.ListingUri);
-            await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
-            throw;
-        }
+        return _fetchService.FetchContentAsync(request, cancellationToken);
     }
 
-    private async Task<IReadOnlyList<BulletinAttachment>> ParseListingAsync(byte[] content, CancellationToken cancellationToken)
+    private async Task<ListingPageResult> ParseListingAsync(Uri pageUri, byte[] content, CancellationToken cancellationToken)
     {
         var html = Encoding.UTF8.GetString(content);
         var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false);
-        var anchors = document.QuerySelectorAll("a[href$='.json.zip']");
-
         var attachments = new List<BulletinAttachment>();
-        foreach (var anchor in anchors)
+        var pagination = new List<Uri>();
+
+        foreach (var anchor in document.QuerySelectorAll("a[href$='.json.zip']"))
         {
             var href = anchor.GetAttribute("href");
             if (string.IsNullOrWhiteSpace(href))
@@ -642,7 +673,7 @@ public sealed class RuNkckiConnector : IFeedConnector
                 continue;
             }
 
-            if (!Uri.TryCreate(_options.BaseAddress, href, out var absoluteUri))
+            if (!Uri.TryCreate(pageUri, href, out var absoluteUri))
             {
                 continue;
             }
@@ -662,7 +693,32 @@ public sealed class RuNkckiConnector : IFeedConnector
             attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id));
         }
 
-        return attachments;
+        foreach (var anchor in document.QuerySelectorAll("a[href]"))
+        {
+            var href = anchor.GetAttribute("href");
+            if (string.IsNullOrWhiteSpace(href))
+            {
+                continue;
+            }
+
+            if (!href.Contains("PAGEN", StringComparison.OrdinalIgnoreCase)
+                && !href.Contains("page=", StringComparison.OrdinalIgnoreCase))
+            {
+                continue;
+            }
+
+            if (Uri.TryCreate(pageUri, href, out var absoluteUri))
+            {
+                pagination.Add(absoluteUri);
+            }
+        }
+
+        var uniquePagination = pagination
+            .DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase)
+            .Take(_options.MaxListingPagesPerFetch)
+            .ToList();
+
+        return new ListingPageResult(attachments, uniquePagination);
     }
 
     private static string DeriveBulletinId(Uri uri)
@@ -821,5 +877,70 @@ public sealed class RuNkckiConnector : IFeedConnector
         return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
     }
 
+    private readonly record struct ListingFetchSummary(IReadOnlyList<BulletinAttachment> Attachments, int PagesVisited);
+
+    private readonly record struct ListingPageResult(IReadOnlyList<BulletinAttachment> Attachments, IReadOnlyList<Uri> PaginationLinks);
+
     private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title);
+
+    private bool ShouldUseListingCache(RuNkckiCursor cursor, DateTimeOffset now)
+    {
+        if (!cursor.LastListingFetchAt.HasValue)
+        {
+            return false;
+        }
+
+        var age = now - cursor.LastListingFetchAt.Value;
+        return age < _options.ListingCacheDuration;
+    }
+
+    private async Task<ListingFetchSummary> LoadListingAsync(CancellationToken cancellationToken)
+    {
+        var attachments = new List<BulletinAttachment>();
+        var visited = 0;
+        var visitedUris = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var queue = new Queue<Uri>();
+        queue.Enqueue(_options.ListingUri);
+
+        while (queue.Count > 0 && visited < _options.MaxListingPagesPerFetch)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var pageUri = queue.Dequeue();
+            if (!visitedUris.Add(pageUri.AbsoluteUri))
+            {
+                continue;
+            }
+
+            _diagnostics.ListingFetchAttempt();
+
+            var listingResult = await FetchListingPageAsync(pageUri, cancellationToken).ConfigureAwait(false);
+            if (!listingResult.IsSuccess || listingResult.Content is null)
+            {
+                _diagnostics.ListingFetchFailure(listingResult.StatusCode.ToString());
+                _logger.LogWarning("NKCKI listing page {ListingUri} returned no content (status={Status})", pageUri, listingResult.StatusCode);
+                continue;
+            }
+
+            visited++;
+
+            var page = await ParseListingAsync(pageUri, listingResult.Content, cancellationToken).ConfigureAwait(false);
+            attachments.AddRange(page.Attachments);
+
+            foreach (var link in page.PaginationLinks)
+            {
+                if (!visitedUris.Contains(link.AbsoluteUri) && queue.Count + visitedUris.Count < _options.MaxListingPagesPerFetch)
+                {
+                    queue.Enqueue(link);
+                }
+            }
+
+            if (attachments.Count >= _options.MaxBulletinsPerFetch * 2)
+            {
+                break;
+            }
+        }
+
+        return new ListingFetchSummary(attachments, visited);
+    }
 }

src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs

@@ -1,8 +1,10 @@
 using System.Net;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Options;
 using StellaOps.Feedser.Source.Common.Http;
 using StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
+using StellaOps.Feedser.Source.Ru.Nkcki.Internal;
 
 namespace StellaOps.Feedser.Source.Ru.Nkcki;
 
@@ -36,6 +38,7 @@ public static class RuNkckiServiceCollectionExtensions
             };
         });
 
+        services.TryAddSingleton<RuNkckiDiagnostics>();
         services.AddTransient<RuNkckiConnector>();
 
         return services;

src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md

@@ -2,10 +2,10 @@
 | Task | Owner(s) | Depends on | Notes |
 |---|---|---|---|
 |FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/feedser-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.|
-|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DOING (2025-10-12)** – Listing fetch now expands `*.json.zip` bulletins into per-vulnerability JSON documents with cursor-tracked bulletin IDs and trust store wiring (`globalsign_r6_bundle.pem`). Parser/mapper emit canonical advisories; remaining work: strengthen pagination/backfill handling and add regression fixtures/telemetry. Offline cache helpers (ProcessCachedBulletinsAsync/TryReadCachedBulletin/TryWriteCachedBulletin) implemented.|
-|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DOING (2025-10-12)** – `RuNkckiJsonParser` extracts per-vulnerability JSON payloads (IDs, CVEs, CVSS, software text, URLs). TODO: extend coverage for optional fields (ICS categories, nested arrays) and add fixture snapshots.|
-|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DOING (2025-10-12)** – `RuNkckiMapper` maps JSON entries to canonical advisories (aliases, references, vendor package, CVSS). Next steps: enrich package parsing (`software_text` tokenisation), consider CVSS v4 metadata, and backfill provenance docs before closing the task.|
-|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DOING (2025-10-12)** – Added mocked listing/bulletin regression harness (`RuNkckiConnectorTests`) with fixtures + snapshot writer. Test run currently blocked on Mongo2Go dependency (libcrypto.so.1.1 missing); follow-up required to get embedded mongod running in CI before marking DONE.|
-|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, and close backlog entry after deliverable ships.|
-|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**TODO** – Once access restored, map Bitrix paging (`?PAGEN_1=`) and advisory taxonomy (alerts vs recommendations). Outline HTML scrape + PDF attachment handling for backfill and decide translation approach for Russian-only content.|
+|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DONE (2025-10-13)** – Listing fetch now honours `maxListingPagesPerFetch`, persists cache hits when listing access fails, and records telemetry via `RuNkckiDiagnostics`. Cursor tracking covers pending documents/mappings and the known bulletin ring buffer.|
+|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-13)** – Parser normalises nested arrays (ICS categories, vulnerable software lists, optional tags), flattens multiline `software_text`, and guarantees deterministic ordering for URLs and tags.|
+|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DONE (2025-10-13)** – Mapper splits structured software entries, emits SemVer range primitives + normalized rules, deduplicates references, and surfaces CVSS v4 metadata alongside existing metrics.|
+|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-13)** – Fixtures refreshed with multi-page pagination + multi-entry bulletins. Tests exercise cache replay and rely on bundled OpenSSL 1.1 libs in `tools/openssl/linux-x64` to keep Mongo2Go green on modern distros.|
+|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-13)** – Added connector-specific metrics (`nkcki.*`) and documented configuration/operational guidance in `docs/ops/feedser-nkcki-operations.md`.|
+|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**DONE (2025-10-13)** – Documented Bitrix pagination/backfill plan (cache-first, offline replay, HTML/PDF capture) in `docs/ops/feedser-nkcki-operations.md`.|
 |FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`feedser:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.|