Rename Concelier Source modules to Connector
This commit is contained in:
master
38
src/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md
Normal file
38
src/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Implement the Russian NKTsKI (formerly NKCKI) advisories connector to ingest NKTsKI vulnerability bulletins for Concelier’s regional coverage.
|
||||
|
||||
## Scope
|
||||
- Identify NKTsKI advisory feeds/APIs (HTML, RSS, CSV) and access/authentication requirements.
|
||||
- Implement fetch/cursor pipeline with dedupe and failure backoff tailored to the source format.
|
||||
- Parse advisories to extract summary, affected vendors/products, recommended mitigation, and CVE identifiers.
|
||||
- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives.
|
||||
- Create deterministic fixtures and regression tests.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP/fetch utilities, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores, source state).
|
||||
- `Concelier.Models` (canonical data structures).
|
||||
- `Concelier.Testing` (integration fixtures, snapshots).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `nkcki:fetch`, `nkcki:parse`, `nkcki:map`.
|
||||
- Persist upstream modification metadata to support incremental updates.
|
||||
- Alias set should include NKTsKI advisory IDs and CVEs when present.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- Core ingestion/mapping pipeline with range primitives.
|
||||
|
||||
Out of scope:
|
||||
- Translation beyond canonical field normalisation.
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch/mapping activity; mark failures with backoff delays.
|
||||
- Handle Cyrillic text encoding and sanitise HTML safely.
|
||||
- Respect upstream rate limiting/politeness.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Ru.Nkcki.Tests` for fetch/parse/map with canned fixtures.
|
||||
- Snapshot canonical advisories; support fixture regeneration via env flag.
|
||||
- Ensure deterministic ordering/time normalisation.
|
||||
@@ -0,0 +1,137 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
|
||||
/// </summary>
|
||||
public sealed class RuNkckiOptions
|
||||
{
|
||||
public const string HttpClientName = "ru-nkcki";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
|
||||
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint used for resolving relative resource links.
|
||||
/// </summary>
|
||||
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the bulletin listing page.
|
||||
/// </summary>
|
||||
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to listing and bulletin fetch requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the listing or attachments cannot be retrieved.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of bulletin attachments downloaded per fetch run.
|
||||
/// </summary>
|
||||
public int MaxBulletinsPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of listing pages visited per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxListingPagesPerFetch { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
|
||||
/// </summary>
|
||||
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
|
||||
/// </summary>
|
||||
public int KnownBulletinCapacity { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between sequential bulletin downloads.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Duration the HTML listing can be cached before forcing a refetch.
|
||||
/// </summary>
|
||||
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
|
||||
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
|
||||
|
||||
/// <summary>
|
||||
/// Absolute URI for the listing page.
|
||||
/// </summary>
|
||||
public Uri ListingUri => new(BaseAddress, ListingPath);
|
||||
|
||||
/// <summary>
|
||||
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; } = null;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ListingPath))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxBulletinsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxListingPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxVulnerabilitiesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (KnownBulletinCapacity <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
|
||||
}
|
||||
|
||||
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AcceptLanguage))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownBulletins,
|
||||
DateTimeOffset? LastListingFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
|
||||
|
||||
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
|
||||
|
||||
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
|
||||
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
|
||||
|
||||
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastListingFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownBulletins"] = new BsonArray(KnownBulletins),
|
||||
};
|
||||
|
||||
if (LastListingFetchAt.HasValue)
|
||||
{
|
||||
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static RuNkckiCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownBulletins = ReadStringArray(document, "knownBulletins");
|
||||
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyBulletins;
|
||||
}
|
||||
|
||||
var result = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
var text = element?.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
result.Add(text);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits telemetry counters for the NKCKI connector lifecycle.
|
||||
/// </summary>
|
||||
public sealed class RuNkckiDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Ru.Nkcki";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _listingFetchAttempts;
|
||||
private readonly Counter<long> _listingFetchSuccess;
|
||||
private readonly Counter<long> _listingFetchFailures;
|
||||
private readonly Histogram<long> _listingPagesVisited;
|
||||
private readonly Histogram<long> _listingAttachmentsDiscovered;
|
||||
private readonly Histogram<long> _listingAttachmentsNew;
|
||||
private readonly Counter<long> _bulletinFetchSuccess;
|
||||
private readonly Counter<long> _bulletinFetchCached;
|
||||
private readonly Counter<long> _bulletinFetchFailures;
|
||||
private readonly Histogram<long> _entriesProcessed;
|
||||
|
||||
public RuNkckiDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_listingFetchAttempts = _meter.CreateCounter<long>(
|
||||
"nkcki.listing.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of listing fetch attempts.");
|
||||
_listingFetchSuccess = _meter.CreateCounter<long>(
|
||||
"nkcki.listing.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of successful listing fetches.");
|
||||
_listingFetchFailures = _meter.CreateCounter<long>(
|
||||
"nkcki.listing.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of listing fetch failures.");
|
||||
_listingPagesVisited = _meter.CreateHistogram<long>(
|
||||
"nkcki.listing.pages.visited",
|
||||
unit: "pages",
|
||||
description: "Listing pages visited per fetch cycle.");
|
||||
_listingAttachmentsDiscovered = _meter.CreateHistogram<long>(
|
||||
"nkcki.listing.attachments.discovered",
|
||||
unit: "attachments",
|
||||
description: "Attachments discovered across listing pages.");
|
||||
_listingAttachmentsNew = _meter.CreateHistogram<long>(
|
||||
"nkcki.listing.attachments.new",
|
||||
unit: "attachments",
|
||||
description: "New bulletin attachments enqueued per fetch cycle.");
|
||||
_bulletinFetchSuccess = _meter.CreateCounter<long>(
|
||||
"nkcki.bulletin.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of bulletin downloads that succeeded.");
|
||||
_bulletinFetchCached = _meter.CreateCounter<long>(
|
||||
"nkcki.bulletin.fetch.cached",
|
||||
unit: "operations",
|
||||
description: "Number of bulletins served from cache.");
|
||||
_bulletinFetchFailures = _meter.CreateCounter<long>(
|
||||
"nkcki.bulletin.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of bulletin download failures.");
|
||||
_entriesProcessed = _meter.CreateHistogram<long>(
|
||||
"nkcki.entries.processed",
|
||||
unit: "entries",
|
||||
description: "Number of vulnerability entries processed per bulletin.");
|
||||
}
|
||||
|
||||
public void ListingFetchAttempt() => _listingFetchAttempts.Add(1);
|
||||
|
||||
public void ListingFetchSuccess(int pagesVisited, int attachmentsDiscovered, int attachmentsNew)
|
||||
{
|
||||
_listingFetchSuccess.Add(1);
|
||||
if (pagesVisited >= 0)
|
||||
{
|
||||
_listingPagesVisited.Record(pagesVisited);
|
||||
}
|
||||
|
||||
if (attachmentsDiscovered >= 0)
|
||||
{
|
||||
_listingAttachmentsDiscovered.Record(attachmentsDiscovered);
|
||||
}
|
||||
|
||||
if (attachmentsNew >= 0)
|
||||
{
|
||||
_listingAttachmentsNew.Record(attachmentsNew);
|
||||
}
|
||||
}
|
||||
|
||||
public void ListingFetchFailure(string reason)
|
||||
=> _listingFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void BulletinFetchSuccess() => _bulletinFetchSuccess.Add(1);
|
||||
|
||||
public void BulletinFetchCached() => _bulletinFetchCached.Add(1);
|
||||
|
||||
public void BulletinFetchFailure(string reason)
|
||||
=> _bulletinFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void EntriesProcessed(int count)
|
||||
{
|
||||
if (count >= 0)
|
||||
{
|
||||
_entriesProcessed.Record(count);
|
||||
}
|
||||
}
|
||||
|
||||
private static KeyValuePair<string, object?> ReasonTag(string reason)
|
||||
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal static class RuNkckiJsonParser
|
||||
{
|
||||
private static readonly Regex ComparatorRegex = new(
|
||||
@"^(?<name>.+?)\s*(?<operator><=|>=|<|>|==|=)\s*(?<version>.+?)$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex RangeRegex = new(
|
||||
@"^(?<name>.+?)\s+(?<start>[\p{L}\p{N}\._-]+)\s*[-–]\s*(?<end>[\p{L}\p{N}\._-]+)$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex QualifierRegex = new(
|
||||
@"^(?<name>.+?)\s+(?<version>[\p{L}\p{N}\._-]+)\s+(?<qualifier>(and\s+earlier|and\s+later|and\s+newer|до\s+и\s+включительно|и\s+ниже|и\s+выше|и\s+старше|и\s+позже))$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Regex QualifierInlineRegex = new(
|
||||
@"верс(ии|ия)\s+(?<version>[\p{L}\p{N}\._-]+)\s+(?<qualifier>и\s+ниже|и\s+выше|и\s+старше)",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Regex VersionWindowRegex = new(
|
||||
@"верс(ии|ия)\s+(?<start>[\p{L}\p{N}\._-]+)\s+по\s+(?<end>[\p{L}\p{N}\._-]+)",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly char[] SoftwareSplitDelimiters = { '\n', ';', '\u2022', '\u2023', '\r' };
|
||||
|
||||
private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase;
|
||||
|
||||
public static RuNkckiVulnerabilityDto Parse(JsonElement element)
|
||||
{
|
||||
var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec)
|
||||
? Normalize(fstec.GetString())
|
||||
: null;
|
||||
var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre)
|
||||
? Normalize(mitre.GetString())
|
||||
: null;
|
||||
|
||||
var datePublished = ParseDate(element.TryGetProperty("date_published", out var published) ? published.GetString() : null);
|
||||
var dateUpdated = ParseDate(element.TryGetProperty("date_updated", out var updated) ? updated.GetString() : null);
|
||||
var cvssRating = Normalize(element.TryGetProperty("cvss_rating", out var rating) ? rating.GetString() : null);
|
||||
bool? patchAvailable = element.TryGetProperty("patch_available", out var patch) ? patch.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => null,
|
||||
} : null;
|
||||
|
||||
var description = ReadJoinedString(element, "description");
|
||||
var mitigation = ReadJoinedString(element, "mitigation");
|
||||
var productCategories = ReadStringCollection(element, "product_category");
|
||||
var impact = ReadJoinedString(element, "impact");
|
||||
var method = ReadJoinedString(element, "method_of_exploitation");
|
||||
bool? userInteraction = element.TryGetProperty("user_interaction", out var uiElement) ? uiElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => null,
|
||||
} : null;
|
||||
|
||||
var (softwareText, softwareHasCpe, softwareEntries) = ParseVulnerableSoftware(element);
|
||||
|
||||
RuNkckiCweDto? cweDto = null;
|
||||
if (element.TryGetProperty("cwe", out var cweElement))
|
||||
{
|
||||
int? number = null;
|
||||
if (cweElement.TryGetProperty("cwe_number", out var numberElement))
|
||||
{
|
||||
if (numberElement.ValueKind == JsonValueKind.Number && numberElement.TryGetInt32(out var parsed))
|
||||
{
|
||||
number = parsed;
|
||||
}
|
||||
else if (int.TryParse(numberElement.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedInt))
|
||||
{
|
||||
number = parsedInt;
|
||||
}
|
||||
}
|
||||
|
||||
var cweDescription = ReadJoinedString(cweElement, "cwe_description") ?? Normalize(cweElement.GetString());
|
||||
if (number.HasValue || !string.IsNullOrWhiteSpace(cweDescription))
|
||||
{
|
||||
cweDto = new RuNkckiCweDto(number, cweDescription);
|
||||
}
|
||||
}
|
||||
|
||||
double? cvssScore = element.TryGetProperty("cvss", out var cvssElement) && cvssElement.TryGetProperty("cvss_score", out var scoreElement)
|
||||
? ParseDouble(scoreElement)
|
||||
: null;
|
||||
var cvssVector = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector", out var vectorElement)
|
||||
? Normalize(vectorElement.GetString())
|
||||
: null;
|
||||
double? cvssScoreV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_score_v4", out var scoreV4Element)
|
||||
? ParseDouble(scoreV4Element)
|
||||
: null;
|
||||
var cvssVectorV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector_v4", out var vectorV4Element)
|
||||
? Normalize(vectorV4Element.GetString())
|
||||
: null;
|
||||
|
||||
var urls = ReadUrls(element);
|
||||
var tags = ReadStringCollection(element, "tags");
|
||||
|
||||
return new RuNkckiVulnerabilityDto(
|
||||
fstecId,
|
||||
mitreId,
|
||||
datePublished,
|
||||
dateUpdated,
|
||||
cvssRating,
|
||||
patchAvailable,
|
||||
description,
|
||||
cweDto,
|
||||
productCategories,
|
||||
mitigation,
|
||||
softwareText,
|
||||
softwareHasCpe,
|
||||
softwareEntries,
|
||||
cvssScore,
|
||||
cvssVector,
|
||||
cvssScoreV4,
|
||||
cvssVectorV4,
|
||||
impact,
|
||||
method,
|
||||
userInteraction,
|
||||
urls,
|
||||
tags);
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ReadUrls(JsonElement element)
|
||||
{
|
||||
if (!element.TryGetProperty("urls", out var urlsElement))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var collected = new List<string>();
|
||||
CollectUrls(urlsElement, collected);
|
||||
if (collected.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
return collected
|
||||
.Select(Normalize)
|
||||
.Where(static url => !string.IsNullOrWhiteSpace(url))
|
||||
.Select(static url => url!)
|
||||
.Distinct(OrdinalIgnoreCase)
|
||||
.OrderBy(static url => url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static void CollectUrls(JsonElement element, ICollection<string> results)
|
||||
{
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.String:
|
||||
var value = element.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
results.Add(value);
|
||||
}
|
||||
break;
|
||||
case JsonValueKind.Array:
|
||||
foreach (var child in element.EnumerateArray())
|
||||
{
|
||||
CollectUrls(child, results);
|
||||
}
|
||||
break;
|
||||
case JsonValueKind.Object:
|
||||
if (element.TryGetProperty("url", out var urlProperty))
|
||||
{
|
||||
CollectUrls(urlProperty, results);
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("href", out var hrefProperty))
|
||||
{
|
||||
CollectUrls(hrefProperty, results);
|
||||
}
|
||||
|
||||
foreach (var property in element.EnumerateObject())
|
||||
{
|
||||
if (property.NameEquals("value") || property.NameEquals("link"))
|
||||
{
|
||||
CollectUrls(property.Value, results);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ReadJoinedString(JsonElement element, string property)
|
||||
{
|
||||
if (!element.TryGetProperty(property, out var target))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var values = ReadStringCollection(target);
|
||||
if (!values.IsDefaultOrEmpty)
|
||||
{
|
||||
return string.Join("; ", values);
|
||||
}
|
||||
|
||||
return Normalize(target.ValueKind == JsonValueKind.String ? target.GetString() : target.ToString());
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ReadStringCollection(JsonElement element, string property)
|
||||
{
|
||||
if (!element.TryGetProperty(property, out var target))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
return ReadStringCollection(target);
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ReadStringCollection(JsonElement element)
|
||||
{
|
||||
var builder = ImmutableArray.CreateBuilder<string>();
|
||||
CollectStrings(element, builder);
|
||||
return Deduplicate(builder);
|
||||
}
|
||||
|
||||
private static void CollectStrings(JsonElement element, ImmutableArray<string>.Builder builder)
|
||||
{
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.String:
|
||||
AddIfPresent(builder, Normalize(element.GetString()));
|
||||
break;
|
||||
case JsonValueKind.Number:
|
||||
AddIfPresent(builder, Normalize(element.ToString()));
|
||||
break;
|
||||
case JsonValueKind.True:
|
||||
builder.Add("true");
|
||||
break;
|
||||
case JsonValueKind.False:
|
||||
builder.Add("false");
|
||||
break;
|
||||
case JsonValueKind.Array:
|
||||
foreach (var child in element.EnumerateArray())
|
||||
{
|
||||
CollectStrings(child, builder);
|
||||
}
|
||||
break;
|
||||
case JsonValueKind.Object:
|
||||
foreach (var property in element.EnumerateObject())
|
||||
{
|
||||
CollectStrings(property.Value, builder);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> Deduplicate(ImmutableArray<string>.Builder builder)
|
||||
{
|
||||
if (builder.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
return builder
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(OrdinalIgnoreCase)
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static void AddIfPresent(ImmutableArray<string>.Builder builder, string? value)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
builder.Add(value!);
|
||||
}
|
||||
}
|
||||
|
||||
private static (string? Text, bool? HasCpe, ImmutableArray<RuNkckiSoftwareEntry> Entries) ParseVulnerableSoftware(JsonElement element)
|
||||
{
|
||||
if (!element.TryGetProperty("vulnerable_software", out var softwareElement))
|
||||
{
|
||||
return (null, null, ImmutableArray<RuNkckiSoftwareEntry>.Empty);
|
||||
}
|
||||
|
||||
string? softwareText = null;
|
||||
if (softwareElement.TryGetProperty("software_text", out var textElement))
|
||||
{
|
||||
softwareText = Normalize(textElement.ValueKind == JsonValueKind.String ? textElement.GetString() : textElement.ToString());
|
||||
}
|
||||
|
||||
bool? softwareHasCpe = null;
|
||||
if (softwareElement.TryGetProperty("cpe", out var cpeElement))
|
||||
{
|
||||
softwareHasCpe = cpeElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => softwareHasCpe,
|
||||
};
|
||||
}
|
||||
|
||||
var entries = new List<RuNkckiSoftwareEntry>();
|
||||
if (softwareElement.TryGetProperty("software", out var softwareNodes))
|
||||
{
|
||||
entries.AddRange(ParseSoftwareEntries(softwareNodes));
|
||||
}
|
||||
|
||||
if (entries.Count == 0 && !string.IsNullOrWhiteSpace(softwareText))
|
||||
{
|
||||
entries.AddRange(SplitSoftwareTextIntoEntries(softwareText));
|
||||
}
|
||||
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
foreach (var fallbackProperty in new[] { "items", "aliases", "software_lines" })
|
||||
{
|
||||
if (softwareElement.TryGetProperty(fallbackProperty, out var fallbackNodes))
|
||||
{
|
||||
entries.AddRange(ParseSoftwareEntries(fallbackNodes));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
return (softwareText, softwareHasCpe, ImmutableArray<RuNkckiSoftwareEntry>.Empty);
|
||||
}
|
||||
|
||||
var grouped = entries
|
||||
.GroupBy(static entry => entry.Identifier, OrdinalIgnoreCase)
|
||||
.Select(static group =>
|
||||
{
|
||||
var evidence = string.Join(
|
||||
"; ",
|
||||
group.Select(static entry => entry.Evidence)
|
||||
.Where(static evidence => !string.IsNullOrWhiteSpace(evidence))
|
||||
.Distinct(OrdinalIgnoreCase));
|
||||
|
||||
var ranges = group
|
||||
.SelectMany(static entry => entry.RangeExpressions)
|
||||
.Where(static range => !string.IsNullOrWhiteSpace(range))
|
||||
.Distinct(OrdinalIgnoreCase)
|
||||
.OrderBy(static range => range, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RuNkckiSoftwareEntry(
|
||||
group.Key,
|
||||
string.IsNullOrWhiteSpace(evidence) ? group.Key : evidence,
|
||||
ranges);
|
||||
})
|
||||
.OrderBy(static entry => entry.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return (softwareText, softwareHasCpe, grouped);
|
||||
}
|
||||
|
||||
private static IEnumerable<RuNkckiSoftwareEntry> ParseSoftwareEntries(JsonElement element)
|
||||
{
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Array:
|
||||
foreach (var child in element.EnumerateArray())
|
||||
{
|
||||
foreach (var entry in ParseSoftwareEntries(child))
|
||||
{
|
||||
yield return entry;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case JsonValueKind.Object:
|
||||
yield return CreateEntryFromObject(element);
|
||||
break;
|
||||
case JsonValueKind.String:
|
||||
foreach (var entry in SplitSoftwareTextIntoEntries(element.GetString() ?? string.Empty))
|
||||
{
|
||||
yield return entry;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static RuNkckiSoftwareEntry CreateEntryFromObject(JsonElement element)
|
||||
{
|
||||
var vendor = ReadFirstString(element, "vendor", "manufacturer", "organisation");
|
||||
var name = ReadFirstString(element, "name", "product", "title");
|
||||
var rawVersion = ReadFirstString(element, "version", "versions", "range");
|
||||
var comment = ReadFirstString(element, "comment", "notes", "summary");
|
||||
|
||||
var identifierParts = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(vendor))
|
||||
{
|
||||
identifierParts.Add(vendor!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
identifierParts.Add(name!);
|
||||
}
|
||||
|
||||
var identifier = identifierParts.Count > 0
|
||||
? string.Join(" ", identifierParts)
|
||||
: ReadFirstString(element, "identifier") ?? name ?? rawVersion ?? comment ?? "unknown";
|
||||
|
||||
var evidenceParts = new List<string>(identifierParts);
|
||||
if (!string.IsNullOrWhiteSpace(rawVersion))
|
||||
{
|
||||
evidenceParts.Add(rawVersion!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(comment))
|
||||
{
|
||||
evidenceParts.Add(comment!);
|
||||
}
|
||||
|
||||
var evidence = string.Join(" ", evidenceParts.Where(static part => !string.IsNullOrWhiteSpace(part))).Trim();
|
||||
|
||||
var rangeHints = new List<string?>();
|
||||
if (!string.IsNullOrWhiteSpace(rawVersion))
|
||||
{
|
||||
rangeHints.Add(rawVersion);
|
||||
}
|
||||
|
||||
if (element.TryGetProperty("range", out var rangeElement))
|
||||
{
|
||||
rangeHints.Add(Normalize(rangeElement.ToString()));
|
||||
}
|
||||
|
||||
return CreateSoftwareEntry(identifier!, evidence, rangeHints);
|
||||
}
|
||||
|
||||
private static IEnumerable<RuNkckiSoftwareEntry> SplitSoftwareTextIntoEntries(string text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var segments = text.Split(SoftwareSplitDelimiters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
segments = new[] { text };
|
||||
}
|
||||
|
||||
foreach (var segment in segments)
|
||||
{
|
||||
var normalized = Normalize(segment);
|
||||
if (string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var (identifier, hints) = ExtractIdentifierAndRangeHints(normalized!);
|
||||
yield return CreateSoftwareEntry(identifier, normalized!, hints);
|
||||
}
|
||||
}
|
||||
|
||||
private static RuNkckiSoftwareEntry CreateSoftwareEntry(string identifier, string evidence, IEnumerable<string?> hints)
|
||||
{
|
||||
var normalizedIdentifier = Normalize(identifier) ?? "unknown";
|
||||
var normalizedEvidence = Normalize(evidence) ?? normalizedIdentifier;
|
||||
|
||||
var ranges = hints
|
||||
.Select(NormalizeRangeHint)
|
||||
.Where(static hint => !string.IsNullOrWhiteSpace(hint))
|
||||
.Select(static hint => hint!)
|
||||
.Distinct(OrdinalIgnoreCase)
|
||||
.OrderBy(static hint => hint, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new RuNkckiSoftwareEntry(normalizedIdentifier, normalizedEvidence!, ranges);
|
||||
}
|
||||
|
||||
private static string? NormalizeRangeHint(string? hint)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hint))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = Normalize(hint)?
|
||||
.Replace("≤", "<=", StringComparison.Ordinal)
|
||||
.Replace("≥", ">=", StringComparison.Ordinal)
|
||||
.Replace("=>", ">=", StringComparison.Ordinal)
|
||||
.Replace("=<", "<=", StringComparison.Ordinal);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static (string Identifier, IReadOnlyList<string?> RangeHints) ExtractIdentifierAndRangeHints(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return ("unknown", Array.Empty<string>());
|
||||
}
|
||||
|
||||
var comparatorMatch = ComparatorRegex.Match(value);
|
||||
if (comparatorMatch.Success)
|
||||
{
|
||||
var name = Normalize(comparatorMatch.Groups["name"].Value);
|
||||
var version = Normalize(comparatorMatch.Groups["version"].Value);
|
||||
var op = comparatorMatch.Groups["operator"].Value;
|
||||
return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $"{op} {version}" });
|
||||
}
|
||||
|
||||
var rangeMatch = RangeRegex.Match(value);
|
||||
if (rangeMatch.Success)
|
||||
{
|
||||
var name = Normalize(rangeMatch.Groups["name"].Value);
|
||||
var start = Normalize(rangeMatch.Groups["start"].Value);
|
||||
var end = Normalize(rangeMatch.Groups["end"].Value);
|
||||
return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" });
|
||||
}
|
||||
|
||||
var qualifierMatch = QualifierRegex.Match(value);
|
||||
if (qualifierMatch.Success)
|
||||
{
|
||||
var name = Normalize(qualifierMatch.Groups["name"].Value);
|
||||
var version = Normalize(qualifierMatch.Groups["version"].Value);
|
||||
var qualifier = qualifierMatch.Groups["qualifier"].Value.ToLowerInvariant();
|
||||
var hint = qualifier.Contains("ниж") || qualifier.Contains("earlier") || qualifier.Contains("включ")
|
||||
? $"<= {version}"
|
||||
: $">= {version}";
|
||||
return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint });
|
||||
}
|
||||
|
||||
var inlineQualifierMatch = QualifierInlineRegex.Match(value);
|
||||
if (inlineQualifierMatch.Success)
|
||||
{
|
||||
var version = Normalize(inlineQualifierMatch.Groups["version"].Value);
|
||||
var qualifier = inlineQualifierMatch.Groups["qualifier"].Value.ToLowerInvariant();
|
||||
var hint = qualifier.Contains("ниж") ? $"<= {version}" : $">= {version}";
|
||||
var name = Normalize(QualifierInlineRegex.Replace(value, string.Empty));
|
||||
return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint });
|
||||
}
|
||||
|
||||
var windowMatch = VersionWindowRegex.Match(value);
|
||||
if (windowMatch.Success)
|
||||
{
|
||||
var start = Normalize(windowMatch.Groups["start"].Value);
|
||||
var end = Normalize(windowMatch.Groups["end"].Value);
|
||||
var name = Normalize(VersionWindowRegex.Replace(value, string.Empty));
|
||||
return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" });
|
||||
}
|
||||
|
||||
return (value, Array.Empty<string>());
|
||||
}
|
||||
|
||||
private static string? ReadFirstString(JsonElement element, params string[] names)
|
||||
{
|
||||
foreach (var name in names)
|
||||
{
|
||||
if (element.TryGetProperty(name, out var property))
|
||||
{
|
||||
switch (property.ValueKind)
|
||||
{
|
||||
case JsonValueKind.String:
|
||||
{
|
||||
var normalized = Normalize(property.GetString());
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case JsonValueKind.Number:
|
||||
{
|
||||
var normalized = Normalize(property.ToString());
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static double? ParseDouble(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
if (element.ValueKind == JsonValueKind.String && double.TryParse(element.GetString(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var ruParsed))
|
||||
{
|
||||
return ruParsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = value
|
||||
.Replace('\r', ' ')
|
||||
.Replace('\n', ' ')
|
||||
.Trim();
|
||||
|
||||
while (normalized.Contains(" ", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = normalized.Replace(" ", " ", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
return normalized.Length == 0 ? null : normalized;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Cvss;
|
||||
using StellaOps.Concelier.Normalization.SemVer;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal static class RuNkckiMapper
|
||||
{
|
||||
private static readonly ImmutableDictionary<string, string> SeverityLookup = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["критический"] = "critical",
|
||||
["высокий"] = "high",
|
||||
["средний"] = "medium",
|
||||
["умеренный"] = "medium",
|
||||
["низкий"] = "low",
|
||||
["информационный"] = "informational",
|
||||
}.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static Advisory Map(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var advisoryProvenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AdvisoryKey,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, document, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss);
|
||||
var severityFromRating = NormalizeSeverity(dto.CvssRating);
|
||||
var severity = severityFromRating ?? severityFromCvss;
|
||||
|
||||
if (severityFromRating is not null && severityFromCvss is not null)
|
||||
{
|
||||
severity = ChooseMoreSevere(severityFromRating, severityFromCvss);
|
||||
}
|
||||
|
||||
var exploitKnown = DetermineExploitKnown(dto);
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.AdvisoryKey,
|
||||
title: dto.Description ?? dto.AdvisoryKey,
|
||||
summary: dto.Description,
|
||||
language: "ru",
|
||||
published: dto.DatePublished,
|
||||
modified: dto.DateUpdated,
|
||||
severity: severity,
|
||||
exploitKnown: exploitKnown,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: cvssMetrics,
|
||||
provenance: new[] { advisoryProvenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
aliases.Add(dto.FstecId!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
aliases.Add(dto.MitreId!);
|
||||
}
|
||||
|
||||
return aliases.ToImmutableSortedSet(StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddReference(string? url, string kind, string? sourceTag, string? summary)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var key = $"{kind}|{url}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References });
|
||||
|
||||
references.Add(new AdvisoryReference(url, kind, sourceTag, summary, provenance));
|
||||
}
|
||||
|
||||
AddReference(document.Uri, "details", "ru-nkcki", null);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
var slug = dto.FstecId!.Contains(':', StringComparison.Ordinal)
|
||||
? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
|
||||
: dto.FstecId;
|
||||
AddReference($"https://bdu.fstec.ru/vul/{slug}", "details", "bdu", null);
|
||||
}
|
||||
|
||||
foreach (var url in dto.Urls)
|
||||
{
|
||||
var kind = url.Contains("cert.gov.ru", StringComparison.OrdinalIgnoreCase) ? "details" : "external";
|
||||
var sourceTag = url.Contains("siemens", StringComparison.OrdinalIgnoreCase) ? "vendor" : null;
|
||||
AddReference(url, kind, sourceTag, null);
|
||||
}
|
||||
|
||||
if (dto.Cwe?.Number is int number)
|
||||
{
|
||||
AddReference(
|
||||
$"https://cwe.mitre.org/data/definitions/{number}.html",
|
||||
"cwe",
|
||||
"cwe",
|
||||
dto.Cwe.Description);
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (!dto.VulnerableSoftwareEntries.IsDefaultOrEmpty && dto.VulnerableSoftwareEntries.Length > 0)
|
||||
{
|
||||
return CreatePackages(dto.VulnerableSoftwareEntries, dto, recordedAt);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.VulnerableSoftwareText))
|
||||
{
|
||||
var fallbackEntry = new RuNkckiSoftwareEntry(
|
||||
dto.VulnerableSoftwareText!,
|
||||
dto.VulnerableSoftwareText!,
|
||||
ImmutableArray<string>.Empty);
|
||||
return CreatePackages(new[] { fallbackEntry }, dto, recordedAt);
|
||||
}
|
||||
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> CreatePackages(IEnumerable<RuNkckiSoftwareEntry> entries, RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var type = DeterminePackageType(dto);
|
||||
var platform = dto.ProductCategories.IsDefaultOrEmpty || dto.ProductCategories.Length == 0
|
||||
? null
|
||||
: string.Join(", ", dto.ProductCategories);
|
||||
|
||||
var packages = new List<AffectedPackage>();
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(entry.Identifier))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var packageProvenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"package",
|
||||
entry.Evidence,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var status = new AffectedPackageStatus(
|
||||
dto.PatchAvailable == true ? AffectedPackageStatusCatalog.Fixed : AffectedPackageStatusCatalog.Affected,
|
||||
new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"package-status",
|
||||
dto.PatchAvailable == true ? "patch_available" : "affected",
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.PackageStatuses }));
|
||||
|
||||
var rangeMetadata = BuildRangeMetadata(entry, recordedAt);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
type,
|
||||
entry.Identifier,
|
||||
platform,
|
||||
rangeMetadata.Ranges,
|
||||
new[] { status },
|
||||
new[] { packageProvenance },
|
||||
rangeMetadata.Normalized));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> BuildCvssMetrics(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity)
|
||||
{
|
||||
severity = null;
|
||||
var metrics = new List<CvssMetric>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize(null, dto.CvssVector, dto.CvssScore, null, out var normalized))
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
normalized.Vector,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics });
|
||||
var metric = normalized.ToModel(provenance);
|
||||
metrics.Add(metric);
|
||||
severity ??= metric.BaseSeverity;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.CvssVectorV4))
|
||||
{
|
||||
var vector = dto.CvssVectorV4.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)
|
||||
? dto.CvssVectorV4
|
||||
: $"CVSS:4.0/{dto.CvssVectorV4}";
|
||||
var score = dto.CvssScoreV4.HasValue
|
||||
? Math.Round(dto.CvssScoreV4.Value, 1, MidpointRounding.AwayFromZero)
|
||||
: 0.0;
|
||||
var severityV4 = DetermineCvss4Severity(score);
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
vector,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics });
|
||||
|
||||
metrics.Add(new CvssMetric("4.0", vector, score, severityV4, provenance));
|
||||
severity ??= severityV4;
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static string? NormalizeSeverity(string? rating)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rating))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = rating.Trim().ToLowerInvariant();
|
||||
|
||||
if (SeverityLookup.TryGetValue(normalized, out var mapped))
|
||||
{
|
||||
return mapped;
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("крит", StringComparison.Ordinal))
|
||||
{
|
||||
return "critical";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("высок", StringComparison.Ordinal))
|
||||
{
|
||||
return "high";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("сред", StringComparison.Ordinal) || normalized.StartsWith("умер", StringComparison.Ordinal))
|
||||
{
|
||||
return "medium";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("низк", StringComparison.Ordinal))
|
||||
{
|
||||
return "low";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("информ", StringComparison.Ordinal))
|
||||
{
|
||||
return "informational";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string ChooseMoreSevere(string first, string second)
|
||||
{
|
||||
var order = new[] { "critical", "high", "medium", "low", "informational" };
|
||||
|
||||
static int IndexOf(ReadOnlySpan<string> levels, string value)
|
||||
{
|
||||
for (var i = 0; i < levels.Length; i++)
|
||||
{
|
||||
if (string.Equals(levels[i], value, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
var firstIndex = IndexOf(order.AsSpan(), first);
|
||||
var secondIndex = IndexOf(order.AsSpan(), second);
|
||||
|
||||
if (firstIndex == -1 && secondIndex == -1)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
if (firstIndex == -1)
|
||||
{
|
||||
return second;
|
||||
}
|
||||
|
||||
if (secondIndex == -1)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
return firstIndex <= secondIndex ? first : second;
|
||||
}
|
||||
|
||||
private static bool DetermineExploitKnown(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(dto.MethodOfExploitation))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.Impact))
|
||||
{
|
||||
var impact = dto.Impact.Trim().ToUpperInvariant();
|
||||
if (impact is "ACE" or "RCE" or "LPE")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string DeterminePackageType(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
if (dto.VulnerableSoftwareHasCpe == true)
|
||||
{
|
||||
return AffectedPackageTypes.Cpe;
|
||||
}
|
||||
|
||||
if (!dto.ProductCategories.IsDefault && dto.ProductCategories.Any(static category =>
|
||||
category.Contains("ics", StringComparison.OrdinalIgnoreCase)
|
||||
|| category.Contains("scada", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return AffectedPackageTypes.IcsVendor;
|
||||
}
|
||||
|
||||
return AffectedPackageTypes.Vendor;
|
||||
}
|
||||
|
||||
private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) BuildRangeMetadata(
|
||||
RuNkckiSoftwareEntry entry,
|
||||
DateTimeOffset recordedAt)
|
||||
{
|
||||
if (entry.RangeExpressions.IsDefaultOrEmpty || entry.RangeExpressions.Length == 0)
|
||||
{
|
||||
return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>());
|
||||
}
|
||||
|
||||
var ranges = new List<AffectedVersionRange>();
|
||||
var normalized = new List<NormalizedVersionRule>();
|
||||
var dedupe = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var expression in entry.RangeExpressions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(expression))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var results = SemVerRangeRuleBuilder.Build(expression, provenanceNote: entry.Evidence);
|
||||
if (results.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var result in results)
|
||||
{
|
||||
var key = $"{result.Primitive.Introduced}|{result.Primitive.Fixed}|{result.Primitive.LastAffected}|{result.Expression}";
|
||||
if (!dedupe.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"package-range",
|
||||
entry.Evidence,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges });
|
||||
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
NormalizedVersionSchemes.SemVer,
|
||||
result.Primitive.Introduced,
|
||||
result.Primitive.Fixed,
|
||||
result.Primitive.LastAffected,
|
||||
result.Expression,
|
||||
provenance,
|
||||
new RangePrimitives(result.Primitive, null, null, null)));
|
||||
|
||||
normalized.Add(result.NormalizedRule);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
ranges.Count == 0 ? Array.Empty<AffectedVersionRange>() : ranges,
|
||||
normalized.Count == 0 ? Array.Empty<NormalizedVersionRule>() : normalized);
|
||||
}
|
||||
private static string DetermineCvss4Severity(double score)
|
||||
{
|
||||
if (score <= 0.0)
|
||||
{
|
||||
return "none";
|
||||
}
|
||||
|
||||
if (score < 4.0)
|
||||
{
|
||||
return "low";
|
||||
}
|
||||
|
||||
if (score < 7.0)
|
||||
{
|
||||
return "medium";
|
||||
}
|
||||
|
||||
if (score < 9.0)
|
||||
{
|
||||
return "high";
|
||||
}
|
||||
|
||||
return "critical";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiVulnerabilityDto(
|
||||
string? FstecId,
|
||||
string? MitreId,
|
||||
DateTimeOffset? DatePublished,
|
||||
DateTimeOffset? DateUpdated,
|
||||
string? CvssRating,
|
||||
bool? PatchAvailable,
|
||||
string? Description,
|
||||
RuNkckiCweDto? Cwe,
|
||||
ImmutableArray<string> ProductCategories,
|
||||
string? Mitigation,
|
||||
string? VulnerableSoftwareText,
|
||||
bool? VulnerableSoftwareHasCpe,
|
||||
ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
|
||||
double? CvssScore,
|
||||
string? CvssVector,
|
||||
double? CvssScoreV4,
|
||||
string? CvssVectorV4,
|
||||
string? Impact,
|
||||
string? MethodOfExploitation,
|
||||
bool? UserInteraction,
|
||||
ImmutableArray<string> Urls,
|
||||
ImmutableArray<string> Tags)
|
||||
{
|
||||
[JsonIgnore]
|
||||
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
|
||||
? FstecId!
|
||||
: !string.IsNullOrWhiteSpace(MitreId)
|
||||
? MitreId!
|
||||
: Guid.NewGuid().ToString();
|
||||
}
|
||||
|
||||
internal sealed record RuNkckiCweDto(int? Number, string? Description);
|
||||
|
||||
internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);
|
||||
43
src/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs
Normal file
43
src/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs
Normal file
@@ -0,0 +1,43 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
internal static class RuNkckiJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ru-nkcki:fetch";
|
||||
public const string Parse = "source:ru-nkcki:parse";
|
||||
public const string Map = "source:ru-nkcki:map";
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiFetchJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiFetchJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiParseJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiParseJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiMapJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiMapJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")]
|
||||
946
src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnector.cs
Normal file
946
src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnector.cs
Normal file
@@ -0,0 +1,946 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private static readonly string[] ListingAcceptHeaders =
|
||||
{
|
||||
"text/html",
|
||||
"application/xhtml+xml;q=0.9",
|
||||
"text/plain;q=0.1",
|
||||
};
|
||||
|
||||
private static readonly string[] BulletinAcceptHeaders =
|
||||
{
|
||||
"application/zip",
|
||||
"application/octet-stream",
|
||||
"application/x-zip-compressed",
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly RuNkckiOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly RuNkckiDiagnostics _diagnostics;
|
||||
private readonly ILogger<RuNkckiConnector> _logger;
|
||||
private readonly string _cacheDirectory;
|
||||
|
||||
private readonly HtmlParser _htmlParser = new();
|
||||
|
||||
public RuNkckiConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<RuNkckiOptions> options,
|
||||
RuNkckiDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<RuNkckiConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory);
|
||||
EnsureCacheDirectory();
|
||||
}
|
||||
|
||||
public string SourceName => RuNkckiConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var knownBulletins = cursor.KnownBulletins.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var processed = 0;
|
||||
|
||||
if (ShouldUseListingCache(cursor, now))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"NKCKI listing fetch skipped (cache duration {CacheDuration:c}); processing cached bulletins only",
|
||||
_options.ListingCacheDuration);
|
||||
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
ListingFetchSummary listingSummary;
|
||||
try
|
||||
{
|
||||
listingSummary = await LoadListingAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins");
|
||||
_diagnostics.ListingFetchFailure(ex.Message);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var uniqueAttachments = listingSummary.Attachments
|
||||
.GroupBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(static group => group.First())
|
||||
.OrderBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
var newAttachments = uniqueAttachments
|
||||
.Where(attachment => !knownBulletins.Contains(attachment.Id))
|
||||
.Take(_options.MaxBulletinsPerFetch)
|
||||
.ToList();
|
||||
|
||||
_diagnostics.ListingFetchSuccess(listingSummary.PagesVisited, uniqueAttachments.Count, newAttachments.Count);
|
||||
|
||||
if (newAttachments.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("NKCKI listing contained no new bulletin attachments");
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var downloaded = 0;
|
||||
var cachedUsed = 0;
|
||||
var failures = 0;
|
||||
|
||||
foreach (var attachment in newAttachments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, attachment.Uri)
|
||||
{
|
||||
AcceptHeaders = BulletinAcceptHeaders,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
var attachmentResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (!attachmentResult.IsSuccess || attachmentResult.Content is null)
|
||||
{
|
||||
if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
|
||||
{
|
||||
_diagnostics.BulletinFetchCached();
|
||||
cachedUsed++;
|
||||
_logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode);
|
||||
processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_diagnostics.BulletinFetchFailure(attachmentResult.StatusCode.ToString());
|
||||
failures++;
|
||||
_logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.BulletinFetchSuccess();
|
||||
downloaded++;
|
||||
TryWriteCachedBulletin(attachment.Id, attachmentResult.Content);
|
||||
processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
|
||||
{
|
||||
_diagnostics.BulletinFetchCached();
|
||||
cachedUsed++;
|
||||
_logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id);
|
||||
processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_diagnostics.BulletinFetchFailure(ex.Message);
|
||||
failures++;
|
||||
_logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (processed >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (processed < _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var normalizedBulletins = NormalizeBulletins(knownBulletins);
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(normalizedBulletins)
|
||||
.WithLastListingFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"NKCKI fetch complete: new bulletins {Downloaded}, cached bulletins {Cached}, failures {Failures}, processed entries {Processed}, pending documents {PendingDocuments}, pending mappings {PendingMappings}",
|
||||
downloaded,
|
||||
cachedUsed,
|
||||
failures,
|
||||
processed,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} missing GridFS payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI unable to download raw document {DocumentId}", documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
RuNkckiVulnerabilityDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(payload, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI failed to deserialize document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} produced null DTO", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow());
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} missing DTO payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
RuNkckiVulnerabilityDto dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI failed to deserialize DTO for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = RuNkckiMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI mapping failed for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<int> ProcessCachedBulletinsAsync(
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
HashSet<string> knownBulletins,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory))
|
||||
{
|
||||
return processed;
|
||||
}
|
||||
|
||||
var updated = processed;
|
||||
var cacheFiles = Directory
|
||||
.EnumerateFiles(_cacheDirectory, "*.json.zip", SearchOption.TopDirectoryOnly)
|
||||
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
foreach (var filePath in cacheFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var bulletinId = ExtractBulletinIdFromCachePath(filePath);
|
||||
if (string.IsNullOrWhiteSpace(bulletinId) || knownBulletins.Contains(bulletinId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] content;
|
||||
try
|
||||
{
|
||||
content = File.ReadAllBytes(filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to read cached bulletin at {CachePath}", filePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.BulletinFetchCached();
|
||||
updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(bulletinId);
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<int> ProcessBulletinEntriesAsync(
|
||||
byte[] content,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (content.Length == 0)
|
||||
{
|
||||
return processed;
|
||||
}
|
||||
|
||||
var updated = processed;
|
||||
using var archiveStream = new MemoryStream(content, writable: false);
|
||||
using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false);
|
||||
|
||||
foreach (var entry in archive.Entries.OrderBy(static e => e.FullName, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var entryStream = entry.Open();
|
||||
using var buffer = new MemoryStream();
|
||||
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (buffer.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer.Position = 0;
|
||||
|
||||
using var document = await JsonDocument.ParseAsync(buffer, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
updated = await ProcessBulletinJsonElementAsync(document.RootElement, entry.FullName, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var delta = updated - processed;
|
||||
if (delta > 0)
|
||||
{
|
||||
_diagnostics.EntriesProcessed(delta);
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<int> ProcessBulletinJsonElementAsync(
|
||||
JsonElement element,
|
||||
string entryName,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var updated = processed;
|
||||
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Array:
|
||||
foreach (var child in element.EnumerateArray())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (child.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await ProcessVulnerabilityObjectAsync(child, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case JsonValueKind.Object:
|
||||
if (await ProcessVulnerabilityObjectAsync(element, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
updated++;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<bool> ProcessVulnerabilityObjectAsync(
|
||||
JsonElement element,
|
||||
string entryName,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RuNkckiVulnerabilityDto dto;
|
||||
try
|
||||
{
|
||||
dto = RuNkckiJsonParser.Parse(element);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to parse vulnerability in bulletin {BulletinId} entry {Entry}", bulletinId, entryName);
|
||||
return false;
|
||||
}
|
||||
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions);
|
||||
var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
var documentUri = BuildDocumentUri(dto);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["ru-nkcki.bulletin"] = bulletinId,
|
||||
["ru-nkcki.entry"] = entryName,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
metadata["ru-nkcki.fstec_id"] = dto.FstecId!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
metadata["ru-nkcki.mitre_id"] = dto.MitreId!;
|
||||
}
|
||||
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
var lastModified = dto.DateUpdated ?? dto.DatePublished;
|
||||
var record = new DocumentRecord(
|
||||
recordId,
|
||||
SourceName,
|
||||
documentUri,
|
||||
now,
|
||||
sha,
|
||||
DocumentStatuses.PendingParse,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: lastModified,
|
||||
GridFsId: gridFsId,
|
||||
ExpiresAt: null);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
pendingMappings.Remove(upserted.Id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private Task<SourceFetchContentResult> FetchListingPageAsync(Uri pageUri, CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, pageUri)
|
||||
{
|
||||
AcceptHeaders = ListingAcceptHeaders,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
return _fetchService.FetchContentAsync(request, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<ListingPageResult> ParseListingAsync(Uri pageUri, byte[] content, CancellationToken cancellationToken)
|
||||
{
|
||||
var html = Encoding.UTF8.GetString(content);
|
||||
var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false);
|
||||
var attachments = new List<BulletinAttachment>();
|
||||
var pagination = new List<Uri>();
|
||||
|
||||
foreach (var anchor in document.QuerySelectorAll("a[href$='.json.zip']"))
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(pageUri, href, out var absoluteUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = DeriveBulletinId(absoluteUri);
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = anchor.GetAttribute("title");
|
||||
if (string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
title = anchor.TextContent?.Trim();
|
||||
}
|
||||
|
||||
attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id));
|
||||
}
|
||||
|
||||
foreach (var anchor in document.QuerySelectorAll("a[href]"))
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!href.Contains("PAGEN", StringComparison.OrdinalIgnoreCase)
|
||||
&& !href.Contains("page=", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(pageUri, href, out var absoluteUri))
|
||||
{
|
||||
pagination.Add(absoluteUri);
|
||||
}
|
||||
}
|
||||
|
||||
var uniquePagination = pagination
|
||||
.DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxListingPagesPerFetch)
|
||||
.ToList();
|
||||
|
||||
return new ListingPageResult(attachments, uniquePagination);
|
||||
}
|
||||
|
||||
private static string DeriveBulletinId(Uri uri)
|
||||
{
|
||||
var fileName = Path.GetFileName(uri.AbsolutePath);
|
||||
if (string.IsNullOrWhiteSpace(fileName))
|
||||
{
|
||||
return Guid.NewGuid().ToString("N");
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^4];
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^5];
|
||||
}
|
||||
|
||||
return fileName.Replace('_', '-');
|
||||
}
|
||||
|
||||
private static string BuildDocumentUri(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
var slug = dto.FstecId.Contains(':', StringComparison.Ordinal)
|
||||
? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
|
||||
: dto.FstecId;
|
||||
return $"https://cert.gov.ru/materialy/uyazvimosti/{slug}";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
return $"https://nvd.nist.gov/vuln/detail/{dto.MitreId}";
|
||||
}
|
||||
|
||||
return $"https://cert.gov.ru/materialy/uyazvimosti/{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
private string ResolveCacheDirectory(string? configuredPath)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(configuredPath))
|
||||
{
|
||||
return Path.GetFullPath(Path.IsPathRooted(configuredPath)
|
||||
? configuredPath
|
||||
: Path.Combine(AppContext.BaseDirectory, configuredPath));
|
||||
}
|
||||
|
||||
return Path.Combine(AppContext.BaseDirectory, "cache", RuNkckiConnectorPlugin.SourceName);
|
||||
}
|
||||
|
||||
private void EnsureCacheDirectory()
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_cacheDirectory);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI unable to ensure cache directory {CachePath}", _cacheDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetBulletinCachePath(string bulletinId)
|
||||
{
|
||||
var fileStem = string.IsNullOrWhiteSpace(bulletinId)
|
||||
? Guid.NewGuid().ToString("N")
|
||||
: Uri.EscapeDataString(bulletinId);
|
||||
return Path.Combine(_cacheDirectory, $"{fileStem}.json.zip");
|
||||
}
|
||||
|
||||
private static string ExtractBulletinIdFromCachePath(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var fileName = Path.GetFileName(path);
|
||||
if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^4];
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^5];
|
||||
}
|
||||
|
||||
return Uri.UnescapeDataString(fileName);
|
||||
}
|
||||
|
||||
private void TryWriteCachedBulletin(string bulletinId, byte[] content)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cachePath = GetBulletinCachePath(bulletinId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!);
|
||||
File.WriteAllBytes(cachePath, content);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to cache bulletin {BulletinId}", bulletinId);
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadCachedBulletin(string bulletinId, out byte[] content)
|
||||
{
|
||||
var cachePath = GetBulletinCachePath(bulletinId);
|
||||
try
|
||||
{
|
||||
if (File.Exists(cachePath))
|
||||
{
|
||||
content = File.ReadAllBytes(cachePath);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to read cached bulletin {BulletinId}", bulletinId);
|
||||
}
|
||||
|
||||
content = Array.Empty<byte>();
|
||||
return false;
|
||||
}
|
||||
|
||||
private IReadOnlyCollection<string> NormalizeBulletins(IEnumerable<string> bulletins)
|
||||
{
|
||||
var normalized = (bulletins ?? Enumerable.Empty<string>())
|
||||
.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (normalized.Count <= _options.KnownBulletinCapacity)
|
||||
{
|
||||
return normalized.ToArray();
|
||||
}
|
||||
|
||||
var skip = normalized.Count - _options.KnownBulletinCapacity;
|
||||
return normalized.Skip(skip).ToArray();
|
||||
}
|
||||
|
||||
private async Task<RuNkckiCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? RuNkckiCursor.Empty : RuNkckiCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
private readonly record struct ListingFetchSummary(IReadOnlyList<BulletinAttachment> Attachments, int PagesVisited);
|
||||
|
||||
private readonly record struct ListingPageResult(IReadOnlyList<BulletinAttachment> Attachments, IReadOnlyList<Uri> PaginationLinks);
|
||||
|
||||
private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title);
|
||||
|
||||
private bool ShouldUseListingCache(RuNkckiCursor cursor, DateTimeOffset now)
|
||||
{
|
||||
if (!cursor.LastListingFetchAt.HasValue)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var age = now - cursor.LastListingFetchAt.Value;
|
||||
return age < _options.ListingCacheDuration;
|
||||
}
|
||||
|
||||
private async Task<ListingFetchSummary> LoadListingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var attachments = new List<BulletinAttachment>();
|
||||
var visited = 0;
|
||||
var visitedUris = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var queue = new Queue<Uri>();
|
||||
queue.Enqueue(_options.ListingUri);
|
||||
|
||||
while (queue.Count > 0 && visited < _options.MaxListingPagesPerFetch)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var pageUri = queue.Dequeue();
|
||||
if (!visitedUris.Add(pageUri.AbsoluteUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.ListingFetchAttempt();
|
||||
|
||||
var listingResult = await FetchListingPageAsync(pageUri, cancellationToken).ConfigureAwait(false);
|
||||
if (!listingResult.IsSuccess || listingResult.Content is null)
|
||||
{
|
||||
_diagnostics.ListingFetchFailure(listingResult.StatusCode.ToString());
|
||||
_logger.LogWarning("NKCKI listing page {ListingUri} returned no content (status={Status})", pageUri, listingResult.StatusCode);
|
||||
continue;
|
||||
}
|
||||
|
||||
visited++;
|
||||
|
||||
var page = await ParseListingAsync(pageUri, listingResult.Content, cancellationToken).ConfigureAwait(false);
|
||||
attachments.AddRange(page.Attachments);
|
||||
|
||||
foreach (var link in page.PaginationLinks)
|
||||
{
|
||||
if (!visitedUris.Contains(link.AbsoluteUri) && queue.Count + visitedUris.Count < _options.MaxListingPagesPerFetch)
|
||||
{
|
||||
queue.Enqueue(link);
|
||||
}
|
||||
}
|
||||
|
||||
if (attachments.Count >= _options.MaxBulletinsPerFetch * 2)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return new ListingFetchSummary(attachments, visited);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ru-nkcki";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ru-nkcki";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddRuNkckiConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<RuNkckiFetchJob>();
|
||||
services.AddTransient<RuNkckiParseJob>();
|
||||
services.AddTransient<RuNkckiMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
|
||||
{
|
||||
if (schedulerOptions.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
schedulerOptions.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
schedulerOptions.DefaultTimeout,
|
||||
schedulerOptions.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public static class RuNkckiServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddRuNkckiConnector(this IServiceCollection services, Action<RuNkckiOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<RuNkckiOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(RuNkckiOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<RuNkckiOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseAddress;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = options.UserAgent;
|
||||
clientOptions.AllowAutoRedirect = true;
|
||||
clientOptions.DefaultRequestHeaders["Accept-Language"] = options.AcceptLanguage;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseAddress.Host);
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
|
||||
handler.AllowAutoRedirect = true;
|
||||
handler.UseCookies = true;
|
||||
handler.CookieContainer = new CookieContainer();
|
||||
};
|
||||
});
|
||||
|
||||
services.TryAddSingleton<RuNkckiDiagnostics>();
|
||||
services.AddTransient<RuNkckiConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AngleSharp" Version="1.1.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
11
src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md
Normal file
11
src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md
Normal file
@@ -0,0 +1,11 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/concelier-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.|
|
||||
|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DONE (2025-10-13)** – Listing fetch now honours `maxListingPagesPerFetch`, persists cache hits when listing access fails, and records telemetry via `RuNkckiDiagnostics`. Cursor tracking covers pending documents/mappings and the known bulletin ring buffer.|
|
||||
|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-13)** – Parser normalises nested arrays (ICS categories, vulnerable software lists, optional tags), flattens multiline `software_text`, and guarantees deterministic ordering for URLs and tags.|
|
||||
|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DONE (2025-10-13)** – Mapper splits structured software entries, emits SemVer range primitives + normalized rules, deduplicates references, and surfaces CVSS v4 metadata alongside existing metrics.|
|
||||
|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-13)** – Fixtures refreshed with multi-page pagination + multi-entry bulletins. Tests exercise cache replay and rely on bundled OpenSSL 1.1 libs in `tools/openssl/linux-x64` to keep Mongo2Go green on modern distros.|
|
||||
|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-13)** – Added connector-specific metrics (`nkcki.*`) and documented configuration/operational guidance in `docs/ops/concelier-nkcki-operations.md`.|
|
||||
|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**DONE (2025-10-13)** – Documented Bitrix pagination/backfill plan (cache-first, offline replay, HTML/PDF capture) in `docs/ops/concelier-nkcki-operations.md`.|
|
||||
|FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`concelier:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.|
|
||||
Reference in New Issue
Block a user