Add NKCKI severity smoothing, fixtures, and regression harness
This commit is contained in:
master
298
src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs
Normal file
298
src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs
Normal file
@@ -0,0 +1,298 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Normalization.Cvss;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
|
||||
|
||||
internal static class RuNkckiMapper
|
||||
{
|
||||
private static readonly ImmutableDictionary<string, string> SeverityLookup = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["критический"] = "critical",
|
||||
["высокий"] = "high",
|
||||
["средний"] = "medium",
|
||||
["умеренный"] = "medium",
|
||||
["низкий"] = "low",
|
||||
["информационный"] = "informational",
|
||||
}.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static Advisory Map(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var advisoryProvenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AdvisoryKey,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, document, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss);
|
||||
var severityFromRating = NormalizeSeverity(dto.CvssRating);
|
||||
var severity = severityFromRating ?? severityFromCvss;
|
||||
|
||||
if (severityFromRating is not null && severityFromCvss is not null)
|
||||
{
|
||||
severity = ChooseMoreSevere(severityFromRating, severityFromCvss);
|
||||
}
|
||||
|
||||
var exploitKnown = DetermineExploitKnown(dto);
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.AdvisoryKey,
|
||||
title: dto.Description ?? dto.AdvisoryKey,
|
||||
summary: dto.Description,
|
||||
language: "ru",
|
||||
published: dto.DatePublished,
|
||||
modified: dto.DateUpdated,
|
||||
severity: severity,
|
||||
exploitKnown: exploitKnown,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: cvssMetrics,
|
||||
provenance: new[] { advisoryProvenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
aliases.Add(dto.FstecId!);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
aliases.Add(dto.MitreId!);
|
||||
}
|
||||
|
||||
return aliases.ToImmutableSortedSet(StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(document.Uri, "details", "ru-nkcki", summary: null, new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
document.Uri,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
var slug = dto.FstecId!.Contains(':', StringComparison.Ordinal)
|
||||
? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
|
||||
: dto.FstecId;
|
||||
var bduUrl = $"https://bdu.fstec.ru/vul/{slug}";
|
||||
references.Add(new AdvisoryReference(bduUrl, "details", "bdu", summary: null, new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
bduUrl,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
foreach (var url in dto.Urls)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var kind = url.Contains("cert.gov.ru", StringComparison.OrdinalIgnoreCase) ? "details" : "external";
|
||||
var sourceTag = url.Contains("siemens", StringComparison.OrdinalIgnoreCase) ? "vendor" : null;
|
||||
references.Add(new AdvisoryReference(url, kind, sourceTag, summary: null, new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
if (dto.Cwe?.Number is int number)
|
||||
{
|
||||
var url = $"https://cwe.mitre.org/data/definitions/{number}.html";
|
||||
references.Add(new AdvisoryReference(url, "cwe", "cwe", dto.Cwe.Description, new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dto.VulnerableSoftwareText))
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var identifier = dto.VulnerableSoftwareText!.Replace('\n', ' ').Replace('\r', ' ').Trim();
|
||||
if (identifier.Length == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packageProvenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var status = new AffectedPackageStatus(
|
||||
dto.PatchAvailable == true ? AffectedPackageStatusCatalog.Fixed : AffectedPackageStatusCatalog.Affected,
|
||||
new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"package-status",
|
||||
dto.PatchAvailable == true ? "patch_available" : "affected",
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.PackageStatuses }));
|
||||
|
||||
return new[]
|
||||
{
|
||||
new AffectedPackage(
|
||||
dto.VulnerableSoftwareHasCpe == true ? AffectedPackageTypes.Cpe : AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: null,
|
||||
statuses: new[] { status },
|
||||
provenance: new[] { packageProvenance })
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> BuildCvssMetrics(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity)
|
||||
{
|
||||
severity = null;
|
||||
var metrics = new List<CvssMetric>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize(null, dto.CvssVector, dto.CvssScore, null, out var normalized))
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuNkckiConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
normalized.Vector,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics });
|
||||
var metric = normalized.ToModel(provenance);
|
||||
metrics.Add(metric);
|
||||
severity ??= metric.BaseSeverity;
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static string? NormalizeSeverity(string? rating)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rating))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = rating.Trim().ToLowerInvariant();
|
||||
|
||||
if (SeverityLookup.TryGetValue(normalized, out var mapped))
|
||||
{
|
||||
return mapped;
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("крит", StringComparison.Ordinal))
|
||||
{
|
||||
return "critical";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("высок", StringComparison.Ordinal))
|
||||
{
|
||||
return "high";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("сред", StringComparison.Ordinal) || normalized.StartsWith("умер", StringComparison.Ordinal))
|
||||
{
|
||||
return "medium";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("низк", StringComparison.Ordinal))
|
||||
{
|
||||
return "low";
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("информ", StringComparison.Ordinal))
|
||||
{
|
||||
return "informational";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string ChooseMoreSevere(string first, string second)
|
||||
{
|
||||
var order = new[] { "critical", "high", "medium", "low", "informational" };
|
||||
|
||||
static int IndexOf(ReadOnlySpan<string> levels, string value)
|
||||
{
|
||||
for (var i = 0; i < levels.Length; i++)
|
||||
{
|
||||
if (string.Equals(levels[i], value, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
var firstIndex = IndexOf(order.AsSpan(), first);
|
||||
var secondIndex = IndexOf(order.AsSpan(), second);
|
||||
|
||||
if (firstIndex == -1 && secondIndex == -1)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
if (firstIndex == -1)
|
||||
{
|
||||
return second;
|
||||
}
|
||||
|
||||
if (secondIndex == -1)
|
||||
{
|
||||
return first;
|
||||
}
|
||||
|
||||
return firstIndex <= secondIndex ? first : second;
|
||||
}
|
||||
|
||||
private static bool DetermineExploitKnown(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(dto.MethodOfExploitation))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.Impact))
|
||||
{
|
||||
var impact = dto.Impact.Trim().ToUpperInvariant();
|
||||
if (impact is "ACE" or "RCE" or "LPE")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,10 @@
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/feedser-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.|
|
||||
|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with custom trust store, optional SOCKS proxy, and Bitrix session bootstrap (`PHPSESSID`, `BITRIX_SM_GUEST_ID`). Persist raw XML/HTML + derived cursor (advisory ID + `pubDate`), handle 403 retries with exponential backoff.|
|
||||
|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**TODO** – Build DTOs for NKTsKI advisories, sanitise HTML, extract vendors/products, CVEs, mitigation guidance.|
|
||||
|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**TODO** – Map advisories into canonical records with aliases, references, and vendor range primitives. Coordinate normalized outputs and provenance per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: normalized payload target `[{"scheme":"semver","type":"range","min":"<start>","minInclusive":true,"max":"<end>","maxInclusive":false,"notes":"ru.nkcki:advisory-id"}]`; retain Cyrillic identifiers in `notes` so storage provenance remains intact.|
|
||||
|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests supporting `UPDATE_NKCKI_FIXTURES=1` for snapshot regeneration.|
|
||||
|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DOING (2025-10-12)** – Listing fetch now expands `*.json.zip` bulletins into per-vulnerability JSON documents with cursor-tracked bulletin IDs and trust store wiring (`globalsign_r6_bundle.pem`). Parser/mapper emit canonical advisories; remaining work: strengthen pagination/backfill handling and add regression fixtures/telemetry. Offline cache helpers (ProcessCachedBulletinsAsync/TryReadCachedBulletin/TryWriteCachedBulletin) implemented.|
|
||||
|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DOING (2025-10-12)** – `RuNkckiJsonParser` extracts per-vulnerability JSON payloads (IDs, CVEs, CVSS, software text, URLs). TODO: extend coverage for optional fields (ICS categories, nested arrays) and add fixture snapshots.|
|
||||
|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DOING (2025-10-12)** – `RuNkckiMapper` maps JSON entries to canonical advisories (aliases, references, vendor package, CVSS). Next steps: enrich package parsing (`software_text` tokenisation), consider CVSS v4 metadata, and backfill provenance docs before closing the task.|
|
||||
|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DOING (2025-10-12)** – Added mocked listing/bulletin regression harness (`RuNkckiConnectorTests`) with fixtures + snapshot writer. Test run currently blocked on Mongo2Go dependency (libcrypto.so.1.1 missing); follow-up required to get embedded mongod running in CI before marking DONE.|
|
||||
|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, and close backlog entry after deliverable ships.|
|
||||
|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**TODO** – Once access restored, map Bitrix paging (`?PAGEN_1=`) and advisory taxonomy (alerts vs recommendations). Outline HTML scrape + PDF attachment handling for backfill and decide translation approach for Russian-only content.|
|
||||
|FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`feedser:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.|
|
||||
|
||||
Reference in New Issue
Block a user