diff --git a/SPRINTS.md b/SPRINTS.md index c511c84e..e90e4e6c 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -69,8 +69,8 @@ | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cccs/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-007 | Atom feed verified 2025-10-11, history/caching review and FR locale enumeration pending. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.CertBund/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-007 | BSI RSS directory confirmed CERT-Bund feed 2025-10-11, history assessment pending. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kisa/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | KNVD RSS endpoint identified 2025-10-11, access headers/session strategy outstanding. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | BDU RSS/Atom catalogue reviewed 2025-10-11, trust-store acquisition blocked by gosuslugi placeholder page. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | cert.gov.ru paginated RSS landing checked 2025-10-11, access enablement plan pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | Build DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | TLS bundle + connectors landed 2025-10-12; fetch/parse/map flow emits advisories, fixtures & telemetry follow-up pending. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md | Build DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | JSON bulletin fetch + canonical mapping live 2025-10-12; regression fixtures added but blocked on Mongo2Go libcrypto dependency for test execution. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-008 | new ICS RSS endpoint logged 2025-10-11 but Akamai blocks direct pulls, fallback strategy task opened. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | openVuln API + RSS reviewed 2025-10-11, auth/pagination memo pending. | | Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md | Research DOING | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-007 | MSRC API docs reviewed 2025-10-11, auth/throttling comparison memo pending.
Instructions to work:
Read ./AGENTS.md plus each module's AGENTS file. Parallelize research, ingestion, mapping, fixtures, and docs using the normalized rule shape from ./src/FASTER_MODELING_AND_NORMALIZATION.md. Coordinate daily with the merge coordination task from Sprint 1. | diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip new file mode 100644 index 00000000..52ef47b2 Binary files /dev/null and b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip differ diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html new file mode 100644 index 00000000..56e04c36 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html @@ -0,0 +1,7 @@ + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json new file mode 100644 index 00000000..f382b68c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json @@ -0,0 +1,165 @@ +[ + { + "advisoryKey": "BDU:2025-01001", + "affectedPackages": [ + { + "type": "vendor", + "identifier": "SampleSCADA <= 4.2", + "platform": null, + "versionRanges": [], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "ru-nkcki", + "kind": "package-status", + "value": "patch_available", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "affectedpackages[].statuses[]" + ] + }, + "status": "fixed" + } + ], + "provenance": [ + { + "source": "ru-nkcki", + "kind": "package", + "value": "SampleSCADA <= 4.2", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "BDU:2025-01001", + "CVE-2025-0101" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 8.5, + "baseSeverity": "high", + "provenance": { + "source": "ru-nkcki", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:H/PR:L/UI:N/S:C/C:H/I:H/A:H", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:N/S:C/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": true, + "language": "ru", + "modified": "2025-09-22T00:00:00+00:00", + "provenance": [ + { + "source": "ru-nkcki", + "kind": "advisory", + "value": "BDU:2025-01001", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-09-20T00:00:00+00:00", + "references": [ + { + "kind": "details", + "provenance": { + "source": "ru-nkcki", + "kind": "reference", + "value": "https://bdu.fstec.ru/vul/2025-01001", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "bdu", + "summary": null, + "url": "https://bdu.fstec.ru/vul/2025-01001" + }, + { + "kind": "details", + "provenance": { + "source": "ru-nkcki", + "kind": "reference", + "value": "https://cert.gov.ru/materialy/uyazvimosti/2025-01001", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://cert.gov.ru/materialy/uyazvimosti/2025-01001" + }, + { + "kind": "details", + "provenance": { + "source": "ru-nkcki", + "kind": "reference", + "value": "https://cert.gov.ru/materialy/uyazvimosti/2025-01001", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "ru-nkcki", + "summary": null, + "url": "https://cert.gov.ru/materialy/uyazvimosti/2025-01001" + }, + { + "kind": "cwe", + "provenance": { + "source": "ru-nkcki", + "kind": "reference", + "value": "https://cwe.mitre.org/data/definitions/321.html", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "cwe", + "summary": "Use of Hard-coded Cryptographic Key", + "url": "https://cwe.mitre.org/data/definitions/321.html" + }, + { + "kind": "external", + "provenance": { + "source": "ru-nkcki", + "kind": "reference", + "value": "https://vendor.example/advisories/sample-scada", + "decisionReason": null, + "recordedAt": "2025-09-22T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example/advisories/sample-scada" + } + ], + "severity": "critical", + "summary": "Authenticated RCE in Sample SCADA", + "title": "Authenticated RCE in Sample SCADA" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs new file mode 100644 index 00000000..84f1c4f9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs @@ -0,0 +1,289 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Ru.Nkcki; +using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Models; +using MongoDB.Driver; +using Xunit; + +namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; + +[Collection("mongo-fixture")] +public sealed class RuNkckiConnectorTests : IAsyncLifetime +{ + private static readonly Uri ListingUri = new("https://cert.gov.ru/materialy/uyazvimosti/"); + private static readonly Uri BulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-sample.json.zip"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public RuNkckiConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesExpectedSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedListingAndBulletin(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + WriteOrAssertSnapshot(snapshot, "nkcki-advisories.snapshot.json"); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(RuNkckiConnectorPlugin.SourceName, "https://cert.gov.ru/materialy/uyazvimosti/BDU:2025-01001", CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(RuNkckiConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(IsEmptyArray(state!.Cursor, "pendingDocuments")); + Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); + } + + [Fact] + public async Task Fetch_ReusesCachedBulletinWhenListingFails() + { + await using var provider = await BuildServiceProviderAsync(); + SeedListingAndBulletin(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + _handler.Clear(); + _handler.AddResponse(ListingUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("error", Encoding.UTF8, "text/plain"), + }); + + var advisoryStore = provider.GetRequiredService(); + var before = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(before); + + await connector.FetchAsync(provider, CancellationToken.None); + + var after = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(before.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key), after.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key)); + + _handler.AssertNoPendingResponses(); + } + + private async Task BuildServiceProviderAsync() + { + try + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } + catch (MongoConnectionException ex) + { + Assert.Skip($"Mongo runner unavailable: {ex.Message}"); + } + catch (TimeoutException ex) + { + Assert.Skip($"Mongo runner unavailable: {ex.Message}"); + } + + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddRuNkckiConnector(options => + { + options.BaseAddress = new Uri("https://cert.gov.ru/"); + options.ListingPath = "/materialy/uyazvimosti/"; + options.MaxBulletinsPerFetch = 2; + options.MaxVulnerabilitiesPerFetch = 50; + var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName); + Directory.CreateDirectory(cacheRoot); + options.CacheDirectory = Path.Combine(cacheRoot, "ru-nkcki"); + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(RuNkckiOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + try + { + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + catch (MongoConnectionException ex) + { + Assert.Skip($"Mongo runner unavailable: {ex.Message}"); + throw; // Unreachable + } + catch (TimeoutException ex) + { + Assert.Skip($"Mongo runner unavailable: {ex.Message}"); + throw; + } + } + + private void SeedListingAndBulletin() + { + var listingHtml = ReadFixture("listing.html"); + _handler.AddTextResponse(ListingUri, listingHtml, "text/html"); + + var bulletinBytes = ReadBulletinFixture("bulletin-sample.json.zip"); + _handler.AddResponse(BulletinUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(bulletinBytes), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 9, 22, 0, 0, 0, TimeSpan.Zero); + return response; + }); + } + + private static bool IsEmptyArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return false; + } + + return array.Count == 0; + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine("Fixtures", filename); + var resolved = ResolveFixturePath(path); + return File.ReadAllText(resolved); + } + + private static byte[] ReadBulletinFixture(string filename) + { + var path = Path.Combine("Fixtures", filename); + var resolved = ResolveFixturePath(path); + return File.ReadAllBytes(resolved); + } + + private static string ResolveFixturePath(string relativePath) + { + var projectRoot = GetProjectRoot(); + var projectPath = Path.Combine(projectRoot, relativePath); + if (File.Exists(projectPath)) + { + return projectPath; + } + + var binaryPath = Path.Combine(AppContext.BaseDirectory, relativePath); + if (File.Exists(binaryPath)) + { + return Path.GetFullPath(binaryPath); + } + + throw new FileNotFoundException($"Fixture not found: {relativePath}"); + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + if (ShouldUpdateFixtures()) + { + var path = GetWritableFixturePath(filename); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, snapshot); + return; + } + + var expectedPath = ResolveFixturePath(Path.Combine("Fixtures", filename)); + if (!File.Exists(expectedPath)) + { + throw new FileNotFoundException($"Expected snapshot missing: {expectedPath}. Set UPDATE_NKCKI_FIXTURES=1 to generate."); + } + + var expected = File.ReadAllText(expectedPath); + Assert.Equal(Normalize(expected), Normalize(snapshot)); + } + + private static string GetWritableFixturePath(string filename) + { + var projectRoot = GetProjectRoot(); + return Path.Combine(projectRoot, "Fixtures", filename); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable("UPDATE_NKCKI_FIXTURES"); + return string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string Normalize(string text) + => text.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string GetProjectRoot() + { + var current = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(current)) + { + var candidate = Path.Combine(current, "StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj"); + if (File.Exists(candidate)) + { + return current; + } + + current = Path.GetDirectoryName(current); + } + + throw new InvalidOperationException("Unable to locate project root for Ru.Nkcki tests."); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + => await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs new file mode 100644 index 00000000..cd74a046 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs @@ -0,0 +1,68 @@ +using System.Collections.Immutable; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Ru.Nkcki.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using Xunit; +using System.Reflection; + +namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; + +public sealed class RuNkckiMapperTests +{ + [Fact] + public void Map_ConstructsCanonicalAdvisory() + { + var dto = new RuNkckiVulnerabilityDto( + FstecId: "BDU:2025-00001", + MitreId: "CVE-2025-0001", + DatePublished: new DateTimeOffset(2025, 9, 1, 0, 0, 0, TimeSpan.Zero), + DateUpdated: new DateTimeOffset(2025, 9, 2, 0, 0, 0, TimeSpan.Zero), + CvssRating: "КРИТИЧЕСКИЙ", + PatchAvailable: true, + Description: "Test NKCKI vulnerability", + Cwe: new RuNkckiCweDto(79, "Cross-site scripting"), + ProductCategory: "Web", + Mitigation: "Apply update", + VulnerableSoftwareText: "ExampleApp <= 1.0", + VulnerableSoftwareHasCpe: false, + CvssScore: 8.8, + CvssVector: "AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + CvssScoreV4: null, + CvssVectorV4: null, + Impact: "ACE", + MethodOfExploitation: "Special request", + UserInteraction: false, + Urls: ImmutableArray.Create("https://example.com/advisory")); + + var document = new DocumentRecord( + Guid.NewGuid(), + RuNkckiConnectorPlugin.SourceName, + "https://cert.gov.ru/materialy/uyazvimosti/2025-00001", + DateTimeOffset.UtcNow, + "abc", + DocumentStatuses.PendingMap, + "application/json", + null, + null, + null, + dto.DateUpdated, + ObjectId.GenerateNewId()); + + Assert.Equal("КРИТИЧЕСКИЙ", dto.CvssRating); + var normalizeSeverity = typeof(RuNkckiMapper).GetMethod("NormalizeSeverity", BindingFlags.NonPublic | BindingFlags.Static)!; + var ratingSeverity = (string?)normalizeSeverity.Invoke(null, new object?[] { dto.CvssRating }); + Assert.Equal("critical", ratingSeverity); + + var advisory = RuNkckiMapper.Map(dto, document, dto.DateUpdated!.Value); + + Assert.Contains("BDU:2025-00001", advisory.Aliases); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Equal("critical", advisory.Severity); + Assert.True(advisory.ExploitKnown); + Assert.Single(advisory.AffectedPackages); + Assert.Single(advisory.CvssMetrics); + Assert.Contains(advisory.References, reference => reference.Url.Contains("example.com", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs new file mode 100644 index 00000000..ad348ca9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs @@ -0,0 +1,298 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; + +internal static class RuNkckiMapper +{ + private static readonly ImmutableDictionary SeverityLookup = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["критический"] = "critical", + ["высокий"] = "high", + ["средний"] = "medium", + ["умеренный"] = "medium", + ["низкий"] = "low", + ["информационный"] = "informational", + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + public static Advisory Map(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var advisoryProvenance = new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "advisory", + dto.AdvisoryKey, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, document, recordedAt); + var packages = BuildPackages(dto, recordedAt); + var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss); + var severityFromRating = NormalizeSeverity(dto.CvssRating); + var severity = severityFromRating ?? severityFromCvss; + + if (severityFromRating is not null && severityFromCvss is not null) + { + severity = ChooseMoreSevere(severityFromRating, severityFromCvss); + } + + var exploitKnown = DetermineExploitKnown(dto); + + return new Advisory( + advisoryKey: dto.AdvisoryKey, + title: dto.Description ?? dto.AdvisoryKey, + summary: dto.Description, + language: "ru", + published: dto.DatePublished, + modified: dto.DateUpdated, + severity: severity, + exploitKnown: exploitKnown, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: cvssMetrics, + provenance: new[] { advisoryProvenance }); + } + + private static IReadOnlyList BuildAliases(RuNkckiVulnerabilityDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(dto.FstecId)) + { + aliases.Add(dto.FstecId!); + } + + if (!string.IsNullOrWhiteSpace(dto.MitreId)) + { + aliases.Add(dto.MitreId!); + } + + return aliases.ToImmutableSortedSet(StringComparer.Ordinal).ToImmutableArray(); + } + + private static IReadOnlyList BuildReferences(RuNkckiVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + var references = new List + { + new(document.Uri, "details", "ru-nkcki", summary: null, new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "reference", + document.Uri, + recordedAt, + new[] { ProvenanceFieldMasks.References })) + }; + + if (!string.IsNullOrWhiteSpace(dto.FstecId)) + { + var slug = dto.FstecId!.Contains(':', StringComparison.Ordinal) + ? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..] + : dto.FstecId; + var bduUrl = $"https://bdu.fstec.ru/vul/{slug}"; + references.Add(new AdvisoryReference(bduUrl, "details", "bdu", summary: null, new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "reference", + bduUrl, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + foreach (var url in dto.Urls) + { + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + var kind = url.Contains("cert.gov.ru", StringComparison.OrdinalIgnoreCase) ? "details" : "external"; + var sourceTag = url.Contains("siemens", StringComparison.OrdinalIgnoreCase) ? "vendor" : null; + references.Add(new AdvisoryReference(url, kind, sourceTag, summary: null, new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "reference", + url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + if (dto.Cwe?.Number is int number) + { + var url = $"https://cwe.mitre.org/data/definitions/{number}.html"; + references.Add(new AdvisoryReference(url, "cwe", "cwe", dto.Cwe.Description, new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "reference", + url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + return references; + } + + private static IReadOnlyList BuildPackages(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(dto.VulnerableSoftwareText)) + { + return Array.Empty(); + } + + var identifier = dto.VulnerableSoftwareText!.Replace('\n', ' ').Replace('\r', ' ').Trim(); + if (identifier.Length == 0) + { + return Array.Empty(); + } + + var packageProvenance = new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + var status = new AffectedPackageStatus( + dto.PatchAvailable == true ? AffectedPackageStatusCatalog.Fixed : AffectedPackageStatusCatalog.Affected, + new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "package-status", + dto.PatchAvailable == true ? "patch_available" : "affected", + recordedAt, + new[] { ProvenanceFieldMasks.PackageStatuses })); + + return new[] + { + new AffectedPackage( + dto.VulnerableSoftwareHasCpe == true ? AffectedPackageTypes.Cpe : AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: null, + statuses: new[] { status }, + provenance: new[] { packageProvenance }) + }; + } + + private static IReadOnlyList BuildCvssMetrics(RuNkckiVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + var metrics = new List(); + + if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize(null, dto.CvssVector, dto.CvssScore, null, out var normalized)) + { + var provenance = new AdvisoryProvenance( + RuNkckiConnectorPlugin.SourceName, + "cvss", + normalized.Vector, + recordedAt, + new[] { ProvenanceFieldMasks.CvssMetrics }); + var metric = normalized.ToModel(provenance); + metrics.Add(metric); + severity ??= metric.BaseSeverity; + } + + return metrics; + } + + private static string? NormalizeSeverity(string? rating) + { + if (string.IsNullOrWhiteSpace(rating)) + { + return null; + } + + var normalized = rating.Trim().ToLowerInvariant(); + + if (SeverityLookup.TryGetValue(normalized, out var mapped)) + { + return mapped; + } + + if (normalized.StartsWith("крит", StringComparison.Ordinal)) + { + return "critical"; + } + + if (normalized.StartsWith("высок", StringComparison.Ordinal)) + { + return "high"; + } + + if (normalized.StartsWith("сред", StringComparison.Ordinal) || normalized.StartsWith("умер", StringComparison.Ordinal)) + { + return "medium"; + } + + if (normalized.StartsWith("низк", StringComparison.Ordinal)) + { + return "low"; + } + + if (normalized.StartsWith("информ", StringComparison.Ordinal)) + { + return "informational"; + } + + return null; + } + + private static string ChooseMoreSevere(string first, string second) + { + var order = new[] { "critical", "high", "medium", "low", "informational" }; + + static int IndexOf(ReadOnlySpan levels, string value) + { + for (var i = 0; i < levels.Length; i++) + { + if (string.Equals(levels[i], value, StringComparison.OrdinalIgnoreCase)) + { + return i; + } + } + + return -1; + } + + var firstIndex = IndexOf(order.AsSpan(), first); + var secondIndex = IndexOf(order.AsSpan(), second); + + if (firstIndex == -1 && secondIndex == -1) + { + return first; + } + + if (firstIndex == -1) + { + return second; + } + + if (secondIndex == -1) + { + return first; + } + + return firstIndex <= secondIndex ? first : second; + } + + private static bool DetermineExploitKnown(RuNkckiVulnerabilityDto dto) + { + if (!string.IsNullOrWhiteSpace(dto.MethodOfExploitation)) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(dto.Impact)) + { + var impact = dto.Impact.Trim().ToUpperInvariant(); + if (impact is "ACE" or "RCE" or "LPE") + { + return true; + } + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md b/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md index 937f0637..739974a4 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md @@ -2,10 +2,10 @@ | Task | Owner(s) | Depends on | Notes | |---|---|---|---| |FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/feedser-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.| -|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**TODO** – Implement fetch job with custom trust store, optional SOCKS proxy, and Bitrix session bootstrap (`PHPSESSID`, `BITRIX_SM_GUEST_ID`). Persist raw XML/HTML + derived cursor (advisory ID + `pubDate`), handle 403 retries with exponential backoff.| -|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**TODO** – Build DTOs for NKTsKI advisories, sanitise HTML, extract vendors/products, CVEs, mitigation guidance.| -|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**TODO** – Map advisories into canonical records with aliases, references, and vendor range primitives. Coordinate normalized outputs and provenance per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.
2025-10-11 research trail: normalized payload target `[{"scheme":"semver","type":"range","min":"","minInclusive":true,"max":"","maxInclusive":false,"notes":"ru.nkcki:advisory-id"}]`; retain Cyrillic identifiers in `notes` so storage provenance remains intact.| -|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests supporting `UPDATE_NKCKI_FIXTURES=1` for snapshot regeneration.| +|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DOING (2025-10-12)** – Listing fetch now expands `*.json.zip` bulletins into per-vulnerability JSON documents with cursor-tracked bulletin IDs and trust store wiring (`globalsign_r6_bundle.pem`). Parser/mapper emit canonical advisories; remaining work: strengthen pagination/backfill handling and add regression fixtures/telemetry. Offline cache helpers (ProcessCachedBulletinsAsync/TryReadCachedBulletin/TryWriteCachedBulletin) implemented.| +|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DOING (2025-10-12)** – `RuNkckiJsonParser` extracts per-vulnerability JSON payloads (IDs, CVEs, CVSS, software text, URLs). TODO: extend coverage for optional fields (ICS categories, nested arrays) and add fixture snapshots.| +|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DOING (2025-10-12)** – `RuNkckiMapper` maps JSON entries to canonical advisories (aliases, references, vendor package, CVSS). Next steps: enrich package parsing (`software_text` tokenisation), consider CVSS v4 metadata, and backfill provenance docs before closing the task.| +|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DOING (2025-10-12)** – Added mocked listing/bulletin regression harness (`RuNkckiConnectorTests`) with fixtures + snapshot writer. Test run currently blocked on Mongo2Go dependency (libcrypto.so.1.1 missing); follow-up required to get embedded mongod running in CI before marking DONE.| |FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**TODO** – Add logging/metrics, document connector configuration, and close backlog entry after deliverable ships.| |FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**TODO** – Once access restored, map Bitrix paging (`?PAGEN_1=`) and advisory taxonomy (alerts vs recommendations). Outline HTML scrape + PDF attachment handling for backfill and decide translation approach for Russian-only content.| |FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`feedser:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.|