tam
This commit is contained in:
@@ -1,29 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public string Name => "ru-nkcki";
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => true;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
|
||||
|
||||
private sealed class StubConnector : IFeedConnector
|
||||
{
|
||||
public StubConnector(string sourceName) => SourceName = sourceName;
|
||||
|
||||
public string SourceName { get; }
|
||||
|
||||
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
|
||||
/// </summary>
|
||||
public sealed class RuNkckiOptions
|
||||
{
|
||||
public const string HttpClientName = "ru-nkcki";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
|
||||
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint used for resolving relative resource links.
|
||||
/// </summary>
|
||||
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the bulletin listing page.
|
||||
/// </summary>
|
||||
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to listing and bulletin fetch requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the listing or attachments cannot be retrieved.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of bulletin attachments downloaded per fetch run.
|
||||
/// </summary>
|
||||
public int MaxBulletinsPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
|
||||
/// </summary>
|
||||
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
|
||||
/// </summary>
|
||||
public int KnownBulletinCapacity { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between sequential bulletin downloads.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Duration the HTML listing can be cached before forcing a refetch.
|
||||
/// </summary>
|
||||
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
|
||||
|
||||
public string UserAgent { get; set; } = "StellaOps/Feedser (+https://stella-ops.org)";
|
||||
|
||||
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
|
||||
|
||||
/// <summary>
|
||||
/// Absolute URI for the listing page.
|
||||
/// </summary>
|
||||
public Uri ListingUri => new(BaseAddress, ListingPath);
|
||||
|
||||
/// <summary>
|
||||
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; } = null;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ListingPath))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxBulletinsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxVulnerabilitiesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (KnownBulletinCapacity <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
|
||||
}
|
||||
|
||||
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AcceptLanguage))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
108
src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiCursor.cs
Normal file
108
src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiCursor.cs
Normal file
@@ -0,0 +1,108 @@
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownBulletins,
|
||||
DateTimeOffset? LastListingFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
|
||||
|
||||
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
|
||||
|
||||
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
|
||||
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
|
||||
|
||||
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastListingFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownBulletins"] = new BsonArray(KnownBulletins),
|
||||
};
|
||||
|
||||
if (LastListingFetchAt.HasValue)
|
||||
{
|
||||
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static RuNkckiCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownBulletins = ReadStringArray(document, "knownBulletins");
|
||||
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyBulletins;
|
||||
}
|
||||
|
||||
var result = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
var text = element?.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
result.Add(text);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
|
||||
|
||||
internal static class RuNkckiJsonParser
|
||||
{
|
||||
public static RuNkckiVulnerabilityDto Parse(JsonElement element)
|
||||
{
|
||||
var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec) ? Normalize(fstec.GetString()) : null;
|
||||
var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre) ? Normalize(mitre.GetString()) : null;
|
||||
|
||||
var datePublished = ParseDate(element.TryGetProperty("date_published", out var published) ? published.GetString() : null);
|
||||
var dateUpdated = ParseDate(element.TryGetProperty("date_updated", out var updated) ? updated.GetString() : null);
|
||||
var cvssRating = Normalize(element.TryGetProperty("cvss_rating", out var rating) ? rating.GetString() : null);
|
||||
bool? patchAvailable = element.TryGetProperty("patch_available", out var patch) ? patch.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => null,
|
||||
} : null;
|
||||
|
||||
var description = Normalize(element.TryGetProperty("description", out var desc) ? desc.GetString() : null);
|
||||
var mitigation = Normalize(element.TryGetProperty("mitigation", out var mitigationElement) ? mitigationElement.GetString() : null);
|
||||
var productCategory = Normalize(element.TryGetProperty("product_category", out var category) ? category.GetString() : null);
|
||||
var impact = Normalize(element.TryGetProperty("impact", out var impactElement) ? impactElement.GetString() : null);
|
||||
var method = Normalize(element.TryGetProperty("method_of_exploitation", out var methodElement) ? methodElement.GetString() : null);
|
||||
|
||||
bool? userInteraction = element.TryGetProperty("user_interaction", out var uiElement) ? uiElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => null,
|
||||
} : null;
|
||||
|
||||
string? softwareText = null;
|
||||
bool? softwareHasCpe = null;
|
||||
if (element.TryGetProperty("vulnerable_software", out var softwareElement))
|
||||
{
|
||||
if (softwareElement.TryGetProperty("software_text", out var textElement))
|
||||
{
|
||||
softwareText = Normalize(textElement.GetString()?.Replace('\r', ' '));
|
||||
}
|
||||
|
||||
if (softwareElement.TryGetProperty("cpe", out var cpeElement))
|
||||
{
|
||||
softwareHasCpe = cpeElement.ValueKind switch
|
||||
{
|
||||
JsonValueKind.True => true,
|
||||
JsonValueKind.False => false,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
RuNkckiCweDto? cweDto = null;
|
||||
if (element.TryGetProperty("cwe", out var cweElement))
|
||||
{
|
||||
int? number = null;
|
||||
if (cweElement.TryGetProperty("cwe_number", out var numberElement))
|
||||
{
|
||||
if (numberElement.ValueKind == JsonValueKind.Number && numberElement.TryGetInt32(out var parsed))
|
||||
{
|
||||
number = parsed;
|
||||
}
|
||||
else if (int.TryParse(numberElement.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedInt))
|
||||
{
|
||||
number = parsedInt;
|
||||
}
|
||||
}
|
||||
|
||||
var cweDescription = Normalize(cweElement.TryGetProperty("cwe_description", out var descElement) ? descElement.GetString() : null);
|
||||
if (number.HasValue || !string.IsNullOrWhiteSpace(cweDescription))
|
||||
{
|
||||
cweDto = new RuNkckiCweDto(number, cweDescription);
|
||||
}
|
||||
}
|
||||
|
||||
double? cvssScore = element.TryGetProperty("cvss", out var cvssElement) && cvssElement.TryGetProperty("cvss_score", out var scoreElement)
|
||||
? ParseDouble(scoreElement)
|
||||
: null;
|
||||
var cvssVector = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector", out var vectorElement)
|
||||
? Normalize(vectorElement.GetString())
|
||||
: null;
|
||||
double? cvssScoreV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_score_v4", out var scoreV4Element)
|
||||
? ParseDouble(scoreV4Element)
|
||||
: null;
|
||||
var cvssVectorV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector_v4", out var vectorV4Element)
|
||||
? Normalize(vectorV4Element.GetString())
|
||||
: null;
|
||||
|
||||
var urls = element.TryGetProperty("urls", out var urlsElement) && urlsElement.ValueKind == JsonValueKind.Array
|
||||
? urlsElement.EnumerateArray()
|
||||
.Select(static url => Normalize(url.GetString()))
|
||||
.Where(static url => !string.IsNullOrWhiteSpace(url))
|
||||
.Cast<string>()
|
||||
.ToImmutableArray()
|
||||
: ImmutableArray<string>.Empty;
|
||||
|
||||
return new RuNkckiVulnerabilityDto(
|
||||
fstecId,
|
||||
mitreId,
|
||||
datePublished,
|
||||
dateUpdated,
|
||||
cvssRating,
|
||||
patchAvailable,
|
||||
description,
|
||||
cweDto,
|
||||
productCategory,
|
||||
mitigation,
|
||||
softwareText,
|
||||
softwareHasCpe,
|
||||
cvssScore,
|
||||
cvssVector,
|
||||
cvssScoreV4,
|
||||
cvssVectorV4,
|
||||
impact,
|
||||
method,
|
||||
userInteraction,
|
||||
urls);
|
||||
}
|
||||
|
||||
private static double? ParseDouble(JsonElement element)
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
if (element.ValueKind == JsonValueKind.String && double.TryParse(element.GetString(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var ruParsed))
|
||||
{
|
||||
return ruParsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.Replace('\r', ' ').Replace('\n', ' ').Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiVulnerabilityDto(
|
||||
string? FstecId,
|
||||
string? MitreId,
|
||||
DateTimeOffset? DatePublished,
|
||||
DateTimeOffset? DateUpdated,
|
||||
string? CvssRating,
|
||||
bool? PatchAvailable,
|
||||
string? Description,
|
||||
RuNkckiCweDto? Cwe,
|
||||
string? ProductCategory,
|
||||
string? Mitigation,
|
||||
string? VulnerableSoftwareText,
|
||||
bool? VulnerableSoftwareHasCpe,
|
||||
double? CvssScore,
|
||||
string? CvssVector,
|
||||
double? CvssScoreV4,
|
||||
string? CvssVectorV4,
|
||||
string? Impact,
|
||||
string? MethodOfExploitation,
|
||||
bool? UserInteraction,
|
||||
ImmutableArray<string> Urls)
|
||||
{
|
||||
[JsonIgnore]
|
||||
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
|
||||
? FstecId!
|
||||
: !string.IsNullOrWhiteSpace(MitreId)
|
||||
? MitreId!
|
||||
: Guid.NewGuid().ToString();
|
||||
}
|
||||
|
||||
internal sealed record RuNkckiCweDto(int? Number, string? Description);
|
||||
43
src/StellaOps.Feedser.Source.Ru.Nkcki/Jobs.cs
Normal file
43
src/StellaOps.Feedser.Source.Ru.Nkcki/Jobs.cs
Normal file
@@ -0,0 +1,43 @@
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
internal static class RuNkckiJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ru-nkcki:fetch";
|
||||
public const string Parse = "source:ru-nkcki:parse";
|
||||
public const string Map = "source:ru-nkcki:map";
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiFetchJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiFetchJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiParseJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiParseJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiMapJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiMapJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Ru.Nkcki.Tests")]
|
||||
825
src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs
Normal file
825
src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs
Normal file
@@ -0,0 +1,825 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
|
||||
using StellaOps.Feedser.Source.Ru.Nkcki.Internal;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private static readonly string[] ListingAcceptHeaders =
|
||||
{
|
||||
"text/html",
|
||||
"application/xhtml+xml;q=0.9",
|
||||
"text/plain;q=0.1",
|
||||
};
|
||||
|
||||
private static readonly string[] BulletinAcceptHeaders =
|
||||
{
|
||||
"application/zip",
|
||||
"application/octet-stream",
|
||||
"application/x-zip-compressed",
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly RuNkckiOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<RuNkckiConnector> _logger;
|
||||
private readonly string _cacheDirectory;
|
||||
|
||||
private readonly HtmlParser _htmlParser = new();
|
||||
|
||||
public RuNkckiConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<RuNkckiOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<RuNkckiConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory);
|
||||
EnsureCacheDirectory();
|
||||
}
|
||||
|
||||
public string SourceName => RuNkckiConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var knownBulletins = cursor.KnownBulletins.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var processed = 0;
|
||||
|
||||
IReadOnlyList<BulletinAttachment> attachments = Array.Empty<BulletinAttachment>();
|
||||
|
||||
try
|
||||
{
|
||||
var listingResult = await FetchListingAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!listingResult.IsSuccess || listingResult.Content is null)
|
||||
{
|
||||
_logger.LogWarning("NKCKI listing fetch returned no content (status={Status})", listingResult.StatusCode);
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
attachments = await ParseListingAsync(listingResult.Content, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins");
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (attachments.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("NKCKI listing contained no bulletin attachments");
|
||||
processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var newAttachments = attachments
|
||||
.Where(attachment => !knownBulletins.Contains(attachment.Id))
|
||||
.Take(_options.MaxBulletinsPerFetch)
|
||||
.ToList();
|
||||
|
||||
if (newAttachments.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(NormalizeBulletins(knownBulletins))
|
||||
.WithLastListingFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var attachment in newAttachments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, attachment.Uri)
|
||||
{
|
||||
AcceptHeaders = BulletinAcceptHeaders,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
var attachmentResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (!attachmentResult.IsSuccess || attachmentResult.Content is null)
|
||||
{
|
||||
if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
|
||||
{
|
||||
_logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode);
|
||||
processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
TryWriteCachedBulletin(attachment.Id, attachmentResult.Content);
|
||||
processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
if (TryReadCachedBulletin(attachment.Id, out var cachedBytes))
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id);
|
||||
processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(attachment.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (processed >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var normalizedBulletins = NormalizeBulletins(knownBulletins);
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownBulletins(normalizedBulletins)
|
||||
.WithLastListingFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} missing GridFS payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI unable to download raw document {DocumentId}", documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
RuNkckiVulnerabilityDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(payload, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI failed to deserialize document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} produced null DTO", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow());
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_logger.LogWarning("NKCKI document {DocumentId} missing DTO payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
RuNkckiVulnerabilityDto dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuNkckiVulnerabilityDto>(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI failed to deserialize DTO for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = RuNkckiMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI mapping failed for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<int> ProcessCachedBulletinsAsync(
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
HashSet<string> knownBulletins,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Directory.Exists(_cacheDirectory))
|
||||
{
|
||||
return processed;
|
||||
}
|
||||
|
||||
var updated = processed;
|
||||
var cacheFiles = Directory
|
||||
.EnumerateFiles(_cacheDirectory, "*.json.zip", SearchOption.TopDirectoryOnly)
|
||||
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
foreach (var filePath in cacheFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var bulletinId = ExtractBulletinIdFromCachePath(filePath);
|
||||
if (string.IsNullOrWhiteSpace(bulletinId) || knownBulletins.Contains(bulletinId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] content;
|
||||
try
|
||||
{
|
||||
content = File.ReadAllBytes(filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to read cached bulletin at {CachePath}", filePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
|
||||
knownBulletins.Add(bulletinId);
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<int> ProcessBulletinEntriesAsync(
|
||||
byte[] content,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (content.Length == 0)
|
||||
{
|
||||
return processed;
|
||||
}
|
||||
|
||||
var updated = processed;
|
||||
using var archiveStream = new MemoryStream(content, writable: false);
|
||||
using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false);
|
||||
|
||||
foreach (var entry in archive.Entries.OrderBy(static e => e.FullName, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var entryStream = entry.Open();
|
||||
using var buffer = new MemoryStream();
|
||||
await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (buffer.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer.Position = 0;
|
||||
|
||||
using var document = await JsonDocument.ParseAsync(buffer, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
updated = await ProcessBulletinJsonElementAsync(document.RootElement, entry.FullName, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<int> ProcessBulletinJsonElementAsync(
|
||||
JsonElement element,
|
||||
string entryName,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
int processed,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var updated = processed;
|
||||
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Array:
|
||||
foreach (var child in element.EnumerateArray())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (updated >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (child.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await ProcessVulnerabilityObjectAsync(child, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case JsonValueKind.Object:
|
||||
if (await ProcessVulnerabilityObjectAsync(element, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
updated++;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async Task<bool> ProcessVulnerabilityObjectAsync(
|
||||
JsonElement element,
|
||||
string entryName,
|
||||
string bulletinId,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
RuNkckiVulnerabilityDto dto;
|
||||
try
|
||||
{
|
||||
dto = RuNkckiJsonParser.Parse(element);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to parse vulnerability in bulletin {BulletinId} entry {Entry}", bulletinId, entryName);
|
||||
return false;
|
||||
}
|
||||
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions);
|
||||
var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
var documentUri = BuildDocumentUri(dto);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["ru-nkcki.bulletin"] = bulletinId,
|
||||
["ru-nkcki.entry"] = entryName,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
metadata["ru-nkcki.fstec_id"] = dto.FstecId!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
metadata["ru-nkcki.mitre_id"] = dto.MitreId!;
|
||||
}
|
||||
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
var lastModified = dto.DateUpdated ?? dto.DatePublished;
|
||||
var record = new DocumentRecord(
|
||||
recordId,
|
||||
SourceName,
|
||||
documentUri,
|
||||
now,
|
||||
sha,
|
||||
DocumentStatuses.PendingParse,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: lastModified,
|
||||
GridFsId: gridFsId,
|
||||
ExpiresAt: null);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
pendingMappings.Remove(upserted.Id);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<SourceFetchContentResult> FetchListingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, _options.ListingUri)
|
||||
{
|
||||
AcceptHeaders = ListingAcceptHeaders,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
return await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "NKCKI listing fetch failed for {ListingUri}", _options.ListingUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<BulletinAttachment>> ParseListingAsync(byte[] content, CancellationToken cancellationToken)
|
||||
{
|
||||
var html = Encoding.UTF8.GetString(content);
|
||||
var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false);
|
||||
var anchors = document.QuerySelectorAll("a[href$='.json.zip']");
|
||||
|
||||
var attachments = new List<BulletinAttachment>();
|
||||
foreach (var anchor in anchors)
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(_options.BaseAddress, href, out var absoluteUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = DeriveBulletinId(absoluteUri);
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var title = anchor.GetAttribute("title");
|
||||
if (string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
title = anchor.TextContent?.Trim();
|
||||
}
|
||||
|
||||
attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id));
|
||||
}
|
||||
|
||||
return attachments;
|
||||
}
|
||||
|
||||
private static string DeriveBulletinId(Uri uri)
|
||||
{
|
||||
var fileName = Path.GetFileName(uri.AbsolutePath);
|
||||
if (string.IsNullOrWhiteSpace(fileName))
|
||||
{
|
||||
return Guid.NewGuid().ToString("N");
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^4];
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^5];
|
||||
}
|
||||
|
||||
return fileName.Replace('_', '-');
|
||||
}
|
||||
|
||||
private static string BuildDocumentUri(RuNkckiVulnerabilityDto dto)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(dto.FstecId))
|
||||
{
|
||||
var slug = dto.FstecId.Contains(':', StringComparison.Ordinal)
|
||||
? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..]
|
||||
: dto.FstecId;
|
||||
return $"https://cert.gov.ru/materialy/uyazvimosti/{slug}";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.MitreId))
|
||||
{
|
||||
return $"https://nvd.nist.gov/vuln/detail/{dto.MitreId}";
|
||||
}
|
||||
|
||||
return $"https://cert.gov.ru/materialy/uyazvimosti/{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
private string ResolveCacheDirectory(string? configuredPath)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(configuredPath))
|
||||
{
|
||||
return Path.GetFullPath(Path.IsPathRooted(configuredPath)
|
||||
? configuredPath
|
||||
: Path.Combine(AppContext.BaseDirectory, configuredPath));
|
||||
}
|
||||
|
||||
return Path.Combine(AppContext.BaseDirectory, "cache", RuNkckiConnectorPlugin.SourceName);
|
||||
}
|
||||
|
||||
private void EnsureCacheDirectory()
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_cacheDirectory);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "NKCKI unable to ensure cache directory {CachePath}", _cacheDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetBulletinCachePath(string bulletinId)
|
||||
{
|
||||
var fileStem = string.IsNullOrWhiteSpace(bulletinId)
|
||||
? Guid.NewGuid().ToString("N")
|
||||
: Uri.EscapeDataString(bulletinId);
|
||||
return Path.Combine(_cacheDirectory, $"{fileStem}.json.zip");
|
||||
}
|
||||
|
||||
private static string ExtractBulletinIdFromCachePath(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var fileName = Path.GetFileName(path);
|
||||
if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^4];
|
||||
}
|
||||
|
||||
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
fileName = fileName[..^5];
|
||||
}
|
||||
|
||||
return Uri.UnescapeDataString(fileName);
|
||||
}
|
||||
|
||||
private void TryWriteCachedBulletin(string bulletinId, byte[] content)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cachePath = GetBulletinCachePath(bulletinId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!);
|
||||
File.WriteAllBytes(cachePath, content);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to cache bulletin {BulletinId}", bulletinId);
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadCachedBulletin(string bulletinId, out byte[] content)
|
||||
{
|
||||
var cachePath = GetBulletinCachePath(bulletinId);
|
||||
try
|
||||
{
|
||||
if (File.Exists(cachePath))
|
||||
{
|
||||
content = File.ReadAllBytes(cachePath);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "NKCKI failed to read cached bulletin {BulletinId}", bulletinId);
|
||||
}
|
||||
|
||||
content = Array.Empty<byte>();
|
||||
return false;
|
||||
}
|
||||
|
||||
private IReadOnlyCollection<string> NormalizeBulletins(IEnumerable<string> bulletins)
|
||||
{
|
||||
var normalized = (bulletins ?? Enumerable.Empty<string>())
|
||||
.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
if (normalized.Count <= _options.KnownBulletinCapacity)
|
||||
{
|
||||
return normalized.ToArray();
|
||||
}
|
||||
|
||||
var skip = normalized.Count - _options.KnownBulletinCapacity;
|
||||
return normalized.Skip(skip).ToArray();
|
||||
}
|
||||
|
||||
private async Task<RuNkckiCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? RuNkckiCursor.Empty : RuNkckiCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ru-nkcki";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "feedser:sources:ru-nkcki";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddRuNkckiConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<RuNkckiFetchJob>();
|
||||
services.AddTransient<RuNkckiParseJob>();
|
||||
services.AddTransient<RuNkckiMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
|
||||
{
|
||||
if (schedulerOptions.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
schedulerOptions.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
schedulerOptions.DefaultTimeout,
|
||||
schedulerOptions.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Ru.Nkcki.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Ru.Nkcki;
|
||||
|
||||
public static class RuNkckiServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddRuNkckiConnector(this IServiceCollection services, Action<RuNkckiOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<RuNkckiOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(RuNkckiOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<RuNkckiOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseAddress;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = options.UserAgent;
|
||||
clientOptions.AllowAutoRedirect = true;
|
||||
clientOptions.DefaultRequestHeaders["Accept-Language"] = options.AcceptLanguage;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseAddress.Host);
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
|
||||
handler.AllowAutoRedirect = true;
|
||||
handler.UseCookies = true;
|
||||
handler.CookieContainer = new CookieContainer();
|
||||
};
|
||||
});
|
||||
|
||||
services.AddTransient<RuNkckiConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,22 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AngleSharp" Version="1.1.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user