Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,38 @@
|
||||
# AGENTS
|
||||
## Role
|
||||
Implement the Russian BDU (Vulnerability Database) connector to ingest advisories published by FSTEC’s BDU catalogue.
|
||||
|
||||
## Scope
|
||||
- Determine accessible BDU feeds/APIs (HTML listings, downloadable CSV, SOAP/REST) and access constraints.
|
||||
- Build fetch/cursor pipeline with dedupe, retries, and backoff appropriate for the data source.
|
||||
- Parse advisory records to extract summary, affected vendors/products, mitigation recommendations, CVE IDs.
|
||||
- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and range primitives.
|
||||
- Provide deterministic fixtures and regression tests for the connector lifecycle.
|
||||
|
||||
## Participants
|
||||
- `Source.Common` (HTTP/fetch utilities, DTO storage).
|
||||
- `Storage.Mongo` (raw/document/DTO/advisory stores + source state).
|
||||
- `Concelier.Models` (canonical data structures).
|
||||
- `Concelier.Testing` (integration harness, snapshot utilities).
|
||||
|
||||
## Interfaces & Contracts
|
||||
- Job kinds: `bdu:fetch`, `bdu:parse`, `bdu:map`.
|
||||
- Persist upstream metadata (e.g., record modification timestamp) to drive incremental updates.
|
||||
- Alias set should include BDU identifiers and CVE IDs when present.
|
||||
|
||||
## In/Out of scope
|
||||
In scope:
|
||||
- Core ingestion/mapping of BDU vulnerability records.
|
||||
|
||||
Out of scope:
|
||||
- Translation beyond normalising required canonical fields.
|
||||
|
||||
## Observability & Security Expectations
|
||||
- Log fetch/mapping statistics and failure details.
|
||||
- Sanitize source payloads, handling Cyrillic text/encodings correctly.
|
||||
- Respect upstream rate limits and mark failures with backoff.
|
||||
|
||||
## Tests
|
||||
- Add `StellaOps.Concelier.Connector.Ru.Bdu.Tests` covering fetch/parse/map with canned fixtures.
|
||||
- Snapshot canonical advisories; support fixture regeneration via env flag.
|
||||
- Ensure deterministic ordering/time normalisation.
|
||||
@@ -0,0 +1,102 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options for the Russian BDU archive ingestion pipeline.
|
||||
/// </summary>
|
||||
public sealed class RuBduOptions
|
||||
{
|
||||
public const string HttpClientName = "ru-bdu";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(30);
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint used for resolving relative resource paths.
|
||||
/// </summary>
|
||||
public Uri BaseAddress { get; set; } = new("https://bdu.fstec.ru/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the zipped vulnerability dataset.
|
||||
/// </summary>
|
||||
public string DataArchivePath { get; set; } = "files/documents/vulxml.zip";
|
||||
|
||||
/// <summary>
|
||||
/// HTTP timeout applied when downloading the archive.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the remote endpoint fails to serve the archive.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// User-Agent header used for outbound requests.
|
||||
/// </summary>
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
/// <summary>
|
||||
/// Accept-Language preference sent with outbound requests.
|
||||
/// </summary>
|
||||
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerabilities ingested per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxVulnerabilitiesPerFetch { get; set; } = 500;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the absolute URI for the archive download.
|
||||
/// </summary>
|
||||
public Uri DataArchiveUri => new(BaseAddress, DataArchivePath);
|
||||
|
||||
/// <summary>
|
||||
/// Optional directory for caching the most recent archive (relative paths resolve under the content root).
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; } = null;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu BaseAddress must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(DataArchivePath))
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu DataArchivePath must be provided.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AcceptLanguage))
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu AcceptLanguage cannot be empty.");
|
||||
}
|
||||
|
||||
if (MaxVulnerabilitiesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu MaxVulnerabilitiesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuBdu CacheDirectory cannot be whitespace.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
internal sealed record RuBduCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
DateTimeOffset? LastSuccessfulFetch)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
|
||||
public static RuBduCursor Empty { get; } = new(EmptyGuids, EmptyGuids, null);
|
||||
|
||||
public RuBduCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuBduCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuBduCursor WithLastSuccessfulFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastSuccessfulFetch = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastSuccessfulFetch.HasValue)
|
||||
{
|
||||
document["lastSuccessfulFetch"] = LastSuccessfulFetch.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static RuBduCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var lastFetch = document.TryGetValue("lastSuccessfulFetch", out var fetchValue)
|
||||
? ParseDate(fetchValue)
|
||||
: null;
|
||||
|
||||
return new RuBduCursor(pendingDocuments, pendingMappings, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
using System;
|
||||
using System.Diagnostics.Metrics;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits RU-BDU specific OpenTelemetry metrics for fetch/parse/map stages.
|
||||
/// </summary>
|
||||
public sealed class RuBduDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Concelier.Connector.Ru.Bdu";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchCacheFallbacks;
|
||||
private readonly Histogram<long> _fetchDocumentAdds;
|
||||
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseSoftwareCount;
|
||||
private readonly Histogram<long> _parseIdentifierCount;
|
||||
private readonly Histogram<long> _parseSourceCount;
|
||||
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapPackageCount;
|
||||
private readonly Histogram<long> _mapAliasCount;
|
||||
|
||||
public RuBduDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
|
||||
_fetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of RU-BDU archive fetch attempts.");
|
||||
_fetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of RU-BDU archive fetches processed successfully.");
|
||||
_fetchFailures = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of RU-BDU archive fetches that failed.");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of RU-BDU archive fetches returning HTTP 304.");
|
||||
_fetchCacheFallbacks = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.fetch.cache_fallbacks",
|
||||
unit: "operations",
|
||||
description: "Number of RU-BDU fetches that fell back to the cached archive.");
|
||||
_fetchDocumentAdds = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.fetch.documents",
|
||||
unit: "documents",
|
||||
description: "Distribution of new documents written per RU-BDU fetch.");
|
||||
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of RU-BDU documents parsed into DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of RU-BDU documents that failed parsing.");
|
||||
_parseSoftwareCount = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.parse.software.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of vulnerable software entries per RU-BDU DTO.");
|
||||
_parseIdentifierCount = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.parse.identifiers.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of external identifiers per RU-BDU DTO.");
|
||||
_parseSourceCount = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.parse.sources.count",
|
||||
unit: "entries",
|
||||
description: "Distribution of source references per RU-BDU DTO.");
|
||||
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the RU-BDU mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "ru.bdu.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of RU-BDU advisory mapping attempts that failed.");
|
||||
_mapPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.map.packages.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages per RU-BDU advisory.");
|
||||
_mapAliasCount = _meter.CreateHistogram<long>(
|
||||
name: "ru.bdu.map.aliases.count",
|
||||
unit: "aliases",
|
||||
description: "Distribution of aliases per RU-BDU advisory.");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void FetchCacheFallback() => _fetchCacheFallbacks.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void FetchSuccess(int addedCount, bool usedCache)
|
||||
{
|
||||
_ = usedCache;
|
||||
_fetchSuccess.Add(1);
|
||||
if (addedCount > 0)
|
||||
{
|
||||
_fetchDocumentAdds.Record(addedCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseSuccess(int softwareCount, int identifierCount, int sourceCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
_parseSoftwareCount.Record(Math.Max(softwareCount, 0));
|
||||
_parseIdentifierCount.Record(Math.Max(identifierCount, 0));
|
||||
_parseSourceCount.Record(Math.Max(sourceCount, 0));
|
||||
}
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void MapSuccess(Advisory advisory)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
_mapPackageCount.Record(advisory.AffectedPackages.Length);
|
||||
_mapAliasCount.Record(advisory.Aliases.Length);
|
||||
}
|
||||
|
||||
public void MapFailure() => _mapFailures.Add(1);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_meter.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,554 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Normalization.Cvss;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
internal static class RuBduMapper
|
||||
{
|
||||
private const string RawVersionScheme = "ru-bdu.raw";
|
||||
|
||||
public static Advisory Map(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var advisoryProvenance = new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.Identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var references = BuildReferences(dto, document, recordedAt);
|
||||
var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss);
|
||||
var severity = severityFromCvss ?? NormalizeSeverity(dto.SeverityText);
|
||||
var exploitKnown = DetermineExploitKnown(dto);
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.Identifier,
|
||||
title: dto.Name ?? dto.Identifier,
|
||||
summary: dto.Description,
|
||||
language: "ru",
|
||||
published: dto.IdentifyDate,
|
||||
modified: dto.IdentifyDate,
|
||||
severity: severity,
|
||||
exploitKnown: exploitKnown,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: cvssMetrics,
|
||||
provenance: new[] { advisoryProvenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(RuBduVulnerabilityDto dto)
|
||||
{
|
||||
var aliases = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.Identifier))
|
||||
{
|
||||
aliases.Add(dto.Identifier.Trim());
|
||||
}
|
||||
|
||||
foreach (var identifier in dto.Identifiers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(identifier.Value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
aliases.Add(identifier.Value.Trim());
|
||||
}
|
||||
|
||||
return aliases.Count == 0 ? Array.Empty<string>() : aliases.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Software.IsDefaultOrEmpty)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Software.Length);
|
||||
foreach (var software in dto.Software)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(software.Name) && string.IsNullOrWhiteSpace(software.Vendor))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var identifier = BuildPackageIdentifier(dto.Identifier, software);
|
||||
var packageProvenance = new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var statuses = BuildPackageStatuses(dto, recordedAt);
|
||||
var ranges = BuildVersionRanges(software, recordedAt);
|
||||
var normalizedVersions = BuildNormalizedVersions(software);
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
DeterminePackageType(software.Types),
|
||||
identifier,
|
||||
platform: NormalizePlatform(software.Platform),
|
||||
versionRanges: ranges,
|
||||
statuses: statuses,
|
||||
provenance: new[] { packageProvenance },
|
||||
normalizedVersions: normalizedVersions));
|
||||
}
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static string BuildPackageIdentifier(string fallbackIdentifier, RuBduSoftwareDto software)
|
||||
{
|
||||
var parts = new[] { software.Vendor, software.Name }
|
||||
.Where(static part => !string.IsNullOrWhiteSpace(part))
|
||||
.Select(static part => part!.Trim())
|
||||
.ToArray();
|
||||
|
||||
if (parts.Length == 0)
|
||||
{
|
||||
return software.Name ?? software.Vendor ?? fallbackIdentifier;
|
||||
}
|
||||
|
||||
return string.Join(" ", parts);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackageStatus> BuildPackageStatuses(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var statuses = new List<AffectedPackageStatus>(capacity: 2);
|
||||
|
||||
if (TryNormalizeVulnerabilityStatus(dto.VulStatus, out var vulnerabilityStatus))
|
||||
{
|
||||
statuses.Add(new AffectedPackageStatus(
|
||||
vulnerabilityStatus!,
|
||||
new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"package-status",
|
||||
dto.VulStatus!,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.PackageStatuses })));
|
||||
}
|
||||
|
||||
if (TryNormalizeFixStatus(dto.FixStatus, out var fixStatus))
|
||||
{
|
||||
statuses.Add(new AffectedPackageStatus(
|
||||
fixStatus!,
|
||||
new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"package-fix-status",
|
||||
dto.FixStatus!,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.PackageStatuses })));
|
||||
}
|
||||
|
||||
return statuses.Count == 0 ? Array.Empty<AffectedPackageStatus>() : statuses;
|
||||
}
|
||||
|
||||
private static bool TryNormalizeVulnerabilityStatus(string? status, out string? normalized)
|
||||
{
|
||||
normalized = null;
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var token = status.Trim().ToLowerInvariant();
|
||||
if (token.Contains("потенциал", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = AffectedPackageStatusCatalog.UnderInvestigation;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (token.Contains("подтвержд", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = AffectedPackageStatusCatalog.Affected;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (token.Contains("актуал", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = AffectedPackageStatusCatalog.Affected;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool TryNormalizeFixStatus(string? status, out string? normalized)
|
||||
{
|
||||
normalized = null;
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var token = status.Trim().ToLowerInvariant();
|
||||
if (token.Contains("устранена", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = AffectedPackageStatusCatalog.Fixed;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (token.Contains("информация об устранении отсутствует", StringComparison.Ordinal))
|
||||
{
|
||||
normalized = AffectedPackageStatusCatalog.Unknown;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(RuBduSoftwareDto software, DateTimeOffset recordedAt)
|
||||
{
|
||||
var tokens = SplitVersionTokens(software.Version).ToArray();
|
||||
if (tokens.Length == 0)
|
||||
{
|
||||
return Array.Empty<AffectedVersionRange>();
|
||||
}
|
||||
|
||||
var ranges = new List<AffectedVersionRange>(tokens.Length);
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
ranges.Add(new AffectedVersionRange(
|
||||
rangeKind: "string",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: token,
|
||||
provenance: new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"package-range",
|
||||
token,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges })));
|
||||
}
|
||||
|
||||
return ranges;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<NormalizedVersionRule> BuildNormalizedVersions(RuBduSoftwareDto software)
|
||||
{
|
||||
var tokens = SplitVersionTokens(software.Version).ToArray();
|
||||
if (tokens.Length == 0)
|
||||
{
|
||||
return Array.Empty<NormalizedVersionRule>();
|
||||
}
|
||||
|
||||
var rules = new List<NormalizedVersionRule>(tokens.Length);
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
rules.Add(new NormalizedVersionRule(
|
||||
RawVersionScheme,
|
||||
NormalizedVersionRuleTypes.Exact,
|
||||
value: token));
|
||||
}
|
||||
|
||||
return rules;
|
||||
}
|
||||
|
||||
private static IEnumerable<string> SplitVersionTokens(string? version)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(version))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var raw = version.Trim();
|
||||
if (raw.Length == 0 || string.Equals(raw, "-", StringComparison.Ordinal))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
var tokens = raw.Split(VersionSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
if (tokens.Length == 0)
|
||||
{
|
||||
yield return raw;
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token) || string.Equals(token, "-", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token.Equals("не указано", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Equals("не указана", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Equals("не определено", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Equals("не определена", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return token;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? NormalizePlatform(string? platform)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(platform))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = platform.Trim();
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (trimmed.Equals("-", StringComparison.Ordinal)
|
||||
|| trimmed.Equals("не указана", StringComparison.OrdinalIgnoreCase)
|
||||
|| trimmed.Equals("не указано", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static string DeterminePackageType(ImmutableArray<string> types)
|
||||
=> IsIcsSoftware(types) ? AffectedPackageTypes.IcsVendor : AffectedPackageTypes.Vendor;
|
||||
|
||||
private static bool IsIcsSoftware(ImmutableArray<string> types)
|
||||
{
|
||||
if (types.IsDefaultOrEmpty)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var type in types)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var token = type.Trim();
|
||||
if (token.Contains("АСУ", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Contains("SCADA", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Contains("ICS", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Contains("промыш", StringComparison.OrdinalIgnoreCase)
|
||||
|| token.Contains("industrial", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddReference(string? url, string kind, string sourceTag, string? summary = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var trimmed = url.Trim();
|
||||
if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri))
|
||||
{
|
||||
if (trimmed.StartsWith("www.", StringComparison.OrdinalIgnoreCase)
|
||||
&& Uri.TryCreate($"https://{trimmed}", UriKind.Absolute, out var prefixed))
|
||||
{
|
||||
uri = prefixed;
|
||||
}
|
||||
else
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var canonical = uri.ToString();
|
||||
if (!seen.Add(canonical))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
canonical,
|
||||
kind,
|
||||
sourceTag,
|
||||
summary,
|
||||
new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
canonical,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
AddReference(document.Uri, "details", RuBduConnectorPlugin.SourceName);
|
||||
|
||||
foreach (var source in dto.Sources)
|
||||
{
|
||||
AddReference(source, "source", RuBduConnectorPlugin.SourceName);
|
||||
}
|
||||
|
||||
foreach (var identifier in dto.Identifiers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(identifier.Link))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sourceTag = NormalizeIdentifierType(identifier.Type);
|
||||
var kind = string.Equals(sourceTag, "cve", StringComparison.Ordinal) ? "cve" : "external";
|
||||
AddReference(identifier.Link, kind, sourceTag, identifier.Value);
|
||||
}
|
||||
|
||||
foreach (var cwe in dto.Cwes)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cwe.Identifier))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var slug = cwe.Identifier.ToUpperInvariant().Replace("CWE-", string.Empty, StringComparison.OrdinalIgnoreCase);
|
||||
if (!slug.All(char.IsDigit))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var url = $"https://cwe.mitre.org/data/definitions/{slug}.html";
|
||||
AddReference(url, "cwe", "cwe", cwe.Name);
|
||||
}
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
private static string NormalizeIdentifierType(string? type)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
return RuBduConnectorPlugin.SourceName;
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(type.Length);
|
||||
foreach (var ch in type)
|
||||
{
|
||||
if (char.IsLetterOrDigit(ch))
|
||||
{
|
||||
builder.Append(char.ToLowerInvariant(ch));
|
||||
}
|
||||
else if (ch is '-' or '_' or '.')
|
||||
{
|
||||
builder.Append(ch);
|
||||
}
|
||||
}
|
||||
|
||||
return builder.Length == 0 ? RuBduConnectorPlugin.SourceName : builder.ToString();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CvssMetric> BuildCvssMetrics(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity)
|
||||
{
|
||||
severity = null;
|
||||
var metrics = new List<CvssMetric>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize("2.0", dto.CvssVector, dto.CvssScore, null, out var normalized))
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
normalized.Vector,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics });
|
||||
var metric = normalized.ToModel(provenance);
|
||||
metrics.Add(metric);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.Cvss3Vector) && CvssMetricNormalizer.TryNormalize("3.1", dto.Cvss3Vector, dto.Cvss3Score, null, out var normalized3))
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
RuBduConnectorPlugin.SourceName,
|
||||
"cvss",
|
||||
normalized3.Vector,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.CvssMetrics });
|
||||
var metric = normalized3.ToModel(provenance);
|
||||
metrics.Add(metric);
|
||||
}
|
||||
|
||||
if (metrics.Count > 1)
|
||||
{
|
||||
metrics = metrics
|
||||
.OrderByDescending(static metric => metric.BaseScore)
|
||||
.ThenBy(static metric => metric.Version, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
severity = metrics.Count > 0 ? metrics[0].BaseSeverity : severity;
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static string? NormalizeSeverity(string? severityText)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(severityText))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var token = severityText.Trim().ToLowerInvariant();
|
||||
if (token.Contains("критич", StringComparison.Ordinal))
|
||||
{
|
||||
return "critical";
|
||||
}
|
||||
|
||||
if (token.Contains("высок", StringComparison.Ordinal))
|
||||
{
|
||||
return "high";
|
||||
}
|
||||
|
||||
if (token.Contains("средн", StringComparison.Ordinal) || token.Contains("умер", StringComparison.Ordinal))
|
||||
{
|
||||
return "medium";
|
||||
}
|
||||
|
||||
if (token.Contains("низк", StringComparison.Ordinal))
|
||||
{
|
||||
return "low";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool DetermineExploitKnown(RuBduVulnerabilityDto dto)
|
||||
{
|
||||
if (dto.IncidentCount.HasValue && dto.IncidentCount.Value > 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.ExploitStatus))
|
||||
{
|
||||
var status = dto.ExploitStatus.Trim().ToLowerInvariant();
|
||||
if (status.Contains("существ", StringComparison.Ordinal) || status.Contains("использ", StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static readonly char[] VersionSeparators = { ',', ';', '\r', '\n', '\t' };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
internal sealed record RuBduVulnerabilityDto(
|
||||
string Identifier,
|
||||
string? Name,
|
||||
string? Description,
|
||||
string? Solution,
|
||||
DateTimeOffset? IdentifyDate,
|
||||
string? SeverityText,
|
||||
string? CvssVector,
|
||||
double? CvssScore,
|
||||
string? Cvss3Vector,
|
||||
double? Cvss3Score,
|
||||
string? ExploitStatus,
|
||||
int? IncidentCount,
|
||||
string? FixStatus,
|
||||
string? VulStatus,
|
||||
string? VulClass,
|
||||
string? VulState,
|
||||
string? Other,
|
||||
ImmutableArray<RuBduSoftwareDto> Software,
|
||||
ImmutableArray<RuBduEnvironmentDto> Environment,
|
||||
ImmutableArray<RuBduCweDto> Cwes,
|
||||
ImmutableArray<string> Sources,
|
||||
ImmutableArray<RuBduExternalIdentifierDto> Identifiers)
|
||||
{
|
||||
[JsonIgnore]
|
||||
public bool HasCvss => !string.IsNullOrWhiteSpace(CvssVector) || !string.IsNullOrWhiteSpace(Cvss3Vector);
|
||||
}
|
||||
|
||||
internal sealed record RuBduSoftwareDto(
|
||||
string? Vendor,
|
||||
string? Name,
|
||||
string? Version,
|
||||
string? Platform,
|
||||
ImmutableArray<string> Types);
|
||||
|
||||
internal sealed record RuBduEnvironmentDto(
|
||||
string? Vendor,
|
||||
string? Name,
|
||||
string? Version,
|
||||
string? Platform);
|
||||
|
||||
internal sealed record RuBduCweDto(string Identifier, string? Name);
|
||||
|
||||
internal sealed record RuBduExternalIdentifierDto(
|
||||
string Type,
|
||||
string Value,
|
||||
string? Link);
|
||||
@@ -0,0 +1,268 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
internal static class RuBduXmlParser
|
||||
{
|
||||
public static RuBduVulnerabilityDto? TryParse(XElement element)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(element);
|
||||
|
||||
var identifier = element.Element("identifier")?.Value?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var name = Normalize(element.Element("name")?.Value);
|
||||
var description = Normalize(element.Element("description")?.Value);
|
||||
var solution = Normalize(element.Element("solution")?.Value);
|
||||
var severity = Normalize(element.Element("severity")?.Value);
|
||||
var exploitStatus = Normalize(element.Element("exploit_status")?.Value);
|
||||
var fixStatus = Normalize(element.Element("fix_status")?.Value);
|
||||
var vulStatus = Normalize(element.Element("vul_status")?.Value);
|
||||
var vulClass = Normalize(element.Element("vul_class")?.Value);
|
||||
var vulState = Normalize(element.Element("vul_state")?.Value);
|
||||
var other = Normalize(element.Element("other")?.Value);
|
||||
var incidentCount = ParseInt(element.Element("vul_incident")?.Value);
|
||||
|
||||
var identifyDate = ParseDate(element.Element("identify_date")?.Value);
|
||||
|
||||
var cvssVectorElement = element.Element("cvss")?.Element("vector");
|
||||
var cvssVector = Normalize(cvssVectorElement?.Value);
|
||||
var cvssScore = ParseDouble(cvssVectorElement?.Attribute("score")?.Value);
|
||||
|
||||
var cvss3VectorElement = element.Element("cvss3")?.Element("vector");
|
||||
var cvss3Vector = Normalize(cvss3VectorElement?.Value);
|
||||
var cvss3Score = ParseDouble(cvss3VectorElement?.Attribute("score")?.Value);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cvssVector))
|
||||
{
|
||||
cvssVector = null;
|
||||
cvssScore = null;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(cvss3Vector))
|
||||
{
|
||||
cvss3Vector = null;
|
||||
cvss3Score = null;
|
||||
}
|
||||
|
||||
var software = ParseSoftware(element.Element("vulnerable_software"));
|
||||
var environment = ParseEnvironment(element.Element("environment"));
|
||||
var cwes = ParseCwes(element.Element("cwes"));
|
||||
var sources = ParseSources(element.Element("sources"));
|
||||
var identifiers = ParseIdentifiers(element.Element("identifiers"));
|
||||
|
||||
return new RuBduVulnerabilityDto(
|
||||
identifier.Trim(),
|
||||
name,
|
||||
description,
|
||||
solution,
|
||||
identifyDate,
|
||||
severity,
|
||||
cvssVector,
|
||||
cvssScore,
|
||||
cvss3Vector,
|
||||
cvss3Score,
|
||||
exploitStatus,
|
||||
incidentCount,
|
||||
fixStatus,
|
||||
vulStatus,
|
||||
vulClass,
|
||||
vulState,
|
||||
other,
|
||||
software,
|
||||
environment,
|
||||
cwes,
|
||||
sources,
|
||||
identifiers);
|
||||
}
|
||||
|
||||
private static ImmutableArray<RuBduSoftwareDto> ParseSoftware(XElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return ImmutableArray<RuBduSoftwareDto>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RuBduSoftwareDto>();
|
||||
foreach (var soft in root.Elements("soft"))
|
||||
{
|
||||
var vendor = Normalize(soft.Element("vendor")?.Value);
|
||||
var name = Normalize(soft.Element("name")?.Value);
|
||||
var version = Normalize(soft.Element("version")?.Value);
|
||||
var platform = Normalize(soft.Element("platform")?.Value);
|
||||
var types = soft.Element("types") is { } typesElement
|
||||
? typesElement.Elements("type").Select(static x => Normalize(x.Value)).Where(static value => !string.IsNullOrWhiteSpace(value)).Cast<string>().ToImmutableArray()
|
||||
: ImmutableArray<string>.Empty;
|
||||
|
||||
builder.Add(new RuBduSoftwareDto(vendor, name, version, platform, types));
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<RuBduEnvironmentDto> ParseEnvironment(XElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return ImmutableArray<RuBduEnvironmentDto>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RuBduEnvironmentDto>();
|
||||
foreach (var os in root.Elements())
|
||||
{
|
||||
var vendor = Normalize(os.Element("vendor")?.Value);
|
||||
var name = Normalize(os.Element("name")?.Value);
|
||||
var version = Normalize(os.Element("version")?.Value);
|
||||
var platform = Normalize(os.Element("platform")?.Value);
|
||||
builder.Add(new RuBduEnvironmentDto(vendor, name, version, platform));
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<RuBduCweDto> ParseCwes(XElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return ImmutableArray<RuBduCweDto>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RuBduCweDto>();
|
||||
foreach (var cwe in root.Elements("cwe"))
|
||||
{
|
||||
var identifier = Normalize(cwe.Element("identifier")?.Value);
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var name = Normalize(cwe.Element("name")?.Value);
|
||||
builder.Add(new RuBduCweDto(identifier, name));
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ParseSources(XElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var raw = root.Value;
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var tokens = raw
|
||||
.Split(SourceSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Select(static token => token.Trim())
|
||||
.Where(static token => !string.IsNullOrWhiteSpace(token))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return tokens.IsDefaultOrEmpty ? ImmutableArray<string>.Empty : tokens;
|
||||
}
|
||||
|
||||
private static ImmutableArray<RuBduExternalIdentifierDto> ParseIdentifiers(XElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return ImmutableArray<RuBduExternalIdentifierDto>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RuBduExternalIdentifierDto>();
|
||||
foreach (var identifier in root.Elements("identifier"))
|
||||
{
|
||||
var value = Normalize(identifier?.Value);
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var type = identifier?.Attribute("type")?.Value?.Trim();
|
||||
var link = identifier?.Attribute("link")?.Value?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
type = "external";
|
||||
}
|
||||
|
||||
builder.Add(new RuBduExternalIdentifierDto(type, value.Trim(), string.IsNullOrWhiteSpace(link) ? null : link));
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static readonly char[] SourceSeparators = { '\r', '\n', '\t', ' ' };
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var isoDate))
|
||||
{
|
||||
return isoDate;
|
||||
}
|
||||
|
||||
if (DateTimeOffset.TryParseExact(trimmed, new[] { "dd.MM.yyyy", "dd.MM.yyyy HH:mm:ss" }, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal, out var ruDate))
|
||||
{
|
||||
return ruDate;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static double? ParseDouble(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (double.TryParse(value.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static int? ParseInt(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (int.TryParse(value.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.Replace('\r', ' ').Replace('\n', ' ').Trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
|
||||
internal static class RuBduJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ru-bdu:fetch";
|
||||
public const string Parse = "source:ru-bdu:parse";
|
||||
public const string Map = "source:ru-bdu:map";
|
||||
}
|
||||
|
||||
internal sealed class RuBduFetchJob : IJob
|
||||
{
|
||||
private readonly RuBduConnector _connector;
|
||||
|
||||
public RuBduFetchJob(RuBduConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuBduParseJob : IJob
|
||||
{
|
||||
private readonly RuBduConnector _connector;
|
||||
|
||||
public RuBduParseJob(RuBduConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuBduMapJob : IJob
|
||||
{
|
||||
private readonly RuBduConnector _connector;
|
||||
|
||||
public RuBduMapJob(RuBduConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Bdu.Tests")]
|
||||
@@ -0,0 +1,40 @@
|
||||
# RU BDU Connector Notes
|
||||
|
||||
## Data source & access requirements
|
||||
|
||||
- **Primary feed**: `https://bdu.fstec.ru/files/documents/vulxml.zip` exposes the full vulnerability catalogue as a zipped XML tree (“`export/export.xml`”). FSTEC refreshes the archive several times per week; incremental diffs are not published, so every run downloads the full bundle.
|
||||
- **TLS trust**: the endpoint presents certificates chained to the Russian Trusted Root/Sub CAs. Bundle the official PEMs inside the deployment (`certificates/russian_trusted_root_ca.pem`, `certificates/russian_trusted_sub_ca.pem`, or the combined `certificates/russian_trusted_bundle.pem`) and point the connector at them, e.g.:
|
||||
|
||||
```yaml
|
||||
concelier:
|
||||
httpClients:
|
||||
source.bdu:
|
||||
trustedRootPaths:
|
||||
- certificates/russian_trusted_bundle.pem
|
||||
allowInvalidCertificates: false
|
||||
timeout: 00:02:00
|
||||
```
|
||||
|
||||
- **Offline Kit**: copy the PEM bundle above into the Offline Kit artefacts and set `concelier:offline:root` (or `CONCELIER_OFFLINE_ROOT`) so air‑gapped installs can resolve relative certificate paths. Package the most recent `vulxml.zip` alongside cached exports when preparing air-gap refreshes.
|
||||
|
||||
The connector keeps a local cache (`cache/ru-bdu/vulxml.zip`) so transient fetch failures can fall back to the last successful archive without blocking the cursor.
|
||||
|
||||
## Telemetry
|
||||
|
||||
The connector publishes an OpenTelemetry meter named `StellaOps.Concelier.Connector.Ru.Bdu`. Instruments include:
|
||||
|
||||
- `ru.bdu.fetch.*` – `attempts`, `success`, `failures`, `not_modified`, `cache_fallbacks`, and histogram `ru.bdu.fetch.documents`.
|
||||
- `ru.bdu.parse.*` – counters for success/failures plus histograms tracking vulnerable software, external identifiers, and source reference counts per DTO.
|
||||
- `ru.bdu.map.*` – counters for success/failures with histograms covering affected package counts and alias fan-out per advisory.
|
||||
|
||||
Use these metrics to alert on repeated cache fallbacks, sustained parse failures, or unexpected advisory fan-out.
|
||||
|
||||
## Regression fixtures
|
||||
|
||||
Deterministic fixtures live under `src/Concelier/__Tests/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures`. Run
|
||||
|
||||
```bash
|
||||
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Connector.Ru.Bdu.Tests
|
||||
```
|
||||
|
||||
to execute the RU BDU snapshot suite, and set `UPDATE_BDU_FIXTURES=1` to refresh stored snapshots when ingest logic changes. The harness records the fetch requests, documents, DTOs, advisories, and state cursor to guarantee reproducible pipelines across machines.
|
||||
@@ -0,0 +1,528 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Normalization.Cvss;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
|
||||
public sealed class RuBduConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly RuBduOptions _options;
|
||||
private readonly RuBduDiagnostics _diagnostics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<RuBduConnector> _logger;
|
||||
|
||||
private readonly string _cacheDirectory;
|
||||
private readonly string _archiveCachePath;
|
||||
|
||||
public RuBduConnector(
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<RuBduOptions> options,
|
||||
RuBduDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<RuBduConnector> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory);
|
||||
_archiveCachePath = Path.Combine(_cacheDirectory, "vulxml.zip");
|
||||
EnsureCacheDirectory();
|
||||
}
|
||||
|
||||
public string SourceName => RuBduConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
_diagnostics.FetchAttempt();
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
SourceFetchContentResult? archiveResult = null;
|
||||
byte[]? archiveContent = null;
|
||||
var usedCache = false;
|
||||
|
||||
try
|
||||
{
|
||||
var request = new SourceFetchRequest(RuBduOptions.HttpClientName, SourceName, _options.DataArchiveUri)
|
||||
{
|
||||
AcceptHeaders = new[]
|
||||
{
|
||||
"application/zip",
|
||||
"application/octet-stream",
|
||||
"application/x-zip-compressed",
|
||||
},
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
var fetchResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
archiveResult = fetchResult;
|
||||
|
||||
if (fetchResult.IsNotModified)
|
||||
{
|
||||
_logger.LogDebug("RU-BDU archive not modified.");
|
||||
_diagnostics.FetchUnchanged();
|
||||
await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (fetchResult.IsSuccess && fetchResult.Content is not null)
|
||||
{
|
||||
archiveContent = fetchResult.Content;
|
||||
TryWriteCachedArchive(archiveContent);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
if (TryReadCachedArchive(out var cachedFallback))
|
||||
{
|
||||
_logger.LogWarning(ex, "RU-BDU archive fetch failed; using cached artefact {CachePath}", _archiveCachePath);
|
||||
archiveContent = cachedFallback;
|
||||
usedCache = true;
|
||||
_diagnostics.FetchCacheFallback();
|
||||
}
|
||||
else
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "RU-BDU archive fetch failed for {ArchiveUri}", _options.DataArchiveUri);
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (archiveContent is null)
|
||||
{
|
||||
if (TryReadCachedArchive(out var cachedFallback))
|
||||
{
|
||||
var status = archiveResult?.StatusCode;
|
||||
_logger.LogWarning("RU-BDU archive unavailable (status={Status}); using cached artefact {CachePath}", status, _archiveCachePath);
|
||||
archiveContent = cachedFallback;
|
||||
usedCache = true;
|
||||
_diagnostics.FetchCacheFallback();
|
||||
}
|
||||
else
|
||||
{
|
||||
var status = archiveResult?.StatusCode;
|
||||
_logger.LogWarning("RU-BDU archive fetch returned no content (status={Status})", status);
|
||||
_diagnostics.FetchSuccess(addedCount: 0, usedCache: false);
|
||||
await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var archiveLastModified = archiveResult?.LastModified;
|
||||
int added;
|
||||
try
|
||||
{
|
||||
added = await ProcessArchiveAsync(archiveContent, now, pendingDocuments, pendingMappings, archiveLastModified, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "RU-BDU archive processing failed");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
_diagnostics.FetchSuccess(added, usedCache);
|
||||
if (added > 0)
|
||||
{
|
||||
_logger.LogInformation("RU-BDU processed {Added} vulnerabilities (cacheUsed={CacheUsed})", added, usedCache);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("RU-BDU fetch completed with no new vulnerabilities (cacheUsed={CacheUsed})", usedCache);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithLastSuccessfulFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_logger.LogWarning("RU-BDU document {DocumentId} missing GridFS payload", documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "RU-BDU unable to download raw document {DocumentId}", documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
throw;
|
||||
}
|
||||
|
||||
RuBduVulnerabilityDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuBduVulnerabilityDto>(payload, SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "RU-BDU failed to deserialize document {DocumentId}", documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_logger.LogWarning("RU-BDU document {DocumentId} produced null DTO", documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-bdu.v1", bson, _timeProvider.GetUtcNow());
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
_diagnostics.ParseSuccess(
|
||||
dto.Software.IsDefaultOrEmpty ? 0 : dto.Software.Length,
|
||||
dto.Identifiers.IsDefaultOrEmpty ? 0 : dto.Identifiers.Length,
|
||||
dto.Sources.IsDefaultOrEmpty ? 0 : dto.Sources.Length);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
_diagnostics.MapFailure();
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_logger.LogWarning("RU-BDU document {DocumentId} missing DTO payload", documentId);
|
||||
_diagnostics.MapFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
RuBduVulnerabilityDto dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<RuBduVulnerabilityDto>(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "RU-BDU failed to deserialize DTO for document {DocumentId}", documentId);
|
||||
_diagnostics.MapFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = RuBduMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
_diagnostics.MapSuccess(advisory);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "RU-BDU mapping failed for document {DocumentId}", documentId);
|
||||
_diagnostics.MapFailure();
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<int> ProcessArchiveAsync(
|
||||
byte[] archiveContent,
|
||||
DateTimeOffset now,
|
||||
HashSet<Guid> pendingDocuments,
|
||||
HashSet<Guid> pendingMappings,
|
||||
DateTimeOffset? archiveLastModified,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var added = 0;
|
||||
using var archiveStream = new MemoryStream(archiveContent, writable: false);
|
||||
using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false);
|
||||
var entry = archive.GetEntry("export/export.xml") ?? archive.Entries.FirstOrDefault();
|
||||
if (entry is null)
|
||||
{
|
||||
_logger.LogWarning("RU-BDU archive does not contain export/export.xml; skipping.");
|
||||
return added;
|
||||
}
|
||||
|
||||
await using var entryStream = entry.Open();
|
||||
using var reader = XmlReader.Create(entryStream, new XmlReaderSettings
|
||||
{
|
||||
IgnoreComments = true,
|
||||
IgnoreWhitespace = true,
|
||||
DtdProcessing = DtdProcessing.Ignore,
|
||||
CloseInput = false,
|
||||
});
|
||||
|
||||
while (reader.Read())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (reader.NodeType != XmlNodeType.Element || !reader.Name.Equals("vul", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (RuBduXmlParser.TryParse(XNode.ReadFrom(reader) as XElement ?? new XElement("vul")) is not { } dto)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions);
|
||||
var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
var documentUri = BuildDocumentUri(dto.Identifier);
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["ru-bdu.identifier"] = dto.Identifier,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.Name))
|
||||
{
|
||||
metadata["ru-bdu.name"] = dto.Name!;
|
||||
}
|
||||
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
var record = new DocumentRecord(
|
||||
recordId,
|
||||
SourceName,
|
||||
documentUri,
|
||||
now,
|
||||
sha,
|
||||
DocumentStatuses.PendingParse,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: archiveLastModified ?? dto.IdentifyDate,
|
||||
GridFsId: gridFsId,
|
||||
ExpiresAt: null);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
pendingMappings.Remove(upserted.Id);
|
||||
added++;
|
||||
|
||||
if (added >= _options.MaxVulnerabilitiesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
private string ResolveCacheDirectory(string? configuredPath)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(configuredPath))
|
||||
{
|
||||
return Path.GetFullPath(Path.IsPathRooted(configuredPath)
|
||||
? configuredPath
|
||||
: Path.Combine(AppContext.BaseDirectory, configuredPath));
|
||||
}
|
||||
|
||||
return Path.Combine(AppContext.BaseDirectory, "cache", RuBduConnectorPlugin.SourceName);
|
||||
}
|
||||
|
||||
private void EnsureCacheDirectory()
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_cacheDirectory);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "RU-BDU unable to ensure cache directory {CachePath}", _cacheDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
private void TryWriteCachedArchive(byte[] content)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(_archiveCachePath)!);
|
||||
File.WriteAllBytes(_archiveCachePath, content);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "RU-BDU failed to write cache archive {CachePath}", _archiveCachePath);
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadCachedArchive(out byte[] content)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(_archiveCachePath))
|
||||
{
|
||||
content = File.ReadAllBytes(_archiveCachePath);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "RU-BDU failed to read cache archive {CachePath}", _archiveCachePath);
|
||||
}
|
||||
|
||||
content = Array.Empty<byte>();
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string BuildDocumentUri(string identifier)
|
||||
{
|
||||
var slug = identifier.Contains(':', StringComparison.Ordinal)
|
||||
? identifier[(identifier.IndexOf(':') + 1)..]
|
||||
: identifier;
|
||||
return $"https://bdu.fstec.ru/vul/{slug}";
|
||||
}
|
||||
|
||||
private async Task<RuBduCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(RuBduCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastSuccessfulFetch ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
|
||||
public sealed class RuBduConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ru-bdu";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<RuBduConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
|
||||
public sealed class RuBduDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ru-bdu";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddRuBduConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<RuBduFetchJob>();
|
||||
services.AddTransient<RuBduParseJob>();
|
||||
services.AddTransient<RuBduMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, RuBduJobKinds.Fetch, typeof(RuBduFetchJob));
|
||||
EnsureJob(options, RuBduJobKinds.Parse, typeof(RuBduParseJob));
|
||||
EnsureJob(options, RuBduJobKinds.Map, typeof(RuBduMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
|
||||
{
|
||||
if (schedulerOptions.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
schedulerOptions.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
schedulerOptions.DefaultTimeout,
|
||||
schedulerOptions.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ru.Bdu.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Bdu;
|
||||
|
||||
public static class RuBduServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddRuBduConnector(this IServiceCollection services, Action<RuBduOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<RuBduOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(RuBduOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<RuBduOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.BaseAddress;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = options.UserAgent;
|
||||
clientOptions.AllowAutoRedirect = true;
|
||||
clientOptions.DefaultRequestHeaders["Accept-Language"] = options.AcceptLanguage;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.BaseAddress.Host);
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
|
||||
handler.AllowAutoRedirect = true;
|
||||
handler.UseCookies = true;
|
||||
handler.CookieContainer = new CookieContainer();
|
||||
};
|
||||
});
|
||||
|
||||
services.AddSingleton<RuBduDiagnostics>();
|
||||
services.AddTransient<RuBduConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,11 @@
|
||||
# TASKS
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-RUBDU-02-001 Identify BDU data source & schema|BE-Conn-BDU|Research|**DONE (2025-10-11)** – Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml`, `...?format=json`) return 403/404 even with `--insecure` because TLS chain requires Russian Trusted Sub CA and WAF expects referer/session headers. Documented request/response samples in `docs/concelier-connector-research-20251011.md`; blocked until trusted root + access strategy from Ops.|
|
||||
|FEEDCONN-RUBDU-02-002 Fetch pipeline & cursor handling|BE-Conn-BDU|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Connector streams `vulxml.zip` through cached fetches, persists JSON payloads via `RawDocumentStorage`, and tracks cursor pending sets. Added cache fallback + deterministic SHA logging and state updates tied to `TimeProvider`.|
|
||||
|FEEDCONN-RUBDU-02-003 DTO/parser implementation|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – `RuBduXmlParser` now captures identifiers, source links, CVSS 2/3 metrics, CWE arrays, and environment/software metadata with coverage for multi-entry fixtures.|
|
||||
|FEEDCONN-RUBDU-02-004 Canonical mapping & range primitives|BE-Conn-BDU|Models|**DONE (2025-10-14)** – `RuBduMapper` emits vendor/ICS packages with normalized `ru-bdu.raw` rules, dual status provenance, alias/reference hydration (CVE, external, source), and CVSS severity normalisation.|
|
||||
|FEEDCONN-RUBDU-02-005 Deterministic fixtures & regression tests|QA|Testing|**DONE (2025-10-14)** – Added connector harness snapshot suite with canned archive, state/documents/dtos/advisories snapshots under `Fixtures/`, gated by `UPDATE_BDU_FIXTURES`.|
|
||||
|FEEDCONN-RUBDU-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Introduced `RuBduDiagnostics` meter (fetch/parse/map counters & histograms) and authored connector README covering configuration, trusted roots, telemetry, and offline behaviour.|
|
||||
|FEEDCONN-RUBDU-02-007 Access & export options assessment|BE-Conn-BDU|Research|**DONE (2025-10-14)** – Documented archive access constraints, offline mirroring expectations, and export packaging in `src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Ru.Bdu/README.md` + flagged Offline Kit bundling requirements.|
|
||||
|FEEDCONN-RUBDU-02-008 Trusted root onboarding plan|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – Validated Russian Trusted Root/Sub CA bundle wiring (`certificates/russian_trusted_bundle.pem`), updated Offline Kit guidance, and surfaced `concelier:httpClients:source.bdu:trustedRootPaths` sample configuration.|
|
||||
Reference in New Issue
Block a user