up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,137 +1,137 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
|
||||
/// </summary>
|
||||
public sealed class RuNkckiOptions
|
||||
{
|
||||
public const string HttpClientName = "ru-nkcki";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
|
||||
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint used for resolving relative resource links.
|
||||
/// </summary>
|
||||
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the bulletin listing page.
|
||||
/// </summary>
|
||||
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to listing and bulletin fetch requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the listing or attachments cannot be retrieved.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of bulletin attachments downloaded per fetch run.
|
||||
/// </summary>
|
||||
public int MaxBulletinsPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of listing pages visited per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxListingPagesPerFetch { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
|
||||
/// </summary>
|
||||
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
|
||||
/// </summary>
|
||||
public int KnownBulletinCapacity { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between sequential bulletin downloads.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Duration the HTML listing can be cached before forcing a refetch.
|
||||
/// </summary>
|
||||
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
|
||||
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
|
||||
|
||||
/// <summary>
|
||||
/// Absolute URI for the listing page.
|
||||
/// </summary>
|
||||
public Uri ListingUri => new(BaseAddress, ListingPath);
|
||||
|
||||
/// <summary>
|
||||
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; } = null;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ListingPath))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxBulletinsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxListingPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxVulnerabilitiesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (KnownBulletinCapacity <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
|
||||
}
|
||||
|
||||
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AcceptLanguage))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
|
||||
/// </summary>
|
||||
public sealed class RuNkckiOptions
|
||||
{
|
||||
public const string HttpClientName = "ru-nkcki";
|
||||
|
||||
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
|
||||
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
|
||||
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
|
||||
|
||||
/// <summary>
|
||||
/// Base endpoint used for resolving relative resource links.
|
||||
/// </summary>
|
||||
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
|
||||
|
||||
/// <summary>
|
||||
/// Relative path to the bulletin listing page.
|
||||
/// </summary>
|
||||
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to listing and bulletin fetch requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Backoff applied when the listing or attachments cannot be retrieved.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of bulletin attachments downloaded per fetch run.
|
||||
/// </summary>
|
||||
public int MaxBulletinsPerFetch { get; set; } = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of listing pages visited per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxListingPagesPerFetch { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
|
||||
/// </summary>
|
||||
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
|
||||
/// </summary>
|
||||
public int KnownBulletinCapacity { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Delay between sequential bulletin downloads.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Duration the HTML listing can be cached before forcing a refetch.
|
||||
/// </summary>
|
||||
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
|
||||
|
||||
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
|
||||
|
||||
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
|
||||
|
||||
/// <summary>
|
||||
/// Absolute URI for the listing page.
|
||||
/// </summary>
|
||||
public Uri ListingUri => new(BaseAddress, ListingPath);
|
||||
|
||||
/// <summary>
|
||||
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; } = null;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(ListingPath))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
|
||||
}
|
||||
|
||||
if (FailureBackoff < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
|
||||
}
|
||||
|
||||
if (MaxBulletinsPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxListingPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxVulnerabilitiesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
|
||||
}
|
||||
|
||||
if (KnownBulletinCapacity <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
|
||||
}
|
||||
|
||||
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UserAgent))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(AcceptLanguage))
|
||||
{
|
||||
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,108 +1,108 @@
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownBulletins,
|
||||
DateTimeOffset? LastListingFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
|
||||
|
||||
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
|
||||
|
||||
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
|
||||
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
|
||||
|
||||
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastListingFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownBulletins"] = new BsonArray(KnownBulletins),
|
||||
};
|
||||
|
||||
if (LastListingFetchAt.HasValue)
|
||||
{
|
||||
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static RuNkckiCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownBulletins = ReadStringArray(document, "knownBulletins");
|
||||
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyBulletins;
|
||||
}
|
||||
|
||||
var result = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
var text = element?.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
result.Add(text);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownBulletins,
|
||||
DateTimeOffset? LastListingFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
|
||||
|
||||
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
|
||||
|
||||
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
|
||||
|
||||
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
|
||||
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
|
||||
|
||||
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastListingFetchAt = timestamp };
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownBulletins"] = new DocumentArray(KnownBulletins),
|
||||
};
|
||||
|
||||
if (LastListingFetchAt.HasValue)
|
||||
{
|
||||
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static RuNkckiCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownBulletins = ReadStringArray(document, "knownBulletins");
|
||||
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
|
||||
? ParseDate(dateValue)
|
||||
: null;
|
||||
|
||||
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var result = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
result.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyBulletins;
|
||||
}
|
||||
|
||||
var result = new List<string>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
var text = element?.ToString();
|
||||
if (!string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
result.Add(text);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
=> value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,40 +1,40 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiVulnerabilityDto(
|
||||
string? FstecId,
|
||||
string? MitreId,
|
||||
DateTimeOffset? DatePublished,
|
||||
DateTimeOffset? DateUpdated,
|
||||
string? CvssRating,
|
||||
bool? PatchAvailable,
|
||||
string? Description,
|
||||
RuNkckiCweDto? Cwe,
|
||||
ImmutableArray<string> ProductCategories,
|
||||
string? Mitigation,
|
||||
string? VulnerableSoftwareText,
|
||||
bool? VulnerableSoftwareHasCpe,
|
||||
ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
|
||||
double? CvssScore,
|
||||
string? CvssVector,
|
||||
double? CvssScoreV4,
|
||||
string? CvssVectorV4,
|
||||
string? Impact,
|
||||
string? MethodOfExploitation,
|
||||
bool? UserInteraction,
|
||||
ImmutableArray<string> Urls,
|
||||
ImmutableArray<string> Tags)
|
||||
{
|
||||
[JsonIgnore]
|
||||
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
|
||||
? FstecId!
|
||||
: !string.IsNullOrWhiteSpace(MitreId)
|
||||
? MitreId!
|
||||
: Guid.NewGuid().ToString();
|
||||
}
|
||||
|
||||
internal sealed record RuNkckiCweDto(int? Number, string? Description);
|
||||
|
||||
internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
|
||||
|
||||
internal sealed record RuNkckiVulnerabilityDto(
|
||||
string? FstecId,
|
||||
string? MitreId,
|
||||
DateTimeOffset? DatePublished,
|
||||
DateTimeOffset? DateUpdated,
|
||||
string? CvssRating,
|
||||
bool? PatchAvailable,
|
||||
string? Description,
|
||||
RuNkckiCweDto? Cwe,
|
||||
ImmutableArray<string> ProductCategories,
|
||||
string? Mitigation,
|
||||
string? VulnerableSoftwareText,
|
||||
bool? VulnerableSoftwareHasCpe,
|
||||
ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
|
||||
double? CvssScore,
|
||||
string? CvssVector,
|
||||
double? CvssScoreV4,
|
||||
string? CvssVectorV4,
|
||||
string? Impact,
|
||||
string? MethodOfExploitation,
|
||||
bool? UserInteraction,
|
||||
ImmutableArray<string> Urls,
|
||||
ImmutableArray<string> Tags)
|
||||
{
|
||||
[JsonIgnore]
|
||||
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
|
||||
? FstecId!
|
||||
: !string.IsNullOrWhiteSpace(MitreId)
|
||||
? MitreId!
|
||||
: Guid.NewGuid().ToString();
|
||||
}
|
||||
|
||||
internal sealed record RuNkckiCweDto(int? Number, string? Description);
|
||||
|
||||
internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);
|
||||
|
||||
@@ -1,43 +1,43 @@
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
internal static class RuNkckiJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ru-nkcki:fetch";
|
||||
public const string Parse = "source:ru-nkcki:parse";
|
||||
public const string Map = "source:ru-nkcki:map";
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiFetchJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiFetchJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiParseJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiParseJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiMapJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiMapJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
internal static class RuNkckiJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ru-nkcki:fetch";
|
||||
public const string Parse = "source:ru-nkcki:parse";
|
||||
public const string Map = "source:ru-nkcki:map";
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiFetchJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiFetchJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiParseJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiParseJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class RuNkckiMapJob : IJob
|
||||
{
|
||||
private readonly RuNkckiConnector _connector;
|
||||
|
||||
public RuNkckiMapJob(RuNkckiConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")]
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")]
|
||||
|
||||
@@ -10,7 +10,7 @@ using System.Text.Json.Serialization;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
@@ -338,7 +338,7 @@ public sealed class RuNkckiConnector : IFeedConnector
|
||||
continue;
|
||||
}
|
||||
|
||||
var bson = StellaOps.Concelier.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var bson = StellaOps.Concelier.Documents.DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow());
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
@@ -876,7 +876,7 @@ public sealed class RuNkckiConnector : IFeedConnector
|
||||
|
||||
private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var document = cursor.ToDocumentObject();
|
||||
var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ru-nkcki";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ru-nkcki";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,53 +1,53 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ru-nkcki";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddRuNkckiConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<RuNkckiFetchJob>();
|
||||
services.AddTransient<RuNkckiParseJob>();
|
||||
services.AddTransient<RuNkckiMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
|
||||
{
|
||||
if (schedulerOptions.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
schedulerOptions.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
schedulerOptions.DefaultTimeout,
|
||||
schedulerOptions.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
|
||||
|
||||
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ru-nkcki";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddRuNkckiConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<RuNkckiFetchJob>();
|
||||
services.AddTransient<RuNkckiParseJob>();
|
||||
services.AddTransient<RuNkckiMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
|
||||
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
|
||||
{
|
||||
if (schedulerOptions.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
schedulerOptions.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
schedulerOptions.DefaultTimeout,
|
||||
schedulerOptions.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user