up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions

View File

@@ -1,137 +1,137 @@
using System.Net;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
/// <summary>
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
/// </summary>
public sealed class RuNkckiOptions
{
public const string HttpClientName = "ru-nkcki";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
/// <summary>
/// Base endpoint used for resolving relative resource links.
/// </summary>
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
/// <summary>
/// Relative path to the bulletin listing page.
/// </summary>
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
/// <summary>
/// Timeout applied to listing and bulletin fetch requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when the listing or attachments cannot be retrieved.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Maximum number of bulletin attachments downloaded per fetch run.
/// </summary>
public int MaxBulletinsPerFetch { get; set; } = 5;
/// <summary>
/// Maximum number of listing pages visited per fetch cycle.
/// </summary>
public int MaxListingPagesPerFetch { get; set; } = 3;
/// <summary>
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
/// </summary>
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
/// <summary>
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
/// </summary>
public int KnownBulletinCapacity { get; set; } = 512;
/// <summary>
/// Delay between sequential bulletin downloads.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Duration the HTML listing can be cached before forcing a refetch.
/// </summary>
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
/// <summary>
/// Absolute URI for the listing page.
/// </summary>
public Uri ListingUri => new(BaseAddress, ListingPath);
/// <summary>
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
/// </summary>
public string? CacheDirectory { get; set; } = null;
public void Validate()
{
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
{
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ListingPath))
{
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
}
if (MaxBulletinsPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
}
if (MaxListingPagesPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
}
if (MaxVulnerabilitiesPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
}
if (KnownBulletinCapacity <= 0)
{
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
}
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
{
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
}
if (string.IsNullOrWhiteSpace(AcceptLanguage))
{
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
}
}
}
using System.Net;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
/// <summary>
/// Connector options for the Russian NKTsKI bulletin ingestion pipeline.
/// </summary>
public sealed class RuNkckiOptions
{
public const string HttpClientName = "ru-nkcki";
private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90);
private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20);
private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10);
/// <summary>
/// Base endpoint used for resolving relative resource links.
/// </summary>
public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute);
/// <summary>
/// Relative path to the bulletin listing page.
/// </summary>
public string ListingPath { get; set; } = "materialy/uyazvimosti/";
/// <summary>
/// Timeout applied to listing and bulletin fetch requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout;
/// <summary>
/// Backoff applied when the listing or attachments cannot be retrieved.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff;
/// <summary>
/// Maximum number of bulletin attachments downloaded per fetch run.
/// </summary>
public int MaxBulletinsPerFetch { get; set; } = 5;
/// <summary>
/// Maximum number of listing pages visited per fetch cycle.
/// </summary>
public int MaxListingPagesPerFetch { get; set; } = 3;
/// <summary>
/// Maximum number of vulnerabilities ingested per fetch cycle across all attachments.
/// </summary>
public int MaxVulnerabilitiesPerFetch { get; set; } = 250;
/// <summary>
/// Maximum bulletin identifiers remembered to avoid refetching historical files.
/// </summary>
public int KnownBulletinCapacity { get; set; } = 512;
/// <summary>
/// Delay between sequential bulletin downloads.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Duration the HTML listing can be cached before forcing a refetch.
/// </summary>
public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache;
public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)";
public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4";
/// <summary>
/// Absolute URI for the listing page.
/// </summary>
public Uri ListingUri => new(BaseAddress, ListingPath);
/// <summary>
/// Optional directory for caching downloaded bulletins (relative paths resolve under the content root).
/// </summary>
public string? CacheDirectory { get; set; } = null;
public void Validate()
{
if (BaseAddress is null || !BaseAddress.IsAbsoluteUri)
{
throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ListingPath))
{
throw new InvalidOperationException("RuNkcki ListingPath must be provided.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("RuNkcki RequestTimeout must be positive.");
}
if (FailureBackoff < TimeSpan.Zero)
{
throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative.");
}
if (MaxBulletinsPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero.");
}
if (MaxListingPagesPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero.");
}
if (MaxVulnerabilitiesPerFetch <= 0)
{
throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero.");
}
if (KnownBulletinCapacity <= 0)
{
throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero.");
}
if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0)
{
throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace.");
}
if (string.IsNullOrWhiteSpace(UserAgent))
{
throw new InvalidOperationException("RuNkcki UserAgent cannot be empty.");
}
if (string.IsNullOrWhiteSpace(AcceptLanguage))
{
throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty.");
}
}
}

View File

@@ -1,108 +1,108 @@
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
internal sealed record RuNkckiCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyCollection<string> KnownBulletins,
DateTimeOffset? LastListingFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
=> this with { LastListingFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownBulletins"] = new BsonArray(KnownBulletins),
};
if (LastListingFetchAt.HasValue)
{
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
}
return document;
}
public static RuNkckiCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var knownBulletins = ReadStringArray(document, "knownBulletins");
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
? ParseDate(dateValue)
: null;
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuids;
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyBulletins;
}
var result = new List<string>(array.Count);
foreach (var element in array)
{
var text = element?.ToString();
if (!string.IsNullOrWhiteSpace(text))
{
result.Add(text);
}
}
return result;
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
internal sealed record RuNkckiCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyCollection<string> KnownBulletins,
DateTimeOffset? LastListingFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyBulletins = Array.Empty<string>();
public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null);
public RuNkckiCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
public RuNkckiCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray() };
public RuNkckiCursor WithKnownBulletins(IEnumerable<string> bulletins)
=> this with { KnownBulletins = (bulletins ?? Enumerable.Empty<string>()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() };
public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp)
=> this with { LastListingFetchAt = timestamp };
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["knownBulletins"] = new DocumentArray(KnownBulletins),
};
if (LastListingFetchAt.HasValue)
{
document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime;
}
return document;
}
public static RuNkckiCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var knownBulletins = ReadStringArray(document, "knownBulletins");
var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue)
? ParseDate(dateValue)
: null;
return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch);
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuids;
}
var result = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var guid))
{
result.Add(guid);
}
}
return result;
}
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyBulletins;
}
var result = new List<string>(array.Count);
foreach (var element in array)
{
var text = element?.ToString();
if (!string.IsNullOrWhiteSpace(text))
{
result.Add(text);
}
}
return result;
}
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -1,40 +1,40 @@
using System.Collections.Immutable;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
internal sealed record RuNkckiVulnerabilityDto(
string? FstecId,
string? MitreId,
DateTimeOffset? DatePublished,
DateTimeOffset? DateUpdated,
string? CvssRating,
bool? PatchAvailable,
string? Description,
RuNkckiCweDto? Cwe,
ImmutableArray<string> ProductCategories,
string? Mitigation,
string? VulnerableSoftwareText,
bool? VulnerableSoftwareHasCpe,
ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
double? CvssScore,
string? CvssVector,
double? CvssScoreV4,
string? CvssVectorV4,
string? Impact,
string? MethodOfExploitation,
bool? UserInteraction,
ImmutableArray<string> Urls,
ImmutableArray<string> Tags)
{
[JsonIgnore]
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
? FstecId!
: !string.IsNullOrWhiteSpace(MitreId)
? MitreId!
: Guid.NewGuid().ToString();
}
internal sealed record RuNkckiCweDto(int? Number, string? Description);
internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);
using System.Collections.Immutable;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal;
internal sealed record RuNkckiVulnerabilityDto(
string? FstecId,
string? MitreId,
DateTimeOffset? DatePublished,
DateTimeOffset? DateUpdated,
string? CvssRating,
bool? PatchAvailable,
string? Description,
RuNkckiCweDto? Cwe,
ImmutableArray<string> ProductCategories,
string? Mitigation,
string? VulnerableSoftwareText,
bool? VulnerableSoftwareHasCpe,
ImmutableArray<RuNkckiSoftwareEntry> VulnerableSoftwareEntries,
double? CvssScore,
string? CvssVector,
double? CvssScoreV4,
string? CvssVectorV4,
string? Impact,
string? MethodOfExploitation,
bool? UserInteraction,
ImmutableArray<string> Urls,
ImmutableArray<string> Tags)
{
[JsonIgnore]
public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId)
? FstecId!
: !string.IsNullOrWhiteSpace(MitreId)
? MitreId!
: Guid.NewGuid().ToString();
}
internal sealed record RuNkckiCweDto(int? Number, string? Description);
internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray<string> RangeExpressions);

View File

@@ -1,43 +1,43 @@
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
internal static class RuNkckiJobKinds
{
public const string Fetch = "source:ru-nkcki:fetch";
public const string Parse = "source:ru-nkcki:parse";
public const string Map = "source:ru-nkcki:map";
}
internal sealed class RuNkckiFetchJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiFetchJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class RuNkckiParseJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiParseJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class RuNkckiMapJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiMapJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
internal static class RuNkckiJobKinds
{
public const string Fetch = "source:ru-nkcki:fetch";
public const string Parse = "source:ru-nkcki:parse";
public const string Map = "source:ru-nkcki:map";
}
internal sealed class RuNkckiFetchJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiFetchJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class RuNkckiParseJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiParseJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class RuNkckiMapJob : IJob
{
private readonly RuNkckiConnector _connector;
public RuNkckiMapJob(RuNkckiConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")]

View File

@@ -10,7 +10,7 @@ using System.Text.Json.Serialization;
using AngleSharp.Html.Parser;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
@@ -338,7 +338,7 @@ public sealed class RuNkckiConnector : IFeedConnector
continue;
}
var bson = StellaOps.Concelier.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var bson = StellaOps.Concelier.Documents.DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow());
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -876,7 +876,7 @@ public sealed class RuNkckiConnector : IFeedConnector
private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}

View File

@@ -1,19 +1,19 @@
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "ru-nkcki";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
}
}
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
public sealed class RuNkckiConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "ru-nkcki";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<RuNkckiConnector>(services);
}
}

View File

@@ -1,53 +1,53 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:ru-nkcki";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddRuNkckiConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<RuNkckiFetchJob>();
services.AddTransient<RuNkckiParseJob>();
services.AddTransient<RuNkckiMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
{
if (schedulerOptions.Definitions.ContainsKey(kind))
{
return;
}
schedulerOptions.Definitions[kind] = new JobDefinition(
kind,
jobType,
schedulerOptions.DefaultTimeout,
schedulerOptions.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration;
namespace StellaOps.Concelier.Connector.Ru.Nkcki;
public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:ru-nkcki";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddRuNkckiConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<RuNkckiFetchJob>();
services.AddTransient<RuNkckiParseJob>();
services.AddTransient<RuNkckiMapJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob));
EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob));
EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType)
{
if (schedulerOptions.Definitions.ContainsKey(kind))
{
return;
}
schedulerOptions.Definitions[kind] = new JobDefinition(
kind,
jobType,
schedulerOptions.DefaultTimeout,
schedulerOptions.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}