up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,53 +1,53 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
public sealed class KasperskyOptions
|
||||
{
|
||||
public static string HttpClientName => "source.ics.kaspersky";
|
||||
|
||||
public Uri FeedUri { get; set; } = new("https://ics-cert.kaspersky.com/feed-advisories/", UriKind.Absolute);
|
||||
|
||||
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
public int MaxPagesPerFetch { get; set; } = 3;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
[MemberNotNull(nameof(FeedUri))]
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("FeedUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowSize must be greater than zero.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (WindowOverlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
|
||||
}
|
||||
|
||||
if (MaxPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
public sealed class KasperskyOptions
|
||||
{
|
||||
public static string HttpClientName => "source.ics.kaspersky";
|
||||
|
||||
public Uri FeedUri { get; set; } = new("https://ics-cert.kaspersky.com/feed-advisories/", UriKind.Absolute);
|
||||
|
||||
public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2);
|
||||
|
||||
public int MaxPagesPerFetch { get; set; } = 3;
|
||||
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
[MemberNotNull(nameof(FeedUri))]
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("FeedUri must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (WindowSize <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowSize must be greater than zero.");
|
||||
}
|
||||
|
||||
if (WindowOverlap < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap cannot be negative.");
|
||||
}
|
||||
|
||||
if (WindowOverlap >= WindowSize)
|
||||
{
|
||||
throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize.");
|
||||
}
|
||||
|
||||
if (MaxPagesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException("MaxPagesPerFetch must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RequestDelay cannot be negative.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal sealed record KasperskyAdvisoryDto(
|
||||
string AdvisoryKey,
|
||||
string Title,
|
||||
string Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary,
|
||||
string Content,
|
||||
ImmutableArray<string> CveIds,
|
||||
ImmutableArray<string> VendorNames);
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal sealed record KasperskyAdvisoryDto(
|
||||
string AdvisoryKey,
|
||||
string Title,
|
||||
string Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary,
|
||||
string Content,
|
||||
ImmutableArray<string> CveIds,
|
||||
ImmutableArray<string> VendorNames);
|
||||
|
||||
@@ -1,172 +1,172 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal static class KasperskyAdvisoryParser
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
|
||||
|
||||
public static KasperskyAdvisoryDto Parse(
|
||||
string advisoryKey,
|
||||
string title,
|
||||
string link,
|
||||
DateTimeOffset published,
|
||||
string? summary,
|
||||
byte[] rawHtml)
|
||||
{
|
||||
var content = ExtractText(rawHtml);
|
||||
var cves = ExtractCves(title, summary, content);
|
||||
var vendors = ExtractVendors(title, summary, content);
|
||||
|
||||
return new KasperskyAdvisoryDto(
|
||||
advisoryKey,
|
||||
title,
|
||||
link,
|
||||
published,
|
||||
summary,
|
||||
content,
|
||||
cves,
|
||||
vendors);
|
||||
}
|
||||
|
||||
private static string ExtractText(byte[] rawHtml)
|
||||
{
|
||||
if (rawHtml.Length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var html = Encoding.UTF8.GetString(rawHtml);
|
||||
html = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
|
||||
html = Regex.Replace(html, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(html);
|
||||
return string.IsNullOrWhiteSpace(decoded) ? string.Empty : WhitespaceRegex.Replace(decoded, " ").Trim();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
void Capture(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Capture(title);
|
||||
Capture(summary);
|
||||
Capture(content);
|
||||
|
||||
return set.OrderBy(static cve => cve, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractVendors(string title, string? summary, string content)
|
||||
{
|
||||
var candidates = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddCandidate(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var segment in SplitSegments(text))
|
||||
{
|
||||
var cleaned = CleanVendorSegment(segment);
|
||||
if (!string.IsNullOrWhiteSpace(cleaned))
|
||||
{
|
||||
candidates.Add(cleaned);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
AddCandidate(title);
|
||||
AddCandidate(summary);
|
||||
AddCandidate(content);
|
||||
|
||||
return candidates.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: candidates
|
||||
.OrderBy(static vendor => vendor, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> SplitSegments(string text)
|
||||
{
|
||||
var separators = new[] { ".", "-", "–", "—", ":" };
|
||||
var queue = new Queue<string>();
|
||||
queue.Enqueue(text);
|
||||
|
||||
foreach (var separator in separators)
|
||||
{
|
||||
var count = queue.Count;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var item = queue.Dequeue();
|
||||
var parts = item.Split(separator, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
queue.Enqueue(part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
private static string? CleanVendorSegment(string value)
|
||||
{
|
||||
var trimmed = value.Trim();
|
||||
if (string.IsNullOrEmpty(trimmed))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var lowered = trimmed.ToLowerInvariant();
|
||||
if (lowered.Contains("cve-", StringComparison.Ordinal) || lowered.Contains("vulnerability", StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = trimmed.Split(new[] { "vulnerability", "vulnerabilities" }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault() ?? trimmed;
|
||||
}
|
||||
|
||||
var providedMatch = Regex.Match(trimmed, "provided by\\s+(?<vendor>[A-Za-z0-9&.,' ]+)", RegexOptions.IgnoreCase);
|
||||
if (providedMatch.Success)
|
||||
{
|
||||
trimmed = providedMatch.Groups["vendor"].Value;
|
||||
}
|
||||
|
||||
var descriptorMatch = Regex.Match(trimmed, "^(?<vendor>[A-Z][A-Za-z0-9&.,' ]{1,80}?)(?:\\s+(?:controllers?|devices?|modules?|products?|gateways?|routers?|appliances?|systems?|solutions?|firmware))\\b", RegexOptions.IgnoreCase);
|
||||
if (descriptorMatch.Success)
|
||||
{
|
||||
trimmed = descriptorMatch.Groups["vendor"].Value;
|
||||
}
|
||||
|
||||
trimmed = trimmed.Replace("’", "'", StringComparison.Ordinal);
|
||||
trimmed = trimmed.Replace("\"", string.Empty, StringComparison.Ordinal);
|
||||
|
||||
if (trimmed.Length > 200)
|
||||
{
|
||||
trimmed = trimmed[..200];
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal static class KasperskyAdvisoryParser
|
||||
{
|
||||
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled);
|
||||
|
||||
public static KasperskyAdvisoryDto Parse(
|
||||
string advisoryKey,
|
||||
string title,
|
||||
string link,
|
||||
DateTimeOffset published,
|
||||
string? summary,
|
||||
byte[] rawHtml)
|
||||
{
|
||||
var content = ExtractText(rawHtml);
|
||||
var cves = ExtractCves(title, summary, content);
|
||||
var vendors = ExtractVendors(title, summary, content);
|
||||
|
||||
return new KasperskyAdvisoryDto(
|
||||
advisoryKey,
|
||||
title,
|
||||
link,
|
||||
published,
|
||||
summary,
|
||||
content,
|
||||
cves,
|
||||
vendors);
|
||||
}
|
||||
|
||||
private static string ExtractText(byte[] rawHtml)
|
||||
{
|
||||
if (rawHtml.Length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var html = Encoding.UTF8.GetString(rawHtml);
|
||||
html = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
|
||||
html = Regex.Replace(html, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(html);
|
||||
return string.IsNullOrWhiteSpace(decoded) ? string.Empty : WhitespaceRegex.Replace(decoded, " ").Trim();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
void Capture(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (Match match in CveRegex.Matches(text))
|
||||
{
|
||||
if (match.Success)
|
||||
{
|
||||
set.Add(match.Value.ToUpperInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Capture(title);
|
||||
Capture(summary);
|
||||
Capture(content);
|
||||
|
||||
return set.OrderBy(static cve => cve, StringComparer.Ordinal).ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractVendors(string title, string? summary, string content)
|
||||
{
|
||||
var candidates = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
void AddCandidate(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var segment in SplitSegments(text))
|
||||
{
|
||||
var cleaned = CleanVendorSegment(segment);
|
||||
if (!string.IsNullOrWhiteSpace(cleaned))
|
||||
{
|
||||
candidates.Add(cleaned);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
AddCandidate(title);
|
||||
AddCandidate(summary);
|
||||
AddCandidate(content);
|
||||
|
||||
return candidates.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: candidates
|
||||
.OrderBy(static vendor => vendor, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> SplitSegments(string text)
|
||||
{
|
||||
var separators = new[] { ".", "-", "–", "—", ":" };
|
||||
var queue = new Queue<string>();
|
||||
queue.Enqueue(text);
|
||||
|
||||
foreach (var separator in separators)
|
||||
{
|
||||
var count = queue.Count;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var item = queue.Dequeue();
|
||||
var parts = item.Split(separator, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
queue.Enqueue(part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
private static string? CleanVendorSegment(string value)
|
||||
{
|
||||
var trimmed = value.Trim();
|
||||
if (string.IsNullOrEmpty(trimmed))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var lowered = trimmed.ToLowerInvariant();
|
||||
if (lowered.Contains("cve-", StringComparison.Ordinal) || lowered.Contains("vulnerability", StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = trimmed.Split(new[] { "vulnerability", "vulnerabilities" }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault() ?? trimmed;
|
||||
}
|
||||
|
||||
var providedMatch = Regex.Match(trimmed, "provided by\\s+(?<vendor>[A-Za-z0-9&.,' ]+)", RegexOptions.IgnoreCase);
|
||||
if (providedMatch.Success)
|
||||
{
|
||||
trimmed = providedMatch.Groups["vendor"].Value;
|
||||
}
|
||||
|
||||
var descriptorMatch = Regex.Match(trimmed, "^(?<vendor>[A-Z][A-Za-z0-9&.,' ]{1,80}?)(?:\\s+(?:controllers?|devices?|modules?|products?|gateways?|routers?|appliances?|systems?|solutions?|firmware))\\b", RegexOptions.IgnoreCase);
|
||||
if (descriptorMatch.Success)
|
||||
{
|
||||
trimmed = descriptorMatch.Groups["vendor"].Value;
|
||||
}
|
||||
|
||||
trimmed = trimmed.Replace("’", "'", StringComparison.Ordinal);
|
||||
trimmed = trimmed.Replace("\"", string.Empty, StringComparison.Ordinal);
|
||||
|
||||
if (trimmed.Length > 200)
|
||||
{
|
||||
trimmed = trimmed[..200];
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,207 +1,207 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Bson;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal sealed record KasperskyCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, KasperskyFetchMetadata> FetchCache)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, KasperskyFetchMetadata> EmptyFetchCache =
|
||||
new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static KasperskyCursor Empty { get; } = new(null, EmptyGuidList, EmptyGuidList, EmptyFetchCache);
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (FetchCache.Count > 0)
|
||||
{
|
||||
var cacheArray = new BsonArray();
|
||||
foreach (var (uri, metadata) in FetchCache)
|
||||
{
|
||||
var cacheDocument = new BsonDocument
|
||||
{
|
||||
["uri"] = uri,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metadata.ETag))
|
||||
{
|
||||
cacheDocument["etag"] = metadata.ETag;
|
||||
}
|
||||
|
||||
if (metadata.LastModified.HasValue)
|
||||
{
|
||||
cacheDocument["lastModified"] = metadata.LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
cacheArray.Add(cacheDocument);
|
||||
}
|
||||
|
||||
document["fetchCache"] = cacheArray;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static KasperskyCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var lastPublishedValue)
|
||||
? ParseDate(lastPublishedValue)
|
||||
: null;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var fetchCache = ReadFetchCache(document);
|
||||
|
||||
return new KasperskyCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache);
|
||||
}
|
||||
|
||||
public KasperskyCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public KasperskyCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public KasperskyCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public KasperskyCursor WithFetchMetadata(string requestUri, string? etag, DateTimeOffset? lastModified)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(requestUri))
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(FetchCache, StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[requestUri] = new KasperskyFetchMetadata(etag, lastModified),
|
||||
};
|
||||
|
||||
return this with { FetchCache = cache };
|
||||
}
|
||||
|
||||
public KasperskyCursor PruneFetchCache(IEnumerable<string> keepUris)
|
||||
{
|
||||
if (FetchCache.Count == 0)
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var keepSet = new HashSet<string>(keepUris ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
|
||||
if (keepSet.Count == 0)
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var uri in keepSet)
|
||||
{
|
||||
if (FetchCache.TryGetValue(uri, out var metadata))
|
||||
{
|
||||
cache[uri] = metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return this with { FetchCache = cache };
|
||||
}
|
||||
|
||||
public bool TryGetFetchMetadata(string requestUri, out KasperskyFetchMetadata metadata)
|
||||
{
|
||||
if (FetchCache.TryGetValue(requestUri, out metadata!))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
metadata = default!;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
{
|
||||
return value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, KasperskyFetchMetadata> ReadFetchCache(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyFetchCache;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is not BsonDocument cacheDocument)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!cacheDocument.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var uri = uriValue.AsString;
|
||||
string? etag = cacheDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null;
|
||||
DateTimeOffset? lastModified = cacheDocument.TryGetValue("lastModified", out var lastModifiedValue)
|
||||
? ParseDate(lastModifiedValue)
|
||||
: null;
|
||||
|
||||
cache[uri] = new KasperskyFetchMetadata(etag, lastModified);
|
||||
}
|
||||
|
||||
return cache.Count == 0 ? EmptyFetchCache : cache;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record KasperskyFetchMetadata(string? ETag, DateTimeOffset? LastModified);
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Documents;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
internal sealed record KasperskyCursor(
|
||||
DateTimeOffset? LastPublished,
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, KasperskyFetchMetadata> FetchCache)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, KasperskyFetchMetadata> EmptyFetchCache =
|
||||
new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public static KasperskyCursor Empty { get; } = new(null, EmptyGuidList, EmptyGuidList, EmptyFetchCache);
|
||||
|
||||
public DocumentObject ToDocumentObject()
|
||||
{
|
||||
var document = new DocumentObject
|
||||
{
|
||||
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (FetchCache.Count > 0)
|
||||
{
|
||||
var cacheArray = new DocumentArray();
|
||||
foreach (var (uri, metadata) in FetchCache)
|
||||
{
|
||||
var cacheDocument = new DocumentObject
|
||||
{
|
||||
["uri"] = uri,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metadata.ETag))
|
||||
{
|
||||
cacheDocument["etag"] = metadata.ETag;
|
||||
}
|
||||
|
||||
if (metadata.LastModified.HasValue)
|
||||
{
|
||||
cacheDocument["lastModified"] = metadata.LastModified.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
cacheArray.Add(cacheDocument);
|
||||
}
|
||||
|
||||
document["fetchCache"] = cacheArray;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static KasperskyCursor FromBson(DocumentObject? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var lastPublishedValue)
|
||||
? ParseDate(lastPublishedValue)
|
||||
: null;
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var fetchCache = ReadFetchCache(document);
|
||||
|
||||
return new KasperskyCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache);
|
||||
}
|
||||
|
||||
public KasperskyCursor WithPendingDocuments(IEnumerable<Guid> ids)
|
||||
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public KasperskyCursor WithPendingMappings(IEnumerable<Guid> ids)
|
||||
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
|
||||
|
||||
public KasperskyCursor WithLastPublished(DateTimeOffset? timestamp)
|
||||
=> this with { LastPublished = timestamp };
|
||||
|
||||
public KasperskyCursor WithFetchMetadata(string requestUri, string? etag, DateTimeOffset? lastModified)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(requestUri))
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(FetchCache, StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[requestUri] = new KasperskyFetchMetadata(etag, lastModified),
|
||||
};
|
||||
|
||||
return this with { FetchCache = cache };
|
||||
}
|
||||
|
||||
public KasperskyCursor PruneFetchCache(IEnumerable<string> keepUris)
|
||||
{
|
||||
if (FetchCache.Count == 0)
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var keepSet = new HashSet<string>(keepUris ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
|
||||
if (keepSet.Count == 0)
|
||||
{
|
||||
return this;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var uri in keepSet)
|
||||
{
|
||||
if (FetchCache.TryGetValue(uri, out var metadata))
|
||||
{
|
||||
cache[uri] = metadata;
|
||||
}
|
||||
}
|
||||
|
||||
return this with { FetchCache = cache };
|
||||
}
|
||||
|
||||
public bool TryGetFetchMetadata(string requestUri, out KasperskyFetchMetadata metadata)
|
||||
{
|
||||
if (FetchCache.TryGetValue(requestUri, out metadata!))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
metadata = default!;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(DocumentValue value)
|
||||
{
|
||||
return value.DocumentType switch
|
||||
{
|
||||
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return Array.Empty<Guid>();
|
||||
}
|
||||
|
||||
var results = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Guid.TryParse(element.ToString(), out var guid))
|
||||
{
|
||||
results.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, KasperskyFetchMetadata> ReadFetchCache(DocumentObject document)
|
||||
{
|
||||
if (!document.TryGetValue("fetchCache", out var value) || value is not DocumentArray array)
|
||||
{
|
||||
return EmptyFetchCache;
|
||||
}
|
||||
|
||||
var cache = new Dictionary<string, KasperskyFetchMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is not DocumentObject cacheDocument)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!cacheDocument.TryGetValue("uri", out var uriValue) || uriValue.DocumentType != DocumentType.String)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var uri = uriValue.AsString;
|
||||
string? etag = cacheDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null;
|
||||
DateTimeOffset? lastModified = cacheDocument.TryGetValue("lastModified", out var lastModifiedValue)
|
||||
? ParseDate(lastModifiedValue)
|
||||
: null;
|
||||
|
||||
cache[uri] = new KasperskyFetchMetadata(etag, lastModified);
|
||||
}
|
||||
|
||||
return cache.Count == 0 ? EmptyFetchCache : cache;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record KasperskyFetchMetadata(string? ETag, DateTimeOffset? LastModified);
|
||||
|
||||
@@ -1,133 +1,133 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
public sealed class KasperskyFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly KasperskyOptions _options;
|
||||
private readonly ILogger<KasperskyFeedClient> _logger;
|
||||
|
||||
private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/";
|
||||
|
||||
public KasperskyFeedClient(IHttpClientFactory httpClientFactory, IOptions<KasperskyOptions> options, ILogger<KasperskyFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<KasperskyFeedItem>> GetItemsAsync(int page, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(KasperskyOptions.HttpClientName);
|
||||
var feedUri = BuildUri(_options.FeedUri, page);
|
||||
|
||||
using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8);
|
||||
var xml = await reader.ReadToEndAsync().ConfigureAwait(false);
|
||||
|
||||
var document = XDocument.Parse(xml, LoadOptions.None);
|
||||
var items = new List<KasperskyFeedItem>();
|
||||
var channel = document.Root?.Element("channel");
|
||||
if (channel is null)
|
||||
{
|
||||
_logger.LogWarning("Feed {FeedUri} is missing channel element", feedUri);
|
||||
return items;
|
||||
}
|
||||
|
||||
foreach (var item in channel.Elements("item"))
|
||||
{
|
||||
var title = item.Element("title")?.Value?.Trim();
|
||||
var linkValue = item.Element("link")?.Value?.Trim();
|
||||
var pubDateValue = item.Element("pubDate")?.Value?.Trim();
|
||||
var summary = item.Element("description")?.Value?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(linkValue) || string.IsNullOrWhiteSpace(pubDateValue))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(linkValue, UriKind.Absolute, out var link))
|
||||
{
|
||||
_logger.LogWarning("Skipping feed item with invalid link: {Link}", linkValue);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(pubDateValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published))
|
||||
{
|
||||
_logger.LogWarning("Skipping feed item {Title} due to invalid pubDate {PubDate}", title, pubDateValue);
|
||||
continue;
|
||||
}
|
||||
|
||||
var encoded = item.Element(ContentNamespace + "encoded")?.Value;
|
||||
if (!string.IsNullOrWhiteSpace(encoded))
|
||||
{
|
||||
summary ??= HtmlToPlainText(encoded);
|
||||
}
|
||||
|
||||
items.Add(new KasperskyFeedItem(title, Canonicalize(link), published.ToUniversalTime(), summary));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static Uri BuildUri(Uri baseUri, int page)
|
||||
{
|
||||
if (page <= 1)
|
||||
{
|
||||
return baseUri;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(baseUri);
|
||||
var trimmed = builder.Query.TrimStart('?');
|
||||
var pageSegment = $"paged={page.ToString(CultureInfo.InvariantCulture)}";
|
||||
builder.Query = string.IsNullOrEmpty(trimmed)
|
||||
? pageSegment
|
||||
: $"{trimmed}&{pageSegment}";
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static Uri Canonicalize(Uri link)
|
||||
{
|
||||
if (string.IsNullOrEmpty(link.Query))
|
||||
{
|
||||
return link;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(link)
|
||||
{
|
||||
Query = string.Empty,
|
||||
};
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static string? HtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var withoutScripts = System.Text.RegularExpressions.Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
var withoutStyles = System.Text.RegularExpressions.Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
var withoutTags = System.Text.RegularExpressions.Regex.Replace(withoutStyles, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
|
||||
return string.IsNullOrWhiteSpace(decoded) ? null : System.Text.RegularExpressions.Regex.Replace(decoded, "\\s+", " ").Trim();
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
public sealed class KasperskyFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly KasperskyOptions _options;
|
||||
private readonly ILogger<KasperskyFeedClient> _logger;
|
||||
|
||||
private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/";
|
||||
|
||||
public KasperskyFeedClient(IHttpClientFactory httpClientFactory, IOptions<KasperskyOptions> options, ILogger<KasperskyFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<KasperskyFeedItem>> GetItemsAsync(int page, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(KasperskyOptions.HttpClientName);
|
||||
var feedUri = BuildUri(_options.FeedUri, page);
|
||||
|
||||
using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8);
|
||||
var xml = await reader.ReadToEndAsync().ConfigureAwait(false);
|
||||
|
||||
var document = XDocument.Parse(xml, LoadOptions.None);
|
||||
var items = new List<KasperskyFeedItem>();
|
||||
var channel = document.Root?.Element("channel");
|
||||
if (channel is null)
|
||||
{
|
||||
_logger.LogWarning("Feed {FeedUri} is missing channel element", feedUri);
|
||||
return items;
|
||||
}
|
||||
|
||||
foreach (var item in channel.Elements("item"))
|
||||
{
|
||||
var title = item.Element("title")?.Value?.Trim();
|
||||
var linkValue = item.Element("link")?.Value?.Trim();
|
||||
var pubDateValue = item.Element("pubDate")?.Value?.Trim();
|
||||
var summary = item.Element("description")?.Value?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(linkValue) || string.IsNullOrWhiteSpace(pubDateValue))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(linkValue, UriKind.Absolute, out var link))
|
||||
{
|
||||
_logger.LogWarning("Skipping feed item with invalid link: {Link}", linkValue);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(pubDateValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published))
|
||||
{
|
||||
_logger.LogWarning("Skipping feed item {Title} due to invalid pubDate {PubDate}", title, pubDateValue);
|
||||
continue;
|
||||
}
|
||||
|
||||
var encoded = item.Element(ContentNamespace + "encoded")?.Value;
|
||||
if (!string.IsNullOrWhiteSpace(encoded))
|
||||
{
|
||||
summary ??= HtmlToPlainText(encoded);
|
||||
}
|
||||
|
||||
items.Add(new KasperskyFeedItem(title, Canonicalize(link), published.ToUniversalTime(), summary));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static Uri BuildUri(Uri baseUri, int page)
|
||||
{
|
||||
if (page <= 1)
|
||||
{
|
||||
return baseUri;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(baseUri);
|
||||
var trimmed = builder.Query.TrimStart('?');
|
||||
var pageSegment = $"paged={page.ToString(CultureInfo.InvariantCulture)}";
|
||||
builder.Query = string.IsNullOrEmpty(trimmed)
|
||||
? pageSegment
|
||||
: $"{trimmed}&{pageSegment}";
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static Uri Canonicalize(Uri link)
|
||||
{
|
||||
if (string.IsNullOrEmpty(link.Query))
|
||||
{
|
||||
return link;
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(link)
|
||||
{
|
||||
Query = string.Empty,
|
||||
};
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
private static string? HtmlToPlainText(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var withoutScripts = System.Text.RegularExpressions.Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
var withoutStyles = System.Text.RegularExpressions.Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
var withoutTags = System.Text.RegularExpressions.Regex.Replace(withoutStyles, "<[^>]+>", " ");
|
||||
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
|
||||
return string.IsNullOrWhiteSpace(decoded) ? null : System.Text.RegularExpressions.Regex.Replace(decoded, "\\s+", " ").Trim();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
public sealed record KasperskyFeedItem(
|
||||
string Title,
|
||||
Uri Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary);
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
public sealed record KasperskyFeedItem(
|
||||
string Title,
|
||||
Uri Link,
|
||||
DateTimeOffset Published,
|
||||
string? Summary);
|
||||
|
||||
@@ -1,46 +1,46 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
internal static class KasperskyJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ics-kaspersky:fetch";
|
||||
public const string Parse = "source:ics-kaspersky:parse";
|
||||
public const string Map = "source:ics-kaspersky:map";
|
||||
}
|
||||
|
||||
internal sealed class KasperskyFetchJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyFetchJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KasperskyParseJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyParseJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KasperskyMapJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyMapJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
internal static class KasperskyJobKinds
|
||||
{
|
||||
public const string Fetch = "source:ics-kaspersky:fetch";
|
||||
public const string Parse = "source:ics-kaspersky:parse";
|
||||
public const string Map = "source:ics-kaspersky:map";
|
||||
}
|
||||
|
||||
internal sealed class KasperskyFetchJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyFetchJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KasperskyParseJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyParseJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.ParseAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
internal sealed class KasperskyMapJob : IJob
|
||||
{
|
||||
private readonly KasperskyConnector _connector;
|
||||
|
||||
public KasperskyMapJob(KasperskyConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.MapAsync(context.Services, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Bson;
|
||||
using StellaOps.Concelier.Documents;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
@@ -269,7 +269,7 @@ public sealed class KasperskyConnector : IFeedConnector
|
||||
}
|
||||
|
||||
var dto = KasperskyAdvisoryParser.Parse(advisoryKey, title, link, published, summary, rawBytes);
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ics.kaspersky/1", payload, _timeProvider.GetUtcNow());
|
||||
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
@@ -315,9 +315,9 @@ public sealed class KasperskyConnector : IFeedConnector
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = dto.Payload.ToJson(new StellaOps.Concelier.Bson.IO.JsonWriterSettings
|
||||
var dtoJson = dto.Payload.ToJson(new StellaOps.Concelier.Documents.IO.JsonWriterSettings
|
||||
{
|
||||
OutputMode = StellaOps.Concelier.Bson.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
OutputMode = StellaOps.Concelier.Documents.IO.JsonOutputMode.RelaxedExtendedJson,
|
||||
});
|
||||
|
||||
KasperskyAdvisoryDto advisoryDto;
|
||||
@@ -447,7 +447,7 @@ public sealed class KasperskyConnector : IFeedConnector
|
||||
|
||||
private async Task UpdateCursorAsync(KasperskyCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string? ExtractSlug(Uri link)
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ics-kaspersky";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<KasperskyConnector>(services);
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "ics-kaspersky";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => services is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<KasperskyConnector>(services);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ics-kaspersky";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddKasperskyIcsConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<KasperskyFetchJob>();
|
||||
services.AddTransient<KasperskyParseJob>();
|
||||
services.AddTransient<KasperskyMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, KasperskyJobKinds.Fetch, typeof(KasperskyFetchJob));
|
||||
EnsureJob(options, KasperskyJobKinds.Parse, typeof(KasperskyParseJob));
|
||||
EnsureJob(options, KasperskyJobKinds.Map, typeof(KasperskyMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Concelier.Core.Jobs;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "concelier:sources:ics-kaspersky";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddKasperskyIcsConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<KasperskyFetchJob>();
|
||||
services.AddTransient<KasperskyParseJob>();
|
||||
services.AddTransient<KasperskyMapJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, KasperskyJobKinds.Fetch, typeof(KasperskyFetchJob));
|
||||
EnsureJob(options, KasperskyJobKinds.Parse, typeof(KasperskyParseJob));
|
||||
EnsureJob(options, KasperskyJobKinds.Map, typeof(KasperskyMapJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +1,37 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public static class KasperskyServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddKasperskyIcsConnector(this IServiceCollection services, Action<KasperskyOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<KasperskyOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(KasperskyOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<KasperskyOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.FeedUri;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.IcsKaspersky/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml";
|
||||
});
|
||||
|
||||
services.AddTransient<KasperskyFeedClient>();
|
||||
services.AddTransient<KasperskyConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration;
|
||||
using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal;
|
||||
|
||||
namespace StellaOps.Concelier.Connector.Ics.Kaspersky;
|
||||
|
||||
public static class KasperskyServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddKasperskyIcsConnector(this IServiceCollection services, Action<KasperskyOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<KasperskyOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
services.AddSourceHttpClient(KasperskyOptions.HttpClientName, (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<KasperskyOptions>>().Value;
|
||||
clientOptions.BaseAddress = options.FeedUri;
|
||||
clientOptions.Timeout = TimeSpan.FromSeconds(30);
|
||||
clientOptions.UserAgent = "StellaOps.Concelier.IcsKaspersky/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml";
|
||||
});
|
||||
|
||||
services.AddTransient<KasperskyFeedClient>();
|
||||
services.AddTransient<KasperskyConnector>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user