stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 1 Releases Wiki Activity

Rename Concelier Source modules to Connector

Browse Source
This commit is contained in:
master
2025-10-18 20:11:18 +03:00
parent 89ede53cc3
commit 052da7a7d0
789 changed files with 1489 additions and 1489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/StellaOps.Concelier.Connector.CertIn/Internal/CertInAdvisoryDto.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInAdvisoryDto(
string AdvisoryId,
string Title,
string Link,
DateTimeOffset Published,
string? Summary,
string Content,
string? Severity,
ImmutableArray<string> CveIds,
ImmutableArray<string> VendorNames,
ImmutableArray<string> ReferenceLinks);

129
src/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs Normal file
View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertInListingItem>> GetListingsAsync(int page, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
items.Add(item);
}
return items;
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)
{
item = null!;
if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String)
{
return false;
}
var advisoryId = idElement.GetString();
if (string.IsNullOrWhiteSpace(advisoryId))
{
return false;
}
var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String
? titleElement.GetString()
: advisoryId;
if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String)
{
return false;
}
if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri))
{
return false;
}
DateTimeOffset published;
if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String)
{
if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published))
{
return false;
}
}
else
{
return false;
}
string? summary = null;
if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String)
{
summary = summaryElement.GetString();
}
item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim());
return true;
}
private static Uri BuildPageUri(Uri baseUri, int page)
{
if (page <= 1)
{
return baseUri;
}
var builder = new UriBuilder(baseUri);
var trimmed = builder.Query.TrimStart('?');
var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(trimmed)
? pageSegment
: $"{trimmed}&{pageSegment}";
return builder.Uri;
}
}

88
src/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs Normal file
View File

@@ -0,0 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal sealed record CertInCursor(
DateTimeOffset? LastPublished,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
public static CertInCursor Empty { get; } = new(null, Array.Empty<Guid>(), Array.Empty<Guid>());
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
return document;
}
public static CertInCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastPublished = document.TryGetValue("lastPublished", out var dateValue)
? ParseDate(dateValue)
: null;
return new CertInCursor(
lastPublished,
ReadGuidArray(document, "pendingDocuments"),
ReadGuidArray(document, "pendingMappings"));
}
public CertInCursor WithLastPublished(DateTimeOffset? timestamp)
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return Array.Empty<Guid>();
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (element is null)
{
continue;
}
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

187
src/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs Normal file
View File

@@ -0,0 +1,187 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
internal static class CertInDetailParser
{
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
ArgumentNullException.ThrowIfNull(listing);
var html = Encoding.UTF8.GetString(rawHtml);
var content = HtmlToPlainText(html);
var summary = listing.Summary ?? ExtractSummary(content);
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
return new CertInAdvisoryDto(
listing.AdvisoryId,
listing.Title,
listing.DetailUri.ToString(),
listing.Published,
summary,
content,
severity,
cves,
vendors,
references);
}
private static string HtmlToPlainText(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return string.Empty;
}
var withoutScripts = Regex.Replace(html, "<script[\\s\\S]*?</script>", string.Empty, RegexOptions.IgnoreCase);
var withoutStyles = Regex.Replace(withoutScripts, "<style[\\s\\S]*?</style>", string.Empty, RegexOptions.IgnoreCase);
var withoutComments = Regex.Replace(withoutStyles, "<!--.*?-->", string.Empty, RegexOptions.Singleline);
var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " ");
var decoded = System.Net.WebUtility.HtmlDecode(withoutTags);
return string.IsNullOrWhiteSpace(decoded)
? string.Empty
: Regex.Replace(decoded, "\\s+", " ").Trim();
}
private static string? ExtractSummary(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var sentenceTerminators = new[] { ".", "!", "?" };
foreach (var terminator in sentenceTerminators)
{
var index = content.IndexOf(terminator, StringComparison.Ordinal);
if (index > 0)
{
return content[..(index + terminator.Length)].Trim();
}
}
return content.Length > 280 ? content[..280].Trim() : content;
}
private static string? ExtractSeverity(string content)
{
var match = SeverityRegex.Match(content);
if (match.Success)
{
return match.Groups["value"].Value.Trim().ToLowerInvariant();
}
return null;
}
private static ImmutableArray<string> ExtractCves(string title, string? summary, string content)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Capture(string? text)
{
if (string.IsNullOrWhiteSpace(text))
{
return;
}
foreach (Match match in CveRegex.Matches(text))
{
if (match.Success)
{
set.Add(match.Value.ToUpperInvariant());
}
}
}
Capture(title);
Capture(summary);
Capture(content);
return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractVendors(string? summary, string content)
{
var vendors = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
void Add(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return;
}
var cleaned = value
.Replace("’", "'", StringComparison.Ordinal)
.Trim();
if (cleaned.Length > 200)
{
cleaned = cleaned[..200];
}
if (!string.IsNullOrWhiteSpace(cleaned))
{
vendors.Add(cleaned);
}
}
if (!string.IsNullOrWhiteSpace(summary))
{
foreach (Match match in VendorRegex.Matches(summary))
{
Add(match.Groups["value"].Value);
}
}
foreach (Match match in VendorRegex.Matches(content))
{
Add(match.Groups["value"].Value);
}
if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary))
{
var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
Add(fallback);
}
return vendors.Count == 0
? ImmutableArray<string>.Empty
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ImmutableArray<string>.Empty;
}
var links = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in LinkRegex.Matches(html))
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
}
}
return links.Count == 0
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
}

10
src/StellaOps.Concelier.Connector.CertIn/Internal/CertInListingItem.cs Normal file
View File

@@ -0,0 +1,10 @@
using System;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed record CertInListingItem(
string AdvisoryId,
string Title,
Uri DetailUri,
DateTimeOffset Published,
string? Summary);