stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

save progress

Browse Source
This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
@@ -17,20 +18,13 @@ using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnector : IFeedConnector
{
private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web)
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
@@ -123,7 +117,7 @@ public sealed class CccsConnector : IFeedConnector
var documentUri = BuildDocumentUri(item, feed);
var rawDocument = CreateRawDocument(item, feed, result.AlertTypes);
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions);
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, SerializerOptions);
var sha = ComputeSha256(payload);
if (knownHashes.TryGetValue(documentUri, out var existingHash)
@@ -145,7 +139,7 @@ public sealed class CccsConnector : IFeedConnector
continue;
}
var recordId = existing?.Id ?? Guid.NewGuid();
var recordId = existing?.Id ?? CreateDeterministicGuid($"cccs:doc:{documentUri}");
_ = await _rawDocumentStorage.UploadAsync(
SourceName,
@@ -291,7 +285,7 @@ public sealed class CccsConnector : IFeedConnector
CccsRawAdvisoryDocument? raw;
try
{
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, RawSerializerOptions);
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, SerializerOptions);
}
catch (Exception ex)
{
@@ -331,9 +325,16 @@ public sealed class CccsConnector : IFeedConnector
continue;
}
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var dtoJson = JsonSerializer.Serialize(dto, SerializerOptions);
var dtoDoc = DocumentObject.Parse(dtoJson);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoDoc, now);
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"cccs:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
dtoDoc,
now,
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -403,7 +404,7 @@ public sealed class CccsConnector : IFeedConnector
try
{
var json = dtoRecord.Payload.ToJson();
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, DtoSerializerOptions);
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, SerializerOptions);
}
catch (Exception ex)
{
@@ -489,13 +490,14 @@ public sealed class CccsConnector : IFeedConnector
private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed)
{
var candidate = item.Url?.Trim();
Uri? resolved = null;
if (!string.IsNullOrWhiteSpace(candidate))
{
if (Uri.TryCreate(candidate, UriKind.Absolute, out var absolute))
{
if (IsHttpScheme(absolute.Scheme))
{
return absolute.ToString();
resolved = absolute;
}
candidate = absolute.PathAndQuery;
@@ -505,13 +507,18 @@ public sealed class CccsConnector : IFeedConnector
}
}
if (!string.IsNullOrWhiteSpace(candidate) && Uri.TryCreate(CanonicalBaseUri, candidate, out var combined))
if (resolved is null && !string.IsNullOrWhiteSpace(candidate) && Uri.TryCreate(CanonicalBaseUri, candidate, out var combined))
{
return combined.ToString();
resolved = combined;
}
}
return new Uri(CanonicalBaseUri, $"/api/cccs/threats/{feed.Language}/{item.Nid}").ToString();
resolved ??= new Uri(CanonicalBaseUri, $"/api/cccs/threats/{feed.Language}/{item.Nid}");
var builder = new UriBuilder(resolved)
{
Fragment = string.Empty,
};
return builder.Uri.ToString();
}
private static bool IsHttpScheme(string? scheme)
@@ -603,7 +610,8 @@ public sealed class CccsConnector : IFeedConnector
}
var overflow = hashes.Count - maxEntries;
foreach (var key in hashes.Keys.Take(overflow).ToList())
var orderedKeys = hashes.Keys.OrderBy(static key => key, StringComparer.Ordinal).ToList();
foreach (var key in orderedKeys.Take(overflow))
{
hashes.Remove(key);
}
@@ -620,4 +628,12 @@ public sealed class CccsConnector : IFeedConnector
private static string ComputeSha256(byte[] payload)
=> Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

19
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -18,13 +19,19 @@ internal sealed record CccsCursor(
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
var distinct = (documents ?? Enumerable.Empty<Guid>())
.Distinct()
.OrderBy(static id => id)
.ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
var distinct = (mappings ?? Enumerable.Empty<Guid>())
.Distinct()
.OrderBy(static id => id)
.ToArray();
return this with { PendingMappings = distinct };
}
@@ -50,7 +57,7 @@ internal sealed record CccsCursor(
if (KnownEntryHashes.Count > 0)
{
var hashes = new DocumentArray();
foreach (var kvp in KnownEntryHashes)
foreach (var kvp in KnownEntryHashes.OrderBy(static entry => entry.Key, StringComparer.Ordinal))
{
hashes.Add(new DocumentObject
{
@@ -139,7 +146,11 @@ internal sealed record CccsCursor(
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

4
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs
View File

@@ -13,6 +13,7 @@ public sealed class CccsDiagnostics : IDisposable
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _taxonomyFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
@@ -27,6 +28,7 @@ public sealed class CccsDiagnostics : IDisposable
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_taxonomyFailures = _meter.CreateCounter<long>("cccs.fetch.taxonomy.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
@@ -44,6 +46,8 @@ public sealed class CccsDiagnostics : IDisposable
public void FetchFailure() => _fetchFailures.Add(1);
public void TaxonomyFailure() => _taxonomyFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs
View File

@@ -29,10 +29,12 @@ public sealed class CccsFeedClient
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
private readonly CccsDiagnostics _diagnostics;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
public CccsFeedClient(SourceFetchService fetchService, CccsDiagnostics diagnostics, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -108,6 +110,7 @@ public sealed class CccsFeedClient
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
@@ -115,6 +118,7 @@ public sealed class CccsFeedClient
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
@@ -132,11 +136,13 @@ public sealed class CccsFeedClient
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs
View File

@@ -12,8 +12,12 @@ namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SerialRegex = new(
@"(?:(Number|Numero|Num\p{L}*)\s*:\s*)(?<id>[A-Z0-9\-\/]+)",
RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex DateRegex = new(
@"(?:(Date|Date de publication)\s*:\s*)(?<date>[\p{L}0-9,\.\s\-]+)",
RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0149-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Cccs. |
| AUDIT-0149-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Cccs. |
| AUDIT-0149-A | TODO | Pending approval for changes. |
| AUDIT-0149-A | DONE | Applied determinism, cursor ordering, diagnostics, and URI normalization. |