stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

save progress

Browse Source
This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
@@ -14,14 +16,13 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertBund;
public sealed class CertBundConnector : IFeedConnector
{
private const string DtoSchemaVersion = "cert-bund.detail.v1";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
@@ -201,11 +202,7 @@ public sealed class CertBundConnector : IFeedConnector
coverageDays ?? double.NaN);
}
var trimmedKnown = knownAdvisories.Count > _options.MaxKnownAdvisories
? knownAdvisories.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase)
.Take(_options.MaxKnownAdvisories)
.ToArray()
: knownAdvisories.ToArray();
var trimmedKnown = TrimKnownAdvisories(knownAdvisories, feedItems, _options.MaxKnownAdvisories);
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
@@ -287,7 +284,14 @@ public sealed class CertBundConnector : IFeedConnector
parsedCount++;
var doc = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", doc, now);
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"certbund:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
doc,
now,
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -432,4 +436,42 @@ public sealed class CertBundConnector : IFeedConnector
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}
private static string[] TrimKnownAdvisories(
HashSet<string> knownAdvisories,
IReadOnlyList<CertBundFeedItem> feedItems,
int maxEntries)
{
if (knownAdvisories.Count <= maxEntries)
{
return knownAdvisories.ToArray();
}
var recency = feedItems
.GroupBy(item => item.AdvisoryId, StringComparer.OrdinalIgnoreCase)
.ToDictionary(
group => group.Key,
group => group.Max(item => item.Published),
StringComparer.OrdinalIgnoreCase);
return knownAdvisories
.Select(id => new
{
Id = id,
Published = recency.TryGetValue(id, out var published) ? published : DateTimeOffset.MinValue,
})
.OrderByDescending(entry => entry.Published)
.ThenBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase)
.Take(maxEntries)
.Select(entry => entry.Id)
.ToArray();
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -35,9 +36,9 @@ internal sealed record CertBundCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)),
};
if (LastPublished.HasValue)
@@ -74,7 +75,7 @@ internal sealed record CertBundCursor(
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
=> values?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
@@ -92,7 +93,7 @@ internal sealed record CertBundCursor(
}
}
return items;
return items.OrderBy(static id => id).ToArray();
}
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
@@ -105,6 +106,7 @@ internal sealed record CertBundCursor(
return array.Select(element => element?.ToString() ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
@@ -112,7 +114,11 @@ internal sealed record CertBundCursor(
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedClient.cs
View File

@@ -64,6 +64,10 @@ public sealed class CertBundFeedClient
var detailUri = _options.BuildDetailUri(advisoryId);
var pubDateText = element.Element("pubDate")?.Value;
var published = ParseDate(pubDateText);
if (!string.IsNullOrWhiteSpace(pubDateText) && published == DateTimeOffset.MinValue)
{
_logger.LogWarning("CERT-Bund feed item {AdvisoryId} has invalid pubDate {PubDate}", advisoryId, pubDateText);
}
var title = element.Element("title")?.Value?.Trim();
var category = element.Element("category")?.Value?.Trim();
@@ -139,5 +143,5 @@ public sealed class CertBundFeedClient
private static DateTimeOffset ParseDate(string? value)
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: DateTimeOffset.UtcNow;
: DateTimeOffset.MinValue;
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/StellaOps.Concelier.Connector.CertBund.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0151-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.CertBund. |
| AUDIT-0151-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.CertBund. |
| AUDIT-0151-A | TODO | Pending approval for changes. |
| AUDIT-0151-A | DONE | Determinism and warning discipline updates applied. |