stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

save progress

Browse Source
This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/DebianConnector.cs
View File

@@ -3,6 +3,8 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -16,8 +18,6 @@ using StellaOps.Concelier.Connector.Distro.Debian.Configuration;
using StellaOps.Concelier.Connector.Distro.Debian.Internal;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Plugin;
@@ -114,7 +114,7 @@ public sealed class DebianConnector : IFeedConnector
throw;
}
var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill);
var lastPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
var processedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
var newProcessedIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
@@ -191,6 +191,16 @@ public sealed class DebianConnector : IFeedConnector
{
cancellationToken.ThrowIfCancellationRequested();
if (entry.Published < lastPublished)
{
continue;
}
if (entry.Published == lastPublished && processedIds.Contains(entry.AdvisoryId))
{
continue;
}
var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId);
var cacheKey = detailUri.ToString();
touchedResources.Add(cacheKey);
@@ -373,7 +383,14 @@ public sealed class DebianConnector : IFeedConnector
}
var payload = ToDocument(dto);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow());
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"debian:dto:{document.Id}:{SchemaVersion}"),
document.Id,
SourceName,
SchemaVersion,
payload,
_timeProvider.GetUtcNow(),
SchemaVersion: SchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -428,19 +445,28 @@ public sealed class DebianConnector : IFeedConnector
continue;
}
var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
try
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
}
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
catch (Exception ex)
{
_logger.LogError(ex, "Debian mapping failed for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
@@ -618,7 +644,11 @@ public sealed class DebianConnector : IFeedConnector
? publishedValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
publishedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => (DateTimeOffset?)null,
}
: null));
@@ -732,4 +762,12 @@ public sealed class DebianConnector : IFeedConnector
}
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

30
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -31,7 +32,11 @@ internal sealed record DebianCursor(
{
lastPublished = lastValue.DocumentType switch
{
DocumentType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
lastValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc),
_ => null,
};
@@ -49,8 +54,8 @@ internal sealed record DebianCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(static id => id.ToString())),
};
if (LastPublished.HasValue)
@@ -60,13 +65,13 @@ internal sealed record DebianCursor(
if (ProcessedAdvisoryIds.Count > 0)
{
document["processedIds"] = new DocumentArray(ProcessedAdvisoryIds);
document["processedIds"] = new DocumentArray(ProcessedAdvisoryIds.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase));
}
if (FetchCache.Count > 0)
{
var cacheDoc = new DocumentObject();
foreach (var (key, entry) in FetchCache)
foreach (var (key, entry) in FetchCache.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase))
{
cacheDoc[key] = entry.ToDocumentObject();
}
@@ -78,10 +83,10 @@ internal sealed record DebianCursor(
}
public DebianCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public DebianCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids)
=> this with
@@ -90,6 +95,7 @@ internal sealed record DebianCursor(
ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray() ?? EmptyIds
};
@@ -134,7 +140,10 @@ internal sealed record DebianCursor(
}
}
return list;
return list
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
@@ -153,7 +162,10 @@ internal sealed record DebianCursor(
}
}
return list;
return list
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
private static IReadOnlyDictionary<string, DebianFetchCacheEntry> ReadCache(DocumentObject document)

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianFetchCacheEntry.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Storage.Contracts;
@@ -33,7 +34,11 @@ internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastM
{
lastModified = modifiedValue.DocumentType switch
{
DocumentType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
modifiedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
_ => null,
};

23
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListParser.cs
View File

@@ -32,7 +32,14 @@ internal static class DebianListParser
continue;
}
if (line[0] == '[')
var trimmed = line.TrimStart();
if (trimmed.Length == 0)
{
continue;
}
if (trimmed[0] == '[')
{
if (currentId is not null && currentTitle is not null && currentPackage is not null)
{
@@ -41,7 +48,9 @@ internal static class DebianListParser
currentDate,
currentTitle,
currentPackage,
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
currentCves.Count == 0
? Array.Empty<string>()
: currentCves.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray()));
}
currentCves.Clear();
@@ -49,7 +58,7 @@ internal static class DebianListParser
currentTitle = null;
currentPackage = null;
var match = HeaderRegex.Match(line);
var match = HeaderRegex.Match(trimmed);
if (!match.Success)
{
continue;
@@ -80,9 +89,9 @@ internal static class DebianListParser
continue;
}
if (line[0] == '{')
if (trimmed[0] == '{')
{
foreach (Match match in CveRegex.Matches(line))
foreach (Match match in CveRegex.Matches(trimmed))
{
if (match.Success && !string.IsNullOrWhiteSpace(match.Value))
{
@@ -99,7 +108,9 @@ internal static class DebianListParser
currentDate,
currentTitle,
currentPackage,
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
currentCves.Count == 0
? Array.Empty<string>()
: currentCves.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray()));
}
return entries;

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0165-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Distro.Debian. |
| AUDIT-0165-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Distro.Debian. |
| AUDIT-0165-A | TODO | Pending approval for changes. |
| AUDIT-0165-A | DONE | Determinism, cursor ordering, and map isolation applied. |