stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

save progress

Browse Source
This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/AnalyzerReleases.Unshipped.md
View File

@@ -2,4 +2,4 @@
### Unreleased
- CONCELIER0004: Flag direct `new HttpClient()` usage inside `StellaOps.Concelier.Connector*` namespaces; require sandboxed `IHttpClientFactory` to enforce allow/deny lists.
- CONCELIER0004: Flag direct `new HttpClient()` usage inside `StellaOps.Concelier.Connector*` namespaces; require sandboxed `IHttpClientFactory` to enforce allow/deny lists. Exempts test assemblies and uses symbol-based namespace matching.

50
src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/ConnectorHttpClientSandboxAnalyzer.cs
View File

@@ -1,3 +1,4 @@
using System;
using System.Collections.Immutable;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
@@ -36,18 +37,59 @@ public sealed class ConnectorHttpClientSandboxAnalyzer : DiagnosticAnalyzer
return;
}
var type = context.SemanticModel.GetTypeInfo(objectCreation, context.CancellationToken).Type;
if (type?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat) != "global::System.Net.Http.HttpClient")
var httpClientSymbol = context.Compilation.GetTypeByMetadataName("System.Net.Http.HttpClient");
if (httpClientSymbol is null)
{
return;
}
var containingSymbol = context.ContainingSymbol?.ContainingNamespace?.ToDisplayString();
if (containingSymbol is null || !containingSymbol.StartsWith("StellaOps.Concelier.Connector"))
var createdType = context.SemanticModel
.GetTypeInfo(objectCreation, context.CancellationToken)
.Type;
if (createdType is null || !SymbolEqualityComparer.Default.Equals(createdType, httpClientSymbol))
{
return;
}
var assemblyName = context.ContainingSymbol?.ContainingAssembly?.Name;
if (IsTestAssembly(assemblyName))
{
return;
}
var containingNamespace = context.ContainingSymbol?.ContainingNamespace;
if (!IsConnectorNamespace(containingNamespace))
{
return;
}
context.ReportDiagnostic(Diagnostic.Create(Rule, objectCreation.GetLocation()));
}
private static bool IsTestAssembly(string? assemblyName)
{
if (string.IsNullOrWhiteSpace(assemblyName))
{
return false;
}
return assemblyName.EndsWith(".Tests", StringComparison.OrdinalIgnoreCase)
|| assemblyName.EndsWith(".Test", StringComparison.OrdinalIgnoreCase)
|| assemblyName.EndsWith(".Testing", StringComparison.OrdinalIgnoreCase);
}
private static bool IsConnectorNamespace(INamespaceSymbol? namespaceSymbol)
{
while (namespaceSymbol is not null && !namespaceSymbol.IsGlobalNamespace)
{
if (namespaceSymbol.ToDisplayString().Equals("StellaOps.Concelier.Connector", StringComparison.Ordinal))
{
return true;
}
namespaceSymbol = namespaceSymbol.ContainingNamespace;
}
return false;
}
}

1
src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/StellaOps.Concelier.Analyzers.csproj
View File

@@ -9,6 +9,7 @@
<IncludeBuildOutput>false</IncludeBuildOutput>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0144-M | DONE | Maintainability audit for StellaOps.Concelier.Analyzers. |
| AUDIT-0144-T | DONE | Test coverage audit for StellaOps.Concelier.Analyzers. |
| AUDIT-0144-A | TODO | Pending approval for changes. |
| AUDIT-0144-A | DONE | Applied analyzer hardening + tests. |

53
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/AdvisoryCacheKeys.cs
View File

@@ -5,6 +5,7 @@
// Description: Key schema for Concelier Valkey cache
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Concelier.Cache.Valkey;
@@ -15,13 +16,13 @@ namespace StellaOps.Concelier.Cache.Valkey;
/// <remarks>
/// Key Schema:
/// <code>
/// advisory:{merge_hash} → JSON(CanonicalAdvisory) - TTL based on interest_score
/// rank:hot → ZSET { merge_hash: interest_score } - max 10,000 entries
/// by:purl:{normalized_purl} → SET { merge_hash, ... } - TTL 24h
/// by:cve:{cve_id} → STRING merge_hash - TTL 24h
/// cache:stats:hits → INCR counter
/// cache:stats:misses → INCR counter
/// cache:warmup:last → STRING ISO8601 timestamp
/// advisory:{merge_hash} -> JSON(CanonicalAdvisory) - TTL based on interest_score
/// rank:hot -> ZSET { merge_hash: interest_score } - max 10,000 entries
/// by:purl:{normalized_purl} -> SET { merge_hash, ... } - TTL 24h
/// by:cve:{cve_id} -> STRING merge_hash - TTL 24h
/// cache:stats:hits -> INCR counter
/// cache:stats:misses -> INCR counter
/// cache:warmup:last -> STRING ISO8601 timestamp
/// </code>
/// </remarks>
public static class AdvisoryCacheKeys
@@ -30,6 +31,7 @@ public static class AdvisoryCacheKeys
/// Default key prefix for all cache keys.
/// </summary>
public const string DefaultPrefix = "concelier:";
public const int MaxPurlKeyLength = 500;
/// <summary>
/// Key for advisory by merge hash.
@@ -158,14 +160,20 @@ public static class AdvisoryCacheKeys
}
}
// Truncate if too long (Redis keys can be up to 512MB, but we want reasonable sizes)
const int MaxKeyLength = 500;
if (sb.Length > MaxKeyLength)
var normalizedKey = sb.ToString();
if (normalizedKey.Length <= MaxPurlKeyLength)
{
return sb.ToString(0, MaxKeyLength);
return normalizedKey;
}
return sb.ToString();
var hash = ComputeHash(normalizedKey);
var prefixLength = Math.Max(0, MaxPurlKeyLength - hash.Length - 1);
if (prefixLength == 0)
{
return hash;
}
return normalizedKey[..prefixLength] + "-" + hash;
}
/// <summary>
@@ -215,4 +223,25 @@ public static class AdvisoryCacheKeys
}
return null;
}
private static string ComputeHash(string value)
{
using var sha = SHA256.Create();
var bytes = Encoding.UTF8.GetBytes(value);
var hash = sha.ComputeHash(bytes);
return ToLowerHex(hash);
}
private static string ToLowerHex(byte[] bytes)
{
const string hex = "0123456789abcdef";
var chars = new char[bytes.Length * 2];
for (var i = 0; i < bytes.Length; i++)
{
var b = bytes[i];
chars[i * 2] = hex[b >> 4];
chars[i * 2 + 1] = hex[b & 0xF];
}
return new string(chars);
}
}

22
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/CacheWarmupHostedService.cs
View File

@@ -42,8 +42,12 @@ public sealed class CacheWarmupHostedService : BackgroundService
return;
}
// Wait a short time for the application to fully start
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
var delay = ResolveWarmupDelay(_options);
if (delay > TimeSpan.Zero)
{
_logger?.LogDebug("Cache warmup delay set to {Delay}", delay);
await Task.Delay(delay, stoppingToken).ConfigureAwait(false);
}
_logger?.LogInformation("Starting cache warmup with limit {Limit}", _options.WarmupLimit);
@@ -61,4 +65,18 @@ public sealed class CacheWarmupHostedService : BackgroundService
_logger?.LogError(ex, "Cache warmup failed");
}
}
private static TimeSpan ResolveWarmupDelay(ConcelierCacheOptions options)
{
var delay = options.WarmupDelay;
var jitter = options.WarmupDelayJitter;
if (jitter <= TimeSpan.Zero)
{
return delay;
}
var jitterMillis = Random.Shared.NextDouble() * jitter.TotalMilliseconds;
return delay + TimeSpan.FromMilliseconds(jitterMillis);
}
}

35
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheConnectionFactory.cs
View File

@@ -38,8 +38,7 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
{
_options = options.Value;
_logger = logger;
_connectionFactory = connectionFactory ??
(config => Task.FromResult<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(config)));
_connectionFactory = connectionFactory ?? DefaultConnectionFactory;
}
/// <summary>
@@ -95,7 +94,8 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
_logger?.LogDebug("Connecting to Valkey at {Endpoint} (database {Database})",
_options.ConnectionString, _options.Database);
_connection = await _connectionFactory(config).ConfigureAwait(false);
_connection = await WaitWithCancellationAsync(_connectionFactory(config), cancellationToken)
.ConfigureAwait(false);
_logger?.LogInformation("Connected to Valkey for Concelier cache");
}
@@ -170,4 +170,33 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
_connectionLock.Dispose();
}
private static async Task<T> WaitWithCancellationAsync<T>(Task<T> task, CancellationToken cancellationToken)
{
if (task.IsCompleted || !cancellationToken.CanBeCanceled)
{
return await task.ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(cancellationToken);
}
var cancellationSignal = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
using var registration = cancellationToken.Register(
static state => ((TaskCompletionSource<bool>)state!).TrySetResult(true),
cancellationSignal);
var completed = await Task.WhenAny(task, cancellationSignal.Task).ConfigureAwait(false);
if (completed == cancellationSignal.Task)
{
throw new OperationCanceledException(cancellationToken);
}
return await task.ConfigureAwait(false);
}
private static async Task<IConnectionMultiplexer> DefaultConnectionFactory(ConfigurationOptions options)
=> await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false);
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheMetrics.cs
View File

@@ -144,7 +144,6 @@ public sealed class ConcelierCacheMetrics : IDisposable
public void Dispose()
{
_meter.Dispose();
ActivitySource.Dispose();
}
}

30
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheOptions.cs
View File

@@ -81,6 +81,16 @@ public sealed class ConcelierCacheOptions
/// Number of advisories to preload during warmup.
/// </summary>
public int WarmupLimit { get; set; } = 1000;
/// <summary>
/// Delay before warmup begins.
/// </summary>
public TimeSpan WarmupDelay { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Optional jitter added to the warmup delay.
/// </summary>
public TimeSpan WarmupDelayJitter { get; set; } = TimeSpan.Zero;
}
/// <summary>
@@ -130,16 +140,24 @@ public sealed class CacheTtlPolicy
/// <returns>TTL for the advisory.</returns>
public TimeSpan GetTtl(double? score)
{
if (!score.HasValue)
if (!score.HasValue || double.IsNaN(score.Value))
{
return LowScoreTtl;
}
return score.Value switch
var highThreshold = HighScoreThreshold;
var mediumThreshold = Math.Min(MediumScoreThreshold, highThreshold);
if (score.Value >= highThreshold)
{
>= 0.7 => HighScoreTtl, // High interest: 24h
>= 0.4 => MediumScoreTtl, // Medium interest: 4h
_ => LowScoreTtl // Low interest: 1h
};
return HighScoreTtl;
}
if (score.Value >= mediumThreshold)
{
return MediumScoreTtl;
}
return LowScoreTtl;
}
}

43
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ServiceCollectionExtensions.cs
View File

@@ -101,14 +101,26 @@ public static class ServiceCollectionExtensions
existingDescriptor.ImplementationType,
existingDescriptor.ImplementationType,
existingDescriptor.Lifetime));
services.Add(new ServiceDescriptor(
typeof(ICanonicalAdvisoryServiceInner),
existingDescriptor.ImplementationType,
existingDescriptor.Lifetime));
}
else if (existingDescriptor.ImplementationFactory is not null)
{
services.Add(new ServiceDescriptor(
typeof(StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService),
sp => existingDescriptor.ImplementationFactory(sp),
typeof(ICanonicalAdvisoryServiceInner),
sp => (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationFactory(sp),
existingDescriptor.Lifetime));
}
else if (existingDescriptor.ImplementationInstance is not null)
{
services.Add(new ServiceDescriptor(
typeof(ICanonicalAdvisoryServiceInner),
existingDescriptor.ImplementationInstance));
}
// Register the decorator as the new ICanonicalAdvisoryService
services.Add(new ServiceDescriptor(
@@ -116,28 +128,7 @@ public static class ServiceCollectionExtensions
sp =>
{
// Resolve the inner service (the original implementation)
StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService innerService;
if (existingDescriptor.ImplementationType is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
sp.GetRequiredService(existingDescriptor.ImplementationType);
}
else if (existingDescriptor.ImplementationFactory is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationFactory(sp);
}
else if (existingDescriptor.ImplementationInstance is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationInstance;
}
else
{
throw new InvalidOperationException(
"Unable to resolve inner ICanonicalAdvisoryService for decorator.");
}
var innerService = sp.GetRequiredService<ICanonicalAdvisoryServiceInner>();
var cache = sp.GetRequiredService<IAdvisoryCacheService>();
var logger = sp.GetRequiredService<Microsoft.Extensions.Logging.ILogger<ValkeyCanonicalAdvisoryService>>();
@@ -166,4 +157,8 @@ public static class ServiceCollectionExtensions
services.AddValkeyCachingDecorator();
return services;
}
internal interface ICanonicalAdvisoryServiceInner : StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService
{
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0145-M | DONE | Maintainability audit for StellaOps.Concelier.Cache.Valkey. |
| AUDIT-0145-T | DONE | Test coverage audit for StellaOps.Concelier.Cache.Valkey. |
| AUDIT-0145-A | TODO | Pending approval for changes. |
| AUDIT-0145-A | DOING | Applying cache hardening + tests. |

101
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ValkeyAdvisoryCacheService.cs
View File

@@ -22,6 +22,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
private readonly ConcelierCacheConnectionFactory _connectionFactory;
private readonly ConcelierCacheOptions _options;
private readonly ConcelierCacheMetrics? _metrics;
private readonly ILogger<ValkeyAdvisoryCacheService>? _logger;
private static readonly JsonSerializerOptions JsonOptions = new()
@@ -36,10 +37,12 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
public ValkeyAdvisoryCacheService(
ConcelierCacheConnectionFactory connectionFactory,
IOptions<ConcelierCacheOptions> options,
ConcelierCacheMetrics? metrics = null,
ILogger<ValkeyAdvisoryCacheService>? logger = null)
{
_connectionFactory = connectionFactory;
_options = options.Value;
_metrics = metrics;
_logger = logger;
}
@@ -51,6 +54,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return null;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -60,11 +64,13 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
if (cached.HasValue)
{
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsHits(_options.KeyPrefix)).ConfigureAwait(false);
_metrics?.RecordHit();
_logger?.LogDebug("Cache hit for advisory {MergeHash}", mergeHash);
return JsonSerializer.Deserialize<CanonicalAdvisory>((string)cached!, JsonOptions);
}
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsMisses(_options.KeyPrefix)).ConfigureAwait(false);
_metrics?.RecordMiss();
_logger?.LogDebug("Cache miss for advisory {MergeHash}", mergeHash);
return null;
}
@@ -73,6 +79,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisory {MergeHash} from cache", mergeHash);
return null;
}
finally
{
StopTiming(sw, "get");
}
}
/// <inheritdoc />
@@ -83,6 +93,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return [];
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -116,6 +127,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisories for PURL {Purl}", purl);
return [];
}
finally
{
StopTiming(sw, "get-by-purl");
}
}
/// <inheritdoc />
@@ -126,6 +141,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return null;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -144,6 +160,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisory for CVE {Cve}", cve);
return null;
}
finally
{
StopTiming(sw, "get-by-cve");
}
}
/// <inheritdoc />
@@ -154,6 +174,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return [];
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -189,6 +210,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get hot advisories");
return [];
}
finally
{
StopTiming(sw, "get-hot");
}
}
/// <inheritdoc />
@@ -199,6 +224,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -232,6 +258,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to cache advisory {MergeHash}", advisory.MergeHash);
}
finally
{
StopTiming(sw, "set");
}
}
/// <inheritdoc />
@@ -242,6 +272,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -254,12 +285,17 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
await db.SortedSetRemoveAsync(hotKey, mergeHash).ConfigureAwait(false);
_metrics?.RecordEviction("invalidate");
_logger?.LogDebug("Invalidated advisory {MergeHash}", mergeHash);
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Failed to invalidate advisory {MergeHash}", mergeHash);
}
finally
{
StopTiming(sw, "invalidate");
}
}
/// <inheritdoc />
@@ -270,6 +306,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -283,12 +320,19 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
if (currentSize > _options.MaxHotSetSize)
{
// Remove lowest scoring entries
await db.SortedSetRemoveRangeByRankAsync(
var removed = await db.SortedSetRemoveRangeByRankAsync(
hotKey,
start: 0,
stop: currentSize - _options.MaxHotSetSize - 1).ConfigureAwait(false);
if (removed > 0)
{
_metrics?.RecordEviction("hotset-trim");
}
}
_metrics?.UpdateHotSetSize(Math.Min(currentSize, _options.MaxHotSetSize));
// Update advisory TTL if cached
var advisoryKey = AdvisoryCacheKeys.Advisory(mergeHash, _options.KeyPrefix);
if (await db.KeyExistsAsync(advisoryKey).ConfigureAwait(false))
@@ -303,6 +347,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to update score for {MergeHash}", mergeHash);
}
finally
{
StopTiming(sw, "update-score");
}
}
/// <inheritdoc />
@@ -313,6 +361,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -327,6 +376,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to index PURL {Purl}", purl);
}
finally
{
StopTiming(sw, "index-purl");
}
}
/// <inheritdoc />
@@ -337,6 +390,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -350,6 +404,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to unindex PURL {Purl}", purl);
}
finally
{
StopTiming(sw, "unindex-purl");
}
}
/// <inheritdoc />
@@ -360,6 +418,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -373,6 +432,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to index CVE {Cve}", cve);
}
finally
{
StopTiming(sw, "index-cve");
}
}
/// <inheritdoc />
@@ -424,6 +487,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Cache warmup failed after {Elapsed}ms", sw.ElapsedMilliseconds);
}
finally
{
StopTiming(sw, "warmup");
}
}
/// <inheritdoc />
@@ -434,6 +501,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return new CacheStatistics { IsHealthy = false };
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -446,6 +514,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
var hits = (long)(await db.StringGetAsync(hitsKey).ConfigureAwait(false));
var misses = (long)(await db.StringGetAsync(missesKey).ConfigureAwait(false));
var hotSetSize = await db.SortedSetLengthAsync(hotKey).ConfigureAwait(false);
_metrics?.UpdateHotSetSize(hotSetSize);
DateTimeOffset? lastWarmup = null;
var warmupStr = await db.StringGetAsync(warmupKey).ConfigureAwait(false);
@@ -478,6 +547,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get cache statistics");
return new CacheStatistics { IsHealthy = false };
}
finally
{
StopTiming(sw, "stats");
}
}
/// <inheritdoc />
@@ -488,6 +561,30 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return false;
}
return await _connectionFactory.PingAsync(cancellationToken).ConfigureAwait(false);
var sw = StartTiming();
try
{
return await _connectionFactory.PingAsync(cancellationToken).ConfigureAwait(false);
}
finally
{
StopTiming(sw, "health");
}
}
private Stopwatch? StartTiming()
{
return _metrics is null ? null : Stopwatch.StartNew();
}
private void StopTiming(Stopwatch? sw, string operation)
{
if (sw is null || _metrics is null)
{
return;
}
sw.Stop();
_metrics.RecordLatency(sw.Elapsed.TotalMilliseconds, operation);
}
}

21
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs
View File

@@ -18,8 +18,6 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Plugin;
@@ -27,7 +25,7 @@ namespace StellaOps.Concelier.Connector.Acsc;
public sealed class AcscConnector : IFeedConnector
{
private static readonly string[] AcceptHeaders =
internal static readonly string[] AcceptHeaders =
{
"application/rss+xml",
"application/atom+xml;q=0.9",
@@ -35,6 +33,8 @@ public sealed class AcscConnector : IFeedConnector
"text/xml;q=0.7",
};
internal static readonly string AcceptHeaderValue = string.Join(", ", AcceptHeaders);
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
@@ -293,10 +293,23 @@ public sealed class AcscConnector : IFeedConnector
var json = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = DocumentObject.Parse(json);
if (dto.Entries.Count > 0
&& payload.TryGetValue("entries", out var entriesValue)
&& entriesValue.AsDocumentArray.Count == 0)
{
var fallbackEntries = new DocumentArray();
foreach (var entry in dto.Entries)
{
var entryJson = JsonSerializer.Serialize(entry, SerializerOptions);
fallbackEntries.Add(DocumentObject.Parse(entryJson));
}
payload["entries"] = fallbackEntries;
}
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "acsc.feed.v1", payload, parsedAt)
? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "acsc.feed.v1", payload, parsedAt, SchemaVersion: "acsc.feed.v1")
: existingDto with
{
Payload = payload,

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs
View File

@@ -39,13 +39,7 @@ public static class AcscServiceCollectionExtensions
clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host);
}
clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[]
{
"application/rss+xml",
"application/atom+xml;q=0.9",
"application/xml;q=0.8",
"text/xml;q=0.7",
});
clientOptions.DefaultRequestHeaders["Accept"] = AcscConnector.AcceptHeaderValue;
});
services.AddSingleton<AcscDiagnostics>();

5
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs
View File

@@ -108,6 +108,11 @@ public sealed class AcscOptions
throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified.");
}
if (ForceRelay && RelayEndpoint is null)
{
throw new InvalidOperationException("ACSC ForceRelay requires RelayEndpoint to be configured.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("ACSC RequestTimeout must be positive.");

13
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs
View File

@@ -1,4 +1,5 @@
using System.Globalization;
using System.IO;
using System.Text;
using System.Xml.Linq;
using AngleSharp.Dom;
@@ -27,7 +28,8 @@ internal static class AcscFeedParser
};
}
var xml = XDocument.Parse(Encoding.UTF8.GetString(payload));
using var stream = new MemoryStream(payload, writable: false);
var xml = XDocument.Load(stream, LoadOptions.None);
var (feedTitle, feedLink, feedUpdated) = ExtractFeedMetadata(xml);
var items = ExtractEntries(xml).ToArray();
@@ -230,7 +232,7 @@ internal static class AcscFeedParser
if (builder.Length == 0)
{
return Guid.NewGuid().ToString("n");
return GenerateStableKey(element.ToString(SaveOptions.DisableFormatting));
}
return GenerateStableKey(builder.ToString());
@@ -297,11 +299,6 @@ internal static class AcscFeedParser
return result.ToUniversalTime();
}
if (DateTimeOffset.TryParse(value, CultureInfo.CurrentCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out result))
{
return result.ToUniversalTime();
}
return null;
}
@@ -502,7 +499,7 @@ internal static class AcscFeedParser
return null;
}
value = value.TrimStart(':', '-', '–', '—', ' ');
value = value.TrimStart(':', '-', ' ');
return value.Trim();
}

17
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs
View File

@@ -3,7 +3,6 @@ using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Acsc.Internal;
@@ -56,7 +55,7 @@ internal static class AcscMapper
"mapping",
entry.EntryId ?? entry.Link ?? advisoryKey,
mappedAt.ToUniversalTime(),
fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" });
fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" });
var provenance = new[]
{
@@ -208,7 +207,7 @@ internal static class AcscMapper
{
var provenance = new[]
{
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }),
new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedPackages" }),
};
packages.Add(new AffectedPackage(
@@ -262,9 +261,17 @@ internal static class AcscMapper
: entry.Title;
var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null;
if (string.IsNullOrEmpty(identifier))
if (string.IsNullOrEmpty(identifier) || string.Equals(identifier, "unknown", StringComparison.Ordinal))
{
identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString());
var seed = string.Join(
"|",
feedSlug ?? string.Empty,
entry.EntryId ?? string.Empty,
entry.Link ?? string.Empty,
entry.Title ?? string.Empty,
entry.Summary ?? string.Empty,
entry.Published?.ToUniversalTime().ToString("O") ?? string.Empty);
identifier = CreateHash(seed);
}
return $"{sourceName}/{slug}/{identifier}";

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/StellaOps.Concelier.Connector.Acsc.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Acsc/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0147-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Acsc. |
| AUDIT-0147-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Acsc. |
| AUDIT-0147-A | TODO | Pending approval for changes. |
| AUDIT-0147-A | BLOCKED | AcscConnectorParseTests returning empty DTO entries despite non-empty raw payload. |

54
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
@@ -17,20 +18,13 @@ using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cccs;
public sealed class CccsConnector : IFeedConnector
{
private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web)
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
@@ -123,7 +117,7 @@ public sealed class CccsConnector : IFeedConnector
var documentUri = BuildDocumentUri(item, feed);
var rawDocument = CreateRawDocument(item, feed, result.AlertTypes);
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions);
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, SerializerOptions);
var sha = ComputeSha256(payload);
if (knownHashes.TryGetValue(documentUri, out var existingHash)
@@ -145,7 +139,7 @@ public sealed class CccsConnector : IFeedConnector
continue;
}
var recordId = existing?.Id ?? Guid.NewGuid();
var recordId = existing?.Id ?? CreateDeterministicGuid($"cccs:doc:{documentUri}");
_ = await _rawDocumentStorage.UploadAsync(
SourceName,
@@ -291,7 +285,7 @@ public sealed class CccsConnector : IFeedConnector
CccsRawAdvisoryDocument? raw;
try
{
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, RawSerializerOptions);
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, SerializerOptions);
}
catch (Exception ex)
{
@@ -331,9 +325,16 @@ public sealed class CccsConnector : IFeedConnector
continue;
}
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var dtoJson = JsonSerializer.Serialize(dto, SerializerOptions);
var dtoDoc = DocumentObject.Parse(dtoJson);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoDoc, now);
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"cccs:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
dtoDoc,
now,
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -403,7 +404,7 @@ public sealed class CccsConnector : IFeedConnector
try
{
var json = dtoRecord.Payload.ToJson();
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, DtoSerializerOptions);
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, SerializerOptions);
}
catch (Exception ex)
{
@@ -489,13 +490,14 @@ public sealed class CccsConnector : IFeedConnector
private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed)
{
var candidate = item.Url?.Trim();
Uri? resolved = null;
if (!string.IsNullOrWhiteSpace(candidate))
{
if (Uri.TryCreate(candidate, UriKind.Absolute, out var absolute))
{
if (IsHttpScheme(absolute.Scheme))
{
return absolute.ToString();
resolved = absolute;
}
candidate = absolute.PathAndQuery;
@@ -505,13 +507,18 @@ public sealed class CccsConnector : IFeedConnector
}
}
if (!string.IsNullOrWhiteSpace(candidate) && Uri.TryCreate(CanonicalBaseUri, candidate, out var combined))
if (resolved is null && !string.IsNullOrWhiteSpace(candidate) && Uri.TryCreate(CanonicalBaseUri, candidate, out var combined))
{
return combined.ToString();
resolved = combined;
}
}
return new Uri(CanonicalBaseUri, $"/api/cccs/threats/{feed.Language}/{item.Nid}").ToString();
resolved ??= new Uri(CanonicalBaseUri, $"/api/cccs/threats/{feed.Language}/{item.Nid}");
var builder = new UriBuilder(resolved)
{
Fragment = string.Empty,
};
return builder.Uri.ToString();
}
private static bool IsHttpScheme(string? scheme)
@@ -603,7 +610,8 @@ public sealed class CccsConnector : IFeedConnector
}
var overflow = hashes.Count - maxEntries;
foreach (var key in hashes.Keys.Take(overflow).ToList())
var orderedKeys = hashes.Keys.OrderBy(static key => key, StringComparer.Ordinal).ToList();
foreach (var key in orderedKeys.Take(overflow))
{
hashes.Remove(key);
}
@@ -620,4 +628,12 @@ public sealed class CccsConnector : IFeedConnector
private static string ComputeSha256(byte[] payload)
=> Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

19
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -18,13 +19,19 @@ internal sealed record CccsCursor(
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
{
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
var distinct = (documents ?? Enumerable.Empty<Guid>())
.Distinct()
.OrderBy(static id => id)
.ToArray();
return this with { PendingDocuments = distinct };
}
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
{
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
var distinct = (mappings ?? Enumerable.Empty<Guid>())
.Distinct()
.OrderBy(static id => id)
.ToArray();
return this with { PendingMappings = distinct };
}
@@ -50,7 +57,7 @@ internal sealed record CccsCursor(
if (KnownEntryHashes.Count > 0)
{
var hashes = new DocumentArray();
foreach (var kvp in KnownEntryHashes)
foreach (var kvp in KnownEntryHashes.OrderBy(static entry => entry.Key, StringComparer.Ordinal))
{
hashes.Add(new DocumentObject
{
@@ -139,7 +146,11 @@ internal sealed record CccsCursor(
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

4
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs
View File

@@ -13,6 +13,7 @@ public sealed class CccsDiagnostics : IDisposable
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _taxonomyFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
@@ -27,6 +28,7 @@ public sealed class CccsDiagnostics : IDisposable
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
_taxonomyFailures = _meter.CreateCounter<long>("cccs.fetch.taxonomy.failures", unit: "operations");
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
@@ -44,6 +46,8 @@ public sealed class CccsDiagnostics : IDisposable
public void FetchFailure() => _fetchFailures.Add(1);
public void TaxonomyFailure() => _taxonomyFailures.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs
View File

@@ -29,10 +29,12 @@ public sealed class CccsFeedClient
private readonly SourceFetchService _fetchService;
private readonly ILogger<CccsFeedClient> _logger;
private readonly CccsDiagnostics _diagnostics;
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
public CccsFeedClient(SourceFetchService fetchService, CccsDiagnostics diagnostics, ILogger<CccsFeedClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -108,6 +110,7 @@ public sealed class CccsFeedClient
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
@@ -115,6 +118,7 @@ public sealed class CccsFeedClient
if (taxonomyResponse is null || taxonomyResponse.Error)
{
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
@@ -132,11 +136,13 @@ public sealed class CccsFeedClient
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
{
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
_diagnostics.TaxonomyFailure();
return new Dictionary<int, string>(0);
}
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs
View File

@@ -12,8 +12,12 @@ namespace StellaOps.Concelier.Connector.Cccs.Internal;
public sealed class CccsHtmlParser
{
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SerialRegex = new(
@"(?:(Number|Numero|Num\p{L}*)\s*:\s*)(?<id>[A-Z0-9\-\/]+)",
RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex DateRegex = new(
@"(?:(Date|Date de publication)\s*:\s*)(?<date>[\p{L}0-9,\.\s\-]+)",
RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cccs/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0149-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Cccs. |
| AUDIT-0149-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Cccs. |
| AUDIT-0149-A | TODO | Pending approval for changes. |
| AUDIT-0149-A | DONE | Applied determinism, cursor ordering, diagnostics, and URI normalization. |

58
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
@@ -14,14 +16,13 @@ using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertBund;
public sealed class CertBundConnector : IFeedConnector
{
private const string DtoSchemaVersion = "cert-bund.detail.v1";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
@@ -201,11 +202,7 @@ public sealed class CertBundConnector : IFeedConnector
coverageDays ?? double.NaN);
}
var trimmedKnown = knownAdvisories.Count > _options.MaxKnownAdvisories
? knownAdvisories.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase)
.Take(_options.MaxKnownAdvisories)
.ToArray()
: knownAdvisories.ToArray();
var trimmedKnown = TrimKnownAdvisories(knownAdvisories, feedItems, _options.MaxKnownAdvisories);
var updatedCursor = cursor
.WithPendingDocuments(pendingDocuments)
@@ -287,7 +284,14 @@ public sealed class CertBundConnector : IFeedConnector
parsedCount++;
var doc = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", doc, now);
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"certbund:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
doc,
now,
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -432,4 +436,42 @@ public sealed class CertBundConnector : IFeedConnector
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
}
private static string[] TrimKnownAdvisories(
HashSet<string> knownAdvisories,
IReadOnlyList<CertBundFeedItem> feedItems,
int maxEntries)
{
if (knownAdvisories.Count <= maxEntries)
{
return knownAdvisories.ToArray();
}
var recency = feedItems
.GroupBy(item => item.AdvisoryId, StringComparer.OrdinalIgnoreCase)
.ToDictionary(
group => group.Key,
group => group.Max(item => item.Published),
StringComparer.OrdinalIgnoreCase);
return knownAdvisories
.Select(id => new
{
Id = id,
Published = recency.TryGetValue(id, out var published) ? published : DateTimeOffset.MinValue,
})
.OrderByDescending(entry => entry.Published)
.ThenBy(entry => entry.Id, StringComparer.OrdinalIgnoreCase)
.Take(maxEntries)
.Select(entry => entry.Id)
.ToArray();
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

18
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -35,9 +36,9 @@ internal sealed record CertBundCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString())),
["knownAdvisories"] = new DocumentArray(KnownAdvisories.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)),
};
if (LastPublished.HasValue)
@@ -74,7 +75,7 @@ internal sealed record CertBundCursor(
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
=> values?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
{
@@ -92,7 +93,7 @@ internal sealed record CertBundCursor(
}
}
return items;
return items.OrderBy(static id => id).ToArray();
}
private static IReadOnlyCollection<string> ReadStringArray(DocumentObject document, string field)
@@ -105,6 +106,7 @@ internal sealed record CertBundCursor(
return array.Select(element => element?.ToString() ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
@@ -112,7 +114,11 @@ internal sealed record CertBundCursor(
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedClient.cs
View File

@@ -64,6 +64,10 @@ public sealed class CertBundFeedClient
var detailUri = _options.BuildDetailUri(advisoryId);
var pubDateText = element.Element("pubDate")?.Value;
var published = ParseDate(pubDateText);
if (!string.IsNullOrWhiteSpace(pubDateText) && published == DateTimeOffset.MinValue)
{
_logger.LogWarning("CERT-Bund feed item {AdvisoryId} has invalid pubDate {PubDate}", advisoryId, pubDateText);
}
var title = element.Element("title")?.Value?.Trim();
var category = element.Element("category")?.Value?.Trim();
@@ -139,5 +143,5 @@ public sealed class CertBundFeedClient
private static DateTimeOffset ParseDate(string? value)
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: DateTimeOffset.UtcNow;
: DateTimeOffset.MinValue;
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/StellaOps.Concelier.Connector.CertBund.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertBund/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0151-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.CertBund. |
| AUDIT-0151-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.CertBund. |
| AUDIT-0151-A | TODO | Pending approval for changes. |
| AUDIT-0151-A | DONE | Determinism and warning discipline updates applied. |

26
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs
View File

@@ -3,6 +3,7 @@ using System.Globalization;
using System.Net;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
@@ -24,6 +25,7 @@ namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnector : IFeedConnector
{
private const string DtoSchemaVersion = "certcc.vince.note.v1";
private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
@@ -155,6 +157,10 @@ public sealed class CertCcConnector : IFeedConnector
if (result.IsNotModified)
{
_diagnostics.SummaryFetchUnchanged(request.Scope);
if (existingSummary is not null)
{
await _documentStore.UpdateStatusAsync(existingSummary.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
}
continue;
}
@@ -166,6 +172,11 @@ public sealed class CertCcConnector : IFeedConnector
_diagnostics.SummaryFetchSuccess(request.Scope);
if (result.Document is not null)
{
await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
}
if (!shouldProcessNotes)
{
continue;
@@ -345,12 +356,13 @@ public sealed class CertCcConnector : IFeedConnector
dto.Vulnerabilities.Count);
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
CreateDeterministicGuid($"certcc:dto:{group.Note.Id}:{DtoSchemaVersion}"),
group.Note.Id,
SourceName,
"certcc.vince.note.v1",
DtoSchemaVersion,
payload,
_timeProvider.GetUtcNow());
_timeProvider.GetUtcNow(),
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(group.Note.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -785,4 +797,12 @@ public sealed class CertCcConnector : IFeedConnector
return null;
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs
View File

@@ -42,7 +42,7 @@ public sealed class CertCcOptions
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// When disabled, parse/map stages skip detail mapping -- useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;

23
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs
View File

@@ -1,5 +1,6 @@
using StellaOps.Concelier.Documents;
using System.Globalization;
using StellaOps.Concelier.Connector.Common.Cursors;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
@@ -30,10 +31,10 @@ internal sealed record CertCcCursor(
SummaryState.WriteTo(summary, "start", "end");
document["summary"] = summary;
document["pendingSummaries"] = new DocumentArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new DocumentArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString()));
document["pendingSummaries"] = new DocumentArray(PendingSummaries.OrderBy(static id => id).Select(static id => id.ToString()));
document["pendingNotes"] = new DocumentArray(PendingNotes.OrderBy(static note => note, StringComparer.OrdinalIgnoreCase));
document["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(static id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(static id => id.ToString()));
if (LastRun.HasValue)
{
@@ -67,7 +68,11 @@ internal sealed record CertCcCursor(
lastRun = lastRunValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
lastRunValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -109,7 +114,7 @@ internal sealed record CertCcCursor(
}
}
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
return results.Count == 0 ? EmptyGuidArray : results.Distinct().OrderBy(static id => id).ToArray();
}
private static string[] ReadStringArray(DocumentObject document, string field)
@@ -139,6 +144,7 @@ internal sealed record CertCcCursor(
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
@@ -174,7 +180,7 @@ internal sealed record CertCcCursor(
}
private static Guid[] NormalizeGuidSet(IEnumerable<Guid>? ids)
=> ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray;
=> ids?.Where(static id => id != Guid.Empty).Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidArray;
private static string[] NormalizeStringSet(IEnumerable<string>? values)
=> values is null
@@ -183,5 +189,6 @@ internal sealed record CertCcCursor(
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray();
}

35
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs
View File

@@ -3,10 +3,11 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
@@ -31,7 +32,7 @@ internal static class CertCcMapper
var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty;
var advisoryKey = BuildAdvisoryKey(metadata);
var advisoryKey = BuildAdvisoryKey(metadata, document);
var title = string.IsNullOrWhiteSpace(metadata.Title) ? advisoryKey : metadata.Title.Trim();
var summary = ExtractSummary(metadata);
@@ -61,12 +62,9 @@ internal static class CertCcMapper
provenance);
}
private static string BuildAdvisoryKey(CertCcNoteMetadata metadata)
private static string BuildAdvisoryKey(CertCcNoteMetadata metadata, DocumentRecord document)
{
if (metadata is null)
{
return $"{AdvisoryPrefix}/{Guid.NewGuid():N}";
}
var fallbackKey = BuildFallbackKey(document);
var vuKey = NormalizeVuId(metadata.VuId);
if (vuKey.Length > 0)
@@ -80,9 +78,30 @@ internal static class CertCcMapper
return $"{AdvisoryPrefix}/vu-{id}";
}
return $"{AdvisoryPrefix}/{Guid.NewGuid():N}";
return fallbackKey;
}
private static string BuildFallbackKey(DocumentRecord document)
{
if (document.Metadata is not null
&& document.Metadata.TryGetValue("certcc.noteId", out var noteId)
&& !string.IsNullOrWhiteSpace(noteId))
{
var normalized = SanitizeToken(noteId);
if (normalized.Length > 0)
{
return $"{AdvisoryPrefix}/note-{normalized}";
}
}
var source = document.Uri ?? string.Empty;
var hash = ComputeSha256(source);
return $"{AdvisoryPrefix}/doc-{hash}";
}
private static string ComputeSha256(string value)
=> Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty))).ToLowerInvariant();
private static string NormalizeVuId(string? value)
{
if (string.IsNullOrWhiteSpace(value))

26
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs
View File

@@ -361,7 +361,7 @@ internal static class CertCcNoteParser
{
if (index > start)
{
AppendSegment(span, start, index - start, baseUri, buffer, ref count);
AppendSegment(span, start, index - start, baseUri, ref buffer, ref count);
}
if (ch == '\r' && index + 1 < span.Length && span[index + 1] == '\n')
@@ -375,7 +375,7 @@ internal static class CertCcNoteParser
if (start < span.Length)
{
AppendSegment(span, start, span.Length - start, baseUri, buffer, ref count);
AppendSegment(span, start, span.Length - start, baseUri, ref buffer, ref count);
}
if (count == 0)
@@ -395,7 +395,7 @@ internal static class CertCcNoteParser
}
}
private static void AppendSegment(ReadOnlySpan<char> span, int start, int length, Uri? baseUri, string[] buffer, ref int count)
private static void AppendSegment(ReadOnlySpan<char> span, int start, int length, Uri? baseUri, ref string[] buffer, ref int count)
{
var segment = span.Slice(start, length).ToString().Trim();
if (segment.Length == 0)
@@ -408,12 +408,28 @@ internal static class CertCcNoteParser
return;
}
if (count >= buffer.Length)
EnsureCapacity(ref buffer, count + 1);
buffer[count++] = normalized.ToString();
}
private static void EnsureCapacity(ref string[] buffer, int required)
{
if (required <= buffer.Length)
{
return;
}
buffer[count++] = normalized.ToString();
var nextSize = buffer.Length * 2;
if (nextSize < required)
{
nextSize = required;
}
var next = ArrayPool<string>.Shared.Rent(nextSize);
Array.Copy(buffer, next, buffer.Length);
ArrayPool<string>.Shared.Return(buffer, clearArray: true);
buffer = next;
}
private static IReadOnlyList<string> ExtractCveIds(JsonElement element, string propertyName)

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
@@ -11,15 +11,14 @@ internal static class CertCcVendorStatementParser
{
"\t",
" - ",
" – ",
" — ",
" -- ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] BulletPrefixes = { '-', '*', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/StellaOps.Concelier.Connector.CertCc.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0153-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.CertCc. |
| AUDIT-0153-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.CertCc. |
| AUDIT-0153-A | TODO | Pending approval for changes. |
| AUDIT-0153-A | DONE | Determinism and parser fixes applied. |

47
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrConnector.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -11,14 +13,13 @@ using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertFr;
public sealed class CertFrConnector : IFeedConnector
{
private const string DtoSchemaVersion = "certfr.detail.v1";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
@@ -34,6 +35,7 @@ public sealed class CertFrConnector : IFeedConnector
private readonly ISourceStateRepository _stateRepository;
private readonly CertFrOptions _options;
private readonly TimeProvider _timeProvider;
private readonly CertFrDiagnostics _diagnostics;
private readonly ILogger<CertFrConnector> _logger;
public CertFrConnector(
@@ -45,6 +47,7 @@ public sealed class CertFrConnector : IFeedConnector
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<CertFrOptions> options,
CertFrDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<CertFrConnector> logger)
{
@@ -57,6 +60,7 @@ public sealed class CertFrConnector : IFeedConnector
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -229,6 +233,7 @@ public sealed class CertFrConnector : IFeedConnector
catch (Exception ex)
{
_logger.LogError(ex, "Cert-FR parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri);
_diagnostics.ParseFailure("parse_error");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingDocuments.Remove(documentId);
pendingMappings.Remove(documentId);
@@ -241,16 +246,24 @@ public sealed class CertFrConnector : IFeedConnector
var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false);
var dtoRecord = existingDto is null
? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certfr.detail.v1", payload, validatedAt)
? new DtoRecord(
CreateDeterministicGuid($"certfr:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
payload,
validatedAt,
SchemaVersion: DtoSchemaVersion)
: existingDto with
{
Payload = payload,
SchemaVersion = "certfr.detail.v1",
SchemaVersion = DtoSchemaVersion,
ValidatedAt = validatedAt,
};
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
_diagnostics.ParseSuccess();
pendingDocuments.Remove(documentId);
if (!pendingMappings.Contains(documentId))
@@ -311,10 +324,20 @@ public sealed class CertFrConnector : IFeedConnector
continue;
}
var mappedAt = _timeProvider.GetUtcNow();
var advisory = CertFrMapper.Map(dto, SourceName, mappedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
try
{
var mappedAt = _timeProvider.GetUtcNow();
var advisory = CertFrMapper.Map(dto, SourceName, mappedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
_diagnostics.MapSuccess();
}
catch (Exception ex)
{
_logger.LogError(ex, "Cert-FR mapping failed for document {DocumentId}", documentId);
_diagnostics.MapFailure("map_error");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
}
@@ -334,4 +357,12 @@ public sealed class CertFrConnector : IFeedConnector
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/CertFrServiceCollectionExtensions.cs
View File

@@ -29,6 +29,7 @@ public static class CertFrServiceCollectionExtensions
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
});
services.TryAddSingleton<CertFrDiagnostics>();
services.TryAddSingleton<CertFrFeedClient>();
services.AddTransient<CertFrConnector>();
return services;

20
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -16,8 +17,8 @@ internal sealed record CertFrCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString())),
};
if (LastPublished.HasValue)
@@ -49,16 +50,20 @@ internal sealed record CertFrCursor(
=> this with { LastPublished = timestamp };
public CertFrCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>() };
public CertFrCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
@@ -83,6 +88,9 @@ internal sealed record CertFrCursor(
}
}
return result;
return result
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
}

87
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDiagnostics.cs Normal file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
public sealed class CertFrDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertFr";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _feedFetchAttempts;
private readonly Counter<long> _feedFetchSuccess;
private readonly Counter<long> _feedFetchFailures;
private readonly Histogram<long> _feedItemCount;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CertFrDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_feedFetchAttempts = _meter.CreateCounter<long>(
name: "certfr.feed.fetch.attempts",
unit: "operations",
description: "Number of RSS feed load attempts.");
_feedFetchSuccess = _meter.CreateCounter<long>(
name: "certfr.feed.fetch.success",
unit: "operations",
description: "Number of successful RSS feed loads.");
_feedFetchFailures = _meter.CreateCounter<long>(
name: "certfr.feed.fetch.failures",
unit: "operations",
description: "Number of RSS feed load failures.");
_feedItemCount = _meter.CreateHistogram<long>(
name: "certfr.feed.items.count",
unit: "items",
description: "Distribution of RSS item counts per fetch.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certfr.parse.success",
unit: "documents",
description: "Number of CERT-FR documents parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certfr.parse.failures",
unit: "documents",
description: "Number of CERT-FR documents that failed to parse.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certfr.map.success",
unit: "advisories",
description: "Number of CERT-FR advisories mapped successfully.");
_mapFailures = _meter.CreateCounter<long>(
name: "certfr.map.failures",
unit: "advisories",
description: "Number of CERT-FR advisories that failed to map.");
}
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
public void FeedFetchSuccess(int itemCount)
{
_feedFetchSuccess.Add(1);
if (itemCount >= 0)
{
_feedItemCount.Record(itemCount);
}
}
public void FeedFetchFailure(string reason = "error")
=> _feedFetchFailures.Add(1, ReasonTag(reason));
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure(string reason = "error")
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure(string reason = "error")
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDocumentMetadata.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
@@ -36,7 +37,12 @@ internal sealed record CertFrDocumentMetadata(
throw new InvalidOperationException("Cert-FR title metadata missing.");
}
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published))
if (!metadata.TryGetValue(PublishedKey, out var publishedRaw)
|| !DateTimeOffset.TryParse(
publishedRaw,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var published))
{
throw new InvalidOperationException("Cert-FR published metadata invalid.");
}

89
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedClient.cs
View File

@@ -16,13 +16,19 @@ public sealed class CertFrFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertFrOptions _options;
private readonly CertFrDiagnostics _diagnostics;
private readonly ILogger<CertFrFeedClient> _logger;
public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions<CertFrOptions> options, ILogger<CertFrFeedClient> logger)
public CertFrFeedClient(
IHttpClientFactory httpClientFactory,
IOptions<CertFrOptions> options,
CertFrDiagnostics diagnostics,
ILogger<CertFrFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -30,45 +36,62 @@ public sealed class CertFrFeedClient
{
var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName);
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_diagnostics.FeedFetchAttempt();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
var now = DateTimeOffset.UtcNow;
foreach (var itemElement in document.Descendants("item"))
try
{
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertFrFeedItem>();
foreach (var itemElement in document.Descendants("item"))
{
continue;
var link = itemElement.Element("link")?.Value;
if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri))
{
continue;
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
var published = ParsePublished(itemElement.Element("pubDate")?.Value);
if (!published.HasValue)
{
_logger.LogWarning("Cert-FR feed item {AdvisoryId} has invalid pubDate", detailUri);
continue;
}
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
}
var title = itemElement.Element("title")?.Value?.Trim();
var summary = itemElement.Element("description")?.Value?.Trim();
_diagnostics.FeedFetchSuccess(items.Count);
var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now;
if (published < windowStart)
{
continue;
}
if (published > windowEnd)
{
published = windowEnd;
}
var advisoryId = ResolveAdvisoryId(itemElement, detailUri);
items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary));
return items
.OrderByDescending(item => item.Published)
.ThenBy(item => item.AdvisoryId, StringComparer.OrdinalIgnoreCase)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
catch (Exception ex)
{
_diagnostics.FeedFetchFailure(ex.GetType().Name);
throw;
}
return items
.OrderBy(item => item.Published)
.Take(_options.MaxItemsPerFetch)
.ToArray();
}
private static DateTimeOffset? ParsePublished(string? value)

16
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/Internal/CertFrParser.cs
View File

@@ -2,12 +2,13 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.CertFr.Internal;
internal static class CertFrParser
{
private static readonly Regex AnchorRegex = new("<a[^>]+href=\"(?<url>https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex AnchorRegex = new("<a[^>]+href\\s*=\\s*(?:\"(?<url>[^\"]+)\"|'(?<url>[^']+)'|(?<url>[^\\s>]+))", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex ScriptRegex = new("<script[\\s\\S]*?</script>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StyleRegex = new("<style[\\s\\S]*?</style>", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled);
@@ -20,7 +21,7 @@ internal static class CertFrParser
var sanitized = SanitizeHtml(html);
var summary = BuildSummary(metadata.Summary, sanitized);
var references = ExtractReferences(html);
var references = ExtractReferences(html, metadata.DetailUri);
return new CertFrDto(
metadata.AdvisoryId,
@@ -62,14 +63,21 @@ internal static class CertFrParser
return content.Length > 280 ? content[..280].Trim() : content;
}
private static IReadOnlyList<string> ExtractReferences(string html)
private static IReadOnlyList<string> ExtractReferences(string html, Uri baseUri)
{
var references = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (Match match in AnchorRegex.Matches(html))
{
if (match.Success)
{
references.Add(match.Groups["url"].Value.Trim());
var candidate = match.Groups["url"].Value.Trim();
if (UrlNormalizer.TryNormalize(candidate, baseUri, out var normalized, stripFragment: true, forceHttps: false)
&& normalized is not null
&& (string.Equals(normalized.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
|| string.Equals(normalized.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)))
{
references.Add(normalized.ToString());
}
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/StellaOps.Concelier.Connector.CertFr.csproj
View File

@@ -4,6 +4,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertFr/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0155-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.CertFr. |
| AUDIT-0155-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.CertFr. |
| AUDIT-0155-A | TODO | Pending approval for changes. |
| AUDIT-0155-A | DONE | Determinism, ordering, and parser fixes applied. |

148
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInConnector.cs
View File

@@ -1,6 +1,9 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
@@ -14,14 +17,13 @@ using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertIn;
public sealed class CertInConnector : IFeedConnector
{
private const string DtoSchemaVersion = "certin.v1";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General)
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
@@ -38,6 +40,7 @@ public sealed class CertInConnector : IFeedConnector
private readonly ISourceStateRepository _stateRepository;
private readonly CertInOptions _options;
private readonly TimeProvider _timeProvider;
private readonly CertInDiagnostics _diagnostics;
private readonly ILogger<CertInConnector> _logger;
public CertInConnector(
@@ -49,6 +52,7 @@ public sealed class CertInConnector : IFeedConnector
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<CertInOptions> options,
CertInDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<CertInConnector> logger)
{
@@ -61,6 +65,7 @@ public sealed class CertInConnector : IFeedConnector
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -106,6 +111,11 @@ public sealed class CertInConnector : IFeedConnector
break;
}
if (listing.Published > maxPublished)
{
maxPublished = listing.Published;
}
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["certin.advisoryId"] = listing.AdvisoryId,
@@ -155,10 +165,6 @@ public sealed class CertInConnector : IFeedConnector
}
pendingDocuments.Add(result.Document.Id);
if (listing.Published > maxPublished)
{
maxPublished = listing.Published;
}
if (_options.RequestDelay > TimeSpan.Zero)
{
@@ -201,6 +207,7 @@ public sealed class CertInConnector : IFeedConnector
if (!document.PayloadId.HasValue)
{
_logger.LogWarning("CERT-In document {DocumentId} missing GridFS payload", document.Id);
_diagnostics.ParseFailure("missing_payload");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
continue;
@@ -209,6 +216,7 @@ public sealed class CertInConnector : IFeedConnector
if (!TryDeserializeListing(document.Metadata, out var listing))
{
_logger.LogWarning("CERT-In metadata missing for {DocumentId}", document.Id);
_diagnostics.ParseFailure("missing_metadata");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
remainingDocuments.Remove(documentId);
continue;
@@ -227,10 +235,18 @@ public sealed class CertInConnector : IFeedConnector
var dto = CertInDetailParser.Parse(listing, rawBytes);
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow());
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"certin:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
payload,
_timeProvider.GetUtcNow(),
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
_diagnostics.ParseSuccess();
remainingDocuments.Remove(documentId);
if (!pendingMappings.Contains(documentId))
@@ -291,8 +307,18 @@ public sealed class CertInConnector : IFeedConnector
}
var advisory = MapAdvisory(dto, document, dtoRecord);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
try
{
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
_diagnostics.MapSuccess();
}
catch (Exception ex)
{
_logger.LogError(ex, "CERT-In mapping failed for {DocumentId}", document.Id);
_diagnostics.MapFailure("map_error");
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
}
@@ -306,6 +332,7 @@ public sealed class CertInConnector : IFeedConnector
var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt);
var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt);
var advisoryKey = NormalizeAdvisoryKey(dto.AdvisoryId);
var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
dto.AdvisoryId,
@@ -316,14 +343,10 @@ public sealed class CertInConnector : IFeedConnector
}
var references = new List<AdvisoryReference>();
var referenceUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
try
{
references.Add(new AdvisoryReference(
dto.Link,
"advisory",
"cert-in",
null,
new AdvisoryProvenance(SourceName, "reference", dto.Link, dtoRecord.ValidatedAt)));
TryAddReference(references, referenceUrls, dto.Link, "advisory", "cert-in", dtoRecord.ValidatedAt);
}
catch (ArgumentException)
{
@@ -333,39 +356,17 @@ public sealed class CertInConnector : IFeedConnector
foreach (var cve in dto.CveIds)
{
var url = $"https://www.cve.org/CVERecord?id={cve}";
try
{
references.Add(new AdvisoryReference(
url,
"advisory",
cve,
null,
new AdvisoryProvenance(SourceName, "reference", url, dtoRecord.ValidatedAt)));
}
catch (ArgumentException)
{
// ignore invalid urls
}
TryAddReference(references, referenceUrls, url, "advisory", cve, dtoRecord.ValidatedAt);
}
foreach (var link in dto.ReferenceLinks)
{
try
{
references.Add(new AdvisoryReference(
link,
"reference",
null,
null,
new AdvisoryProvenance(SourceName, "reference", link, dtoRecord.ValidatedAt)));
}
catch (ArgumentException)
{
// ignore invalid urls
}
TryAddReference(references, referenceUrls, link, "reference", null, dtoRecord.ValidatedAt);
}
var affectedPackages = dto.VendorNames.Select(vendor =>
var affectedPackages = dto.VendorNames
.OrderBy(static vendor => vendor, StringComparer.OrdinalIgnoreCase)
.Select(vendor =>
{
var provenance = new AdvisoryProvenance(SourceName, "affected", vendor, dtoRecord.ValidatedAt);
var primitives = new RangePrimitives(
@@ -400,7 +401,7 @@ public sealed class CertInConnector : IFeedConnector
.ToArray();
return new Advisory(
dto.AdvisoryId,
advisoryKey,
dto.Title,
dto.Summary ?? dto.Content,
language: "en",
@@ -449,7 +450,12 @@ public sealed class CertInConnector : IFeedConnector
return false;
}
if (!metadata.TryGetValue("certin.published", out var publishedText) || !DateTimeOffset.TryParse(publishedText, out var published))
if (!metadata.TryGetValue("certin.published", out var publishedText)
|| !DateTimeOffset.TryParse(
publishedText,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var published))
{
return false;
}
@@ -459,4 +465,58 @@ public sealed class CertInConnector : IFeedConnector
listing = new CertInListingItem(advisoryId, title, detailUri, published.ToUniversalTime(), summary);
return true;
}
private static string NormalizeAdvisoryKey(string advisoryId)
{
if (string.IsNullOrWhiteSpace(advisoryId))
{
return "certin/unknown";
}
return advisoryId.StartsWith("certin/", StringComparison.OrdinalIgnoreCase)
? advisoryId.Trim()
: $"certin/{advisoryId.Trim()}";
}
private void TryAddReference(
ICollection<AdvisoryReference> references,
ISet<string> referenceUrls,
string? url,
string kind,
string? sourceTag,
DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
var trimmed = url.Trim();
if (!referenceUrls.Add(trimmed))
{
return;
}
try
{
references.Add(new AdvisoryReference(
trimmed,
kind,
sourceTag,
null,
new AdvisoryProvenance(SourceName, "reference", trimmed, recordedAt)));
}
catch (ArgumentException)
{
// ignore invalid urls
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/CertInServiceCollectionExtensions.cs
View File

@@ -1,5 +1,6 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertIn.Configuration;
using StellaOps.Concelier.Connector.CertIn.Internal;
@@ -29,6 +30,7 @@ public static class CertInServiceCollectionExtensions
clientOptions.DefaultRequestHeaders["Accept"] = "application/json";
});
services.TryAddSingleton<CertInDiagnostics>();
services.AddTransient<CertInClient>();
services.AddTransient<CertInConnector>();

55
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs
View File

@@ -17,13 +17,19 @@ public sealed class CertInClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertInOptions _options;
private readonly CertInDiagnostics _diagnostics;
private readonly ILogger<CertInClient> _logger;
public CertInClient(IHttpClientFactory httpClientFactory, IOptions<CertInOptions> options, ILogger<CertInClient> logger)
public CertInClient(
IHttpClientFactory httpClientFactory,
IOptions<CertInOptions> options,
CertInDiagnostics diagnostics,
ILogger<CertInClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -32,31 +38,42 @@ public sealed class CertInClient
var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName);
var requestUri = BuildPageUri(_options.AlertsEndpoint, page);
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_diagnostics.ListingFetchAttempt();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
try
{
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Array)
{
continue;
_logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri);
return Array.Empty<CertInListingItem>();
}
items.Add(item);
}
var items = new List<CertInListingItem>(capacity: root.GetArrayLength());
foreach (var element in root.EnumerateArray())
{
if (!TryParseListing(element, out var item))
{
continue;
}
return items;
items.Add(item);
}
_diagnostics.ListingFetchSuccess(items.Count);
return items;
}
catch (Exception ex)
{
_diagnostics.ListingFetchFailure(ex.GetType().Name);
throw;
}
}
private static bool TryParseListing(JsonElement element, out CertInListingItem item)

20
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -16,8 +17,8 @@ internal sealed record CertInCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString())),
};
if (LastPublished.HasValue)
@@ -49,16 +50,20 @@ internal sealed record CertInCursor(
=> this with { LastPublished = timestamp };
public CertInCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>() };
public CertInCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty<Guid>() };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>() };
private static DateTimeOffset? ParseDate(DocumentValue value)
=> value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
@@ -83,6 +88,9 @@ internal sealed record CertInCursor(
}
}
return results;
return results
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
}

64
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs
View File

@@ -1,9 +1,10 @@
using System;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using StellaOps.Concelier.Connector.Common.Url;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
@@ -12,7 +13,7 @@ internal static class CertInDetailParser
private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?<value>[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?<value>[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex LinkRegex = new("href\\s*=\\s*(?:\"(?<url>[^\"]+)\"|'(?<url>[^']+)'|(?<url>[^\\s>]+))", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml)
{
@@ -24,7 +25,7 @@ internal static class CertInDetailParser
var severity = ExtractSeverity(content);
var cves = ExtractCves(listing.Title, summary, content);
var vendors = ExtractVendors(summary, content);
var references = ExtractLinks(html);
var references = ExtractLinks(html, listing.DetailUri);
return new CertInAdvisoryDto(
listing.AdvisoryId,
@@ -125,9 +126,7 @@ internal static class CertInDetailParser
return;
}
var cleaned = value
.Replace("’", "'", StringComparison.Ordinal)
.Trim();
var cleaned = NormalizeVendorText(value).Trim();
if (cleaned.Length > 200)
{
@@ -164,7 +163,7 @@ internal static class CertInDetailParser
: vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static ImmutableArray<string> ExtractLinks(string html)
private static ImmutableArray<string> ExtractLinks(string html, Uri baseUri)
{
if (string.IsNullOrWhiteSpace(html))
{
@@ -176,7 +175,14 @@ internal static class CertInDetailParser
{
if (match.Success)
{
links.Add(match.Groups[1].Value);
var candidate = match.Groups["url"].Value.Trim();
if (UrlNormalizer.TryNormalize(candidate, baseUri, out var normalized, stripFragment: true, forceHttps: false)
&& normalized is not null
&& (string.Equals(normalized.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
|| string.Equals(normalized.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)))
{
links.Add(normalized.ToString());
}
}
}
@@ -184,4 +190,46 @@ internal static class CertInDetailParser
? ImmutableArray<string>.Empty
: links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray();
}
private static string NormalizeVendorText(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var normalized = value
.Replace("\u00E2\u20AC\u2122", "'", StringComparison.Ordinal)
.Replace("\u00E2\u20AC\u201C", "-", StringComparison.Ordinal)
.Replace("\u00E2\u20AC\u201D", "-", StringComparison.Ordinal)
.Replace("\u00C6\u2019", "'", StringComparison.Ordinal);
var builder = new StringBuilder(normalized.Length);
foreach (var ch in normalized)
{
if (ch <= 0x7F)
{
builder.Append(ch);
continue;
}
switch (ch)
{
case '\u2019':
case '\u2018':
builder.Append('\'');
break;
case '\u201C':
case '\u201D':
builder.Append('"');
break;
case '\u2013':
case '\u2014':
builder.Append('-');
break;
}
}
return builder.ToString();
}
}

87
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/Internal/CertInDiagnostics.cs Normal file
View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.CertIn.Internal;
public sealed class CertInDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertIn";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _listingFetchAttempts;
private readonly Counter<long> _listingFetchSuccess;
private readonly Counter<long> _listingFetchFailures;
private readonly Histogram<long> _listingCount;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CertInDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_listingFetchAttempts = _meter.CreateCounter<long>(
name: "certin.listings.fetch.attempts",
unit: "operations",
description: "Number of CERT-In listings fetch attempts.");
_listingFetchSuccess = _meter.CreateCounter<long>(
name: "certin.listings.fetch.success",
unit: "operations",
description: "Number of successful CERT-In listings fetches.");
_listingFetchFailures = _meter.CreateCounter<long>(
name: "certin.listings.fetch.failures",
unit: "operations",
description: "Number of failed CERT-In listings fetches.");
_listingCount = _meter.CreateHistogram<long>(
name: "certin.listings.items.count",
unit: "items",
description: "Distribution of listings returned per page.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certin.parse.success",
unit: "documents",
description: "Number of CERT-In documents parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certin.parse.failures",
unit: "documents",
description: "Number of CERT-In documents that failed to parse.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certin.map.success",
unit: "advisories",
description: "Number of CERT-In advisories mapped successfully.");
_mapFailures = _meter.CreateCounter<long>(
name: "certin.map.failures",
unit: "advisories",
description: "Number of CERT-In advisories that failed to map.");
}
public void ListingFetchAttempt() => _listingFetchAttempts.Add(1);
public void ListingFetchSuccess(int itemCount)
{
_listingFetchSuccess.Add(1);
if (itemCount >= 0)
{
_listingCount.Record(itemCount);
}
}
public void ListingFetchFailure(string reason = "error")
=> _listingFetchFailures.Add(1, ReasonTag(reason));
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure(string reason = "error")
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure(string reason = "error")
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertIn/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0157-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.CertIn. |
| AUDIT-0157-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.CertIn. |
| AUDIT-0157-A | TODO | Pending approval for changes. |
| AUDIT-0157-A | DONE | Determinism, ordering, and parser fixes applied. |

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs
View File

@@ -1,3 +1,4 @@
using System.Globalization;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Common.Cursors;
@@ -69,7 +70,11 @@ public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, Date
return value.DocumentType switch
{
DocumentType.DateTime => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

41
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/RawDocumentStorage.cs
View File

@@ -1,5 +1,7 @@
using System.Collections.Concurrent;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Common.Fetch;
@@ -9,12 +11,16 @@ namespace StellaOps.Concelier.Connector.Common.Fetch;
/// </summary>
public sealed class RawDocumentStorage
{
private readonly ConcurrentDictionary<Guid, byte[]> _blobs = new();
private readonly IDocumentStore? _documentStore;
private sealed record RawDocumentEntry(byte[] Payload, DateTimeOffset? ExpiresAt);
public RawDocumentStorage(IDocumentStore? documentStore = null)
private readonly ConcurrentDictionary<Guid, RawDocumentEntry> _blobs = new();
private readonly IDocumentStore? _documentStore;
private readonly TimeProvider _timeProvider;
public RawDocumentStorage(IDocumentStore? documentStore = null, TimeProvider? timeProvider = null)
{
_documentStore = documentStore;
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task<Guid> UploadAsync(
@@ -37,20 +43,30 @@ public sealed class RawDocumentStorage
ArgumentException.ThrowIfNullOrEmpty(sourceName);
ArgumentException.ThrowIfNullOrEmpty(uri);
ArgumentNullException.ThrowIfNull(content);
cancellationToken.ThrowIfCancellationRequested();
var id = documentId ?? Guid.NewGuid();
var id = documentId ?? CreateDeterministicGuid($"{sourceName}:{uri}");
var copy = new byte[content.Length];
Buffer.BlockCopy(content, 0, copy, 0, content.Length);
_blobs[id] = copy;
_blobs[id] = new RawDocumentEntry(copy, ExpiresAt);
await Task.CompletedTask.ConfigureAwait(false);
return id;
}
public async Task<byte[]> DownloadAsync(Guid id, CancellationToken cancellationToken)
{
if (_blobs.TryGetValue(id, out var bytes))
cancellationToken.ThrowIfCancellationRequested();
if (_blobs.TryGetValue(id, out var entry))
{
return bytes;
if (entry.ExpiresAt.HasValue && entry.ExpiresAt.Value <= _timeProvider.GetUtcNow())
{
_blobs.TryRemove(id, out _);
}
else
{
return entry.Payload;
}
}
if (_documentStore is not null)
@@ -58,7 +74,7 @@ public sealed class RawDocumentStorage
var record = await _documentStore.FindAsync(id, cancellationToken).ConfigureAwait(false);
if (record?.Payload is { Length: > 0 })
{
_blobs[id] = record.Payload;
_blobs[id] = new RawDocumentEntry(record.Payload, record.ExpiresAt);
return record.Payload;
}
}
@@ -68,7 +84,16 @@ public sealed class RawDocumentStorage
public async Task DeleteAsync(Guid id, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
_blobs.TryRemove(id, out _);
await Task.CompletedTask.ConfigureAwait(false);
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

12
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

@@ -7,6 +7,7 @@ using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Documents;
@@ -182,7 +183,7 @@ public sealed class SourceFetchService
}
var existing = await _storageDocumentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
var recordId = existing?.Id ?? Guid.NewGuid();
var recordId = existing?.Id ?? CreateDeterministicGuid($"{request.SourceName}:{request.RequestUri}");
var payloadId = await _rawDocumentStorage.UploadAsync(
request.SourceName,
@@ -701,6 +702,7 @@ public sealed class SourceFetchService
maxAttempts: options.MaxAttempts,
baseDelay: options.BaseDelay,
_jitterSource,
_timeProvider,
context => SourceDiagnostics.RecordRetry(
request.SourceName,
request.ClientName,
@@ -770,6 +772,14 @@ public sealed class SourceFetchService
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
private static string? TryGetHeaderValue(HttpResponseHeaders headers, string name)
{
if (headers.TryGetValues(name, out var values))

15
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs
View File

@@ -16,12 +16,14 @@ internal static class SourceRetryPolicy
int maxAttempts,
TimeSpan baseDelay,
IJitterSource jitterSource,
TimeProvider timeProvider,
Action<SourceRetryAttemptContext>? onRetry,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(requestFactory);
ArgumentNullException.ThrowIfNull(sender);
ArgumentNullException.ThrowIfNull(jitterSource);
ArgumentNullException.ThrowIfNull(timeProvider);
var attempt = 0;
@@ -48,7 +50,7 @@ internal static class SourceRetryPolicy
var delay = ComputeDelay(
baseDelay,
attempt,
GetRetryAfter(response),
GetRetryAfter(response, timeProvider),
jitterSource);
onRetry?.Invoke(new SourceRetryAttemptContext(attempt, response, null, delay));
response.Dispose();
@@ -76,7 +78,7 @@ internal static class SourceRetryPolicy
return status >= 500 && status < 600;
}
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null)
private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter, IJitterSource jitterSource)
{
if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
{
@@ -84,8 +86,7 @@ internal static class SourceRetryPolicy
}
var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1));
var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250))
?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250));
var jitter = jitterSource.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250));
return exponential + jitter;
}
@@ -130,7 +131,7 @@ internal static class SourceRetryPolicy
return false;
}
private static TimeSpan? GetRetryAfter(HttpResponseMessage response)
private static TimeSpan? GetRetryAfter(HttpResponseMessage response, TimeProvider timeProvider)
{
var retryAfter = response.Headers.RetryAfter;
if (retryAfter is not null)
@@ -142,7 +143,7 @@ internal static class SourceRetryPolicy
if (retryAfter.Date.HasValue)
{
var delta = retryAfter.Date.Value - DateTimeOffset.UtcNow;
var delta = retryAfter.Date.Value - timeProvider.GetUtcNow();
if (delta > TimeSpan.Zero)
{
return delta;
@@ -168,7 +169,7 @@ internal static class SourceRetryPolicy
if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var epochSeconds))
{
var resetTime = DateTimeOffset.FromUnixTimeSeconds(epochSeconds);
var delta = resetTime - DateTimeOffset.UtcNow;
var delta = resetTime - timeProvider.GetUtcNow();
if (delta > TimeSpan.Zero)
{
return delta;

4
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs
View File

@@ -7,7 +7,7 @@ namespace StellaOps.Concelier.Connector.Common.Http;
/// </summary>
internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
{
private readonly IReadOnlyCollection<string> _allowedHosts;
private readonly HashSet<string> _allowedHosts;
public AllowlistedHttpMessageHandler(SourceHttpClientOptions options)
{
@@ -18,7 +18,7 @@ internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler
throw new InvalidOperationException("Source HTTP client must configure at least one allowed host.");
}
_allowedHosts = snapshot;
_allowedHosts = new HashSet<string>(snapshot, StringComparer.OrdinalIgnoreCase);
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)

10
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs
View File

@@ -42,7 +42,7 @@ public sealed class PdfTextExtractor
{
page = document.GetPage(index);
}
catch (InvalidOperationException ex) when (ex.Message.Contains("empty stack", StringComparison.OrdinalIgnoreCase))
catch (InvalidOperationException)
{
continue;
}
@@ -70,7 +70,7 @@ public sealed class PdfTextExtractor
text = FlattenWords(page.GetWords());
}
}
catch (InvalidOperationException ex) when (ex.Message.Contains("empty stack", StringComparison.OrdinalIgnoreCase))
catch (InvalidOperationException)
{
try
{
@@ -97,7 +97,11 @@ public sealed class PdfTextExtractor
if (builder.Length == 0)
{
var raw = Encoding.ASCII.GetString(rawBytes);
var raw = Encoding.UTF8.GetString(rawBytes);
if (raw.Contains('\uFFFD', StringComparison.Ordinal))
{
raw = Encoding.Latin1.GetString(rawBytes);
}
var matches = Regex.Matches(raw, "\\(([^\\)]+)\\)", RegexOptions.CultureInvariant);
foreach (Match match in matches)
{

11
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs
View File

@@ -1,3 +1,5 @@
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Concelier.Documents;
@@ -144,7 +146,7 @@ public sealed class SourceStateSeedProcessor
var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false);
var recordId = document.DocumentId ?? existing?.Id ?? Guid.NewGuid();
var recordId = document.DocumentId ?? existing?.Id ?? CreateDeterministicGuid($"{source}:{document.Uri}");
if (existing?.PayloadId is { } oldGridId)
{
@@ -332,4 +334,11 @@ public sealed class SourceStateSeedProcessor
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj
View File

@@ -4,6 +4,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="JsonSchema.Net" />

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0159-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Common. |
| AUDIT-0159-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Common. |
| AUDIT-0159-A | TODO | Pending approval for changes. |
| AUDIT-0159-A | DONE | Determinism and telemetry fixes applied. |

43
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/CveConnector.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
@@ -17,14 +18,13 @@ using StellaOps.Concelier.Connector.Cve.Configuration;
using StellaOps.Concelier.Connector.Cve.Internal;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Cve;
public sealed class CveConnector : IFeedConnector
{
private const string DtoSchemaVersion = "cve/5.0";
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
@@ -164,6 +164,11 @@ public sealed class CveConnector : IFeedConnector
{
_diagnostics.FetchUnchanged();
listUnchangedCount++;
hasMorePages = false;
if (!maxModified.HasValue || windowEnd > maxModified)
{
maxModified = windowEnd;
}
break;
}
@@ -379,12 +384,13 @@ public sealed class CveConnector : IFeedConnector
var payload = DocumentObject.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
CreateDeterministicGuid($"cve:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
"cve/5.0",
DtoSchemaVersion,
payload,
_timeProvider.GetUtcNow());
_timeProvider.GetUtcNow(),
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -440,12 +446,21 @@ public sealed class CveConnector : IFeedConnector
}
var recordedAt = dtoRecord.ValidatedAt;
var advisory = CveMapper.Map(dto, document, recordedAt);
try
{
var advisory = CveMapper.Map(dto, document, recordedAt);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
_diagnostics.MapSuccess(1);
}
catch (Exception ex)
{
_logger.LogError(ex, "CVE mapping failed for {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
_diagnostics.MapFailure();
}
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
_diagnostics.MapSuccess(1);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
@@ -506,7 +521,7 @@ public sealed class CveConnector : IFeedConnector
var uri = $"seed://{dto.CveId}";
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri, cancellationToken).ConfigureAwait(false);
var documentId = existing?.Id ?? Guid.NewGuid();
var documentId = existing?.Id ?? CreateDeterministicGuid($"cve:seed:{dto.CveId}");
var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
var lastModified = dto.Modified ?? dto.Published ?? now;
@@ -590,4 +605,12 @@ public sealed class CveConnector : IFeedConnector
var encoded = Uri.EscapeDataString(cveId);
return new Uri($"cve/{encoded}", UriKind.Relative);
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

20
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -27,8 +28,8 @@ internal sealed record CveCursor(
var document = new DocumentObject
{
["nextPage"] = NextPage,
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString())),
};
if (LastModifiedExclusive.HasValue)
@@ -82,10 +83,10 @@ internal sealed record CveCursor(
}
public CveCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public CveCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public CveCursor WithLastModifiedExclusive(DateTimeOffset? timestamp)
=> this with { LastModifiedExclusive = timestamp };
@@ -104,7 +105,11 @@ internal sealed record CveCursor(
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -130,6 +135,9 @@ internal sealed record CveCursor(
}
}
return results;
return results
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
}

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/Internal/CveDiagnostics.cs
View File

@@ -17,6 +17,7 @@ public sealed class CveDiagnostics : IDisposable
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _parseQuarantine;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
public CveDiagnostics()
{
@@ -57,6 +58,10 @@ public sealed class CveDiagnostics : IDisposable
name: "cve.map.success",
unit: "advisories",
description: "Count of canonical advisories emitted by the CVE mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "cve.map.failures",
unit: "advisories",
description: "Count of CVE advisories that failed to map.");
}
public void FetchAttempt() => _fetchAttempts.Add(1);
@@ -77,5 +82,7 @@ public sealed class CveDiagnostics : IDisposable
public void MapSuccess(long count) => _mapSuccess.Add(count);
public void MapFailure() => _mapFailures.Add(1);
public void Dispose() => _meter.Dispose();
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/Internal/CveRecordParser.cs
View File

@@ -50,7 +50,7 @@ internal static class CveRecordParser
State = state,
Published = published,
Modified = modified,
Aliases = aliases.ToArray(),
Aliases = aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(),
References = references,
Affected = affected,
Metrics = metrics,

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/StellaOps.Concelier.Connector.Cve.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Cve/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0161-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Cve. |
| AUDIT-0161-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Cve. |
| AUDIT-0161-A | TODO | Pending approval for changes. |
| AUDIT-0161-A | DONE | Determinism, cursor ordering, and map isolation applied. |

52
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/AlpineConnector.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
@@ -228,7 +229,14 @@ public sealed class AlpineConnector : IFeedConnector
}
var payload = ToDocument(dto);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow());
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"alpine:dto:{document.Id}:{SchemaVersion}"),
document.Id,
SourceName,
SchemaVersion,
payload,
_timeProvider.GetUtcNow(),
SchemaVersion: SchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -271,32 +279,50 @@ public sealed class AlpineConnector : IFeedConnector
}
AlpineSecDbDto dto;
IReadOnlyList<Advisory> advisories;
try
{
dto = FromDocument(dtoRecord.Payload);
dto = ApplyMetadataFallbacks(dto, document);
advisories = AlpineMapper.Map(dto, document, _timeProvider.GetUtcNow());
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to deserialize Alpine secdb DTO for document {DocumentId}", documentId);
_logger.LogError(ex, "Failed to deserialize or map Alpine secdb DTO for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
continue;
}
var advisories = AlpineMapper.Map(dto, document, _timeProvider.GetUtcNow());
var hadFailures = false;
foreach (var advisory in advisories)
{
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
try
{
await IngestToCanonicalAsync(advisory, document.FetchedAt, cancellationToken).ConfigureAwait(false);
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
{
await IngestToCanonicalAsync(advisory, document.FetchedAt, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception ex)
{
hadFailures = true;
_logger.LogError(ex, "Alpine advisory upsert failed for {AdvisoryKey}", advisory.AdvisoryKey);
}
}
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
if (hadFailures)
{
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
else
{
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
if (advisories.Count > 0)
@@ -632,4 +658,12 @@ public sealed class AlpineConnector : IFeedConnector
}
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

15
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/Internal/AlpineCursor.cs
View File

@@ -34,14 +34,14 @@ internal sealed record AlpineCursor(
{
var doc = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString()))
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(id => id.ToString()))
};
if (FetchCache.Count > 0)
{
var cacheDoc = new DocumentObject();
foreach (var (key, entry) in FetchCache)
foreach (var (key, entry) in FetchCache.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase))
{
cacheDoc[key] = entry.ToDocumentObject();
}
@@ -53,10 +53,10 @@ internal sealed record AlpineCursor(
}
public AlpineCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public AlpineCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public AlpineCursor WithFetchCache(IDictionary<string, AlpineFetchCacheEntry>? cache)
{
@@ -95,7 +95,10 @@ internal sealed record AlpineCursor(
}
}
return list;
return list
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
private static IReadOnlyDictionary<string, AlpineFetchCacheEntry> ReadCache(DocumentObject document)

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/Internal/AlpineFetchCacheEntry.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using StellaOps.Concelier.Documents;
using StorageContracts = StellaOps.Concelier.Storage.Contracts;
@@ -31,7 +32,11 @@ internal sealed record AlpineFetchCacheEntry(string? ETag, DateTimeOffset? LastM
lastModified = modifiedValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
modifiedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null
};
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/StellaOps.Concelier.Connector.Distro.Alpine.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Alpine/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0163-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Distro.Alpine. |
| AUDIT-0163-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Distro.Alpine. |
| AUDIT-0163-A | TODO | Pending approval for changes. |
| AUDIT-0163-A | DONE | Determinism, cursor ordering, and map isolation applied. |

66
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/DebianConnector.cs
View File

@@ -3,6 +3,8 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -16,8 +18,6 @@ using StellaOps.Concelier.Connector.Distro.Debian.Configuration;
using StellaOps.Concelier.Connector.Distro.Debian.Internal;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Plugin;
@@ -114,7 +114,7 @@ public sealed class DebianConnector : IFeedConnector
throw;
}
var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill);
var lastPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
var processedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase);
var newProcessedIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
@@ -191,6 +191,16 @@ public sealed class DebianConnector : IFeedConnector
{
cancellationToken.ThrowIfCancellationRequested();
if (entry.Published < lastPublished)
{
continue;
}
if (entry.Published == lastPublished && processedIds.Contains(entry.AdvisoryId))
{
continue;
}
var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId);
var cacheKey = detailUri.ToString();
touchedResources.Add(cacheKey);
@@ -373,7 +383,14 @@ public sealed class DebianConnector : IFeedConnector
}
var payload = ToDocument(dto);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow());
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"debian:dto:{document.Id}:{SchemaVersion}"),
document.Id,
SourceName,
SchemaVersion,
payload,
_timeProvider.GetUtcNow(),
SchemaVersion: SchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -428,19 +445,28 @@ public sealed class DebianConnector : IFeedConnector
continue;
}
var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
try
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
}
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
catch (Exception ex)
{
_logger.LogError(ex, "Debian mapping failed for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
@@ -618,7 +644,11 @@ public sealed class DebianConnector : IFeedConnector
? publishedValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
publishedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => (DateTimeOffset?)null,
}
: null));
@@ -732,4 +762,12 @@ public sealed class DebianConnector : IFeedConnector
}
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

30
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -31,7 +32,11 @@ internal sealed record DebianCursor(
{
lastPublished = lastValue.DocumentType switch
{
DocumentType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
lastValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc),
_ => null,
};
@@ -49,8 +54,8 @@ internal sealed record DebianCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(static id => id.ToString())),
};
if (LastPublished.HasValue)
@@ -60,13 +65,13 @@ internal sealed record DebianCursor(
if (ProcessedAdvisoryIds.Count > 0)
{
document["processedIds"] = new DocumentArray(ProcessedAdvisoryIds);
document["processedIds"] = new DocumentArray(ProcessedAdvisoryIds.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase));
}
if (FetchCache.Count > 0)
{
var cacheDoc = new DocumentObject();
foreach (var (key, entry) in FetchCache)
foreach (var (key, entry) in FetchCache.OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase))
{
cacheDoc[key] = entry.ToDocumentObject();
}
@@ -78,10 +83,10 @@ internal sealed record DebianCursor(
}
public DebianCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public DebianCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids)
=> this with
@@ -90,6 +95,7 @@ internal sealed record DebianCursor(
ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray() ?? EmptyIds
};
@@ -134,7 +140,10 @@ internal sealed record DebianCursor(
}
}
return list;
return list
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyCollection<Guid> ReadGuidArray(DocumentObject document, string field)
@@ -153,7 +162,10 @@ internal sealed record DebianCursor(
}
}
return list;
return list
.Distinct()
.OrderBy(static id => id)
.ToArray();
}
private static IReadOnlyDictionary<string, DebianFetchCacheEntry> ReadCache(DocumentObject document)

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianFetchCacheEntry.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Storage.Contracts;
@@ -33,7 +34,11 @@ internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastM
{
lastModified = modifiedValue.DocumentType switch
{
DocumentType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
modifiedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
_ => null,
};

23
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListParser.cs
View File

@@ -32,7 +32,14 @@ internal static class DebianListParser
continue;
}
if (line[0] == '[')
var trimmed = line.TrimStart();
if (trimmed.Length == 0)
{
continue;
}
if (trimmed[0] == '[')
{
if (currentId is not null && currentTitle is not null && currentPackage is not null)
{
@@ -41,7 +48,9 @@ internal static class DebianListParser
currentDate,
currentTitle,
currentPackage,
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
currentCves.Count == 0
? Array.Empty<string>()
: currentCves.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray()));
}
currentCves.Clear();
@@ -49,7 +58,7 @@ internal static class DebianListParser
currentTitle = null;
currentPackage = null;
var match = HeaderRegex.Match(line);
var match = HeaderRegex.Match(trimmed);
if (!match.Success)
{
continue;
@@ -80,9 +89,9 @@ internal static class DebianListParser
continue;
}
if (line[0] == '{')
if (trimmed[0] == '{')
{
foreach (Match match in CveRegex.Matches(line))
foreach (Match match in CveRegex.Matches(trimmed))
{
if (match.Success && !string.IsNullOrWhiteSpace(match.Value))
{
@@ -99,7 +108,9 @@ internal static class DebianListParser
currentDate,
currentTitle,
currentPackage,
currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves)));
currentCves.Count == 0
? Array.Empty<string>()
: currentCves.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray()));
}
return entries;

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Debian/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0165-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Distro.Debian. |
| AUDIT-0165-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Distro.Debian. |
| AUDIT-0165-A | TODO | Pending approval for changes. |
| AUDIT-0165-A | DONE | Determinism, cursor ordering, and map isolation applied. |

28
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -48,12 +49,15 @@ internal sealed record RedHatCursor(
document["lastReleasedOn"] = LastReleasedOn.Value.UtcDateTime;
}
document["processedAdvisories"] = new DocumentArray(ProcessedAdvisoryIds);
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString()));
document["processedAdvisories"] = new DocumentArray(
ProcessedAdvisoryIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase));
document["pendingDocuments"] = new DocumentArray(
PendingDocuments.OrderBy(id => id).Select(id => id.ToString()));
document["pendingMappings"] = new DocumentArray(
PendingMappings.OrderBy(id => id).Select(id => id.ToString()));
var cacheArray = new DocumentArray();
foreach (var (key, metadata) in FetchCache)
foreach (var (key, metadata) in FetchCache.OrderBy(entry => entry.Key, StringComparer.OrdinalIgnoreCase))
{
var cacheDoc = new DocumentObject
{
@@ -82,6 +86,7 @@ internal sealed record RedHatCursor(
var normalizedIds = advisoryIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return this with
@@ -107,18 +112,21 @@ internal sealed record RedHatCursor(
}
}
return this with { ProcessedAdvisoryIds = set.ToArray() };
return this with
{
ProcessedAdvisoryIds = set.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase).ToArray()
};
}
public RedHatCursor WithPendingDocuments(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
var list = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>();
return this with { PendingDocuments = list };
}
public RedHatCursor WithPendingMappings(IEnumerable<Guid> ids)
{
var list = ids?.Distinct().ToArray() ?? Array.Empty<Guid>();
var list = ids?.Distinct().OrderBy(static id => id).ToArray() ?? Array.Empty<Guid>();
return this with { PendingMappings = list };
}
@@ -245,7 +253,11 @@ internal sealed record RedHatCursor(
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatMapper.cs
View File

@@ -10,7 +10,6 @@ using StellaOps.Concelier.Normalization.Distro;
using StellaOps.Concelier.Normalization.Identifiers;
using StellaOps.Concelier.Normalization.Text;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
@@ -102,7 +101,7 @@ internal static class RedHatMapper
}
}
return aliases;
return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static NormalizedDescription NormalizeSummary(RedHatDocumentSection documentSection)
@@ -288,7 +287,7 @@ internal static class RedHatMapper
var affected = new List<AffectedPackage>(rpmPackages.Count + baseProducts.Count);
foreach (var rpm in rpmPackages.Values)
foreach (var rpm in rpmPackages.OrderBy(static entry => entry.Key, StringComparer.OrdinalIgnoreCase).Select(static entry => entry.Value))
{
if (rpm.Statuses.Count == 0)
{
@@ -359,7 +358,7 @@ internal static class RedHatMapper
new[] { provenance }));
}
foreach (var baseEntry in baseProducts.Values)
foreach (var baseEntry in baseProducts.OrderBy(static entry => entry.Key, StringComparer.OrdinalIgnoreCase).Select(static entry => entry.Value))
{
if (baseEntry.Statuses.Count == 0)
{

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatSummaryItem.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal;
@@ -44,7 +45,11 @@ internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOff
return false;
}
if (!DateTimeOffset.TryParse(releasedProperty.GetString(), out var releasedOn))
if (!DateTimeOffset.TryParse(
releasedProperty.GetString(),
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var releasedOn))
{
return false;
}

27
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnector.cs
View File

@@ -2,6 +2,8 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -12,10 +14,8 @@ using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Distro.RedHat.Configuration;
using StellaOps.Concelier.Connector.Distro.RedHat.Internal;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Plugin;
@@ -23,6 +23,7 @@ namespace StellaOps.Concelier.Connector.Distro.RedHat;
public sealed class RedHatConnector : IFeedConnector
{
private const string DtoSchemaVersion = "redhat.csaf.v2";
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
@@ -293,6 +294,8 @@ public sealed class RedHatConnector : IFeedConnector
foreach (var documentId in cursor.PendingDocuments)
{
cancellationToken.ThrowIfCancellationRequested();
DocumentRecord? document = null;
try
@@ -319,12 +322,13 @@ public sealed class RedHatConnector : IFeedConnector
var payload = DocumentObject.Parse(sanitized);
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
CreateDeterministicGuid($"redhat:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
"redhat.csaf.v2",
DtoSchemaVersion,
payload,
_timeProvider.GetUtcNow());
_timeProvider.GetUtcNow(),
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -365,6 +369,8 @@ public sealed class RedHatConnector : IFeedConnector
foreach (var documentId in cursor.PendingMappings)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
@@ -385,6 +391,7 @@ public sealed class RedHatConnector : IFeedConnector
var advisory = RedHatMapper.Map(SourceName, dto, document, jsonDocument);
if (advisory is null)
{
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
continue;
}
@@ -403,6 +410,8 @@ public sealed class RedHatConnector : IFeedConnector
catch (Exception ex)
{
_logger.LogError(ex, "Red Hat map failed for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
pendingMappings.Remove(documentId);
}
}
@@ -528,4 +537,12 @@ public sealed class RedHatConnector : IFeedConnector
}
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/StellaOps.Concelier.Connector.Distro.RedHat.csproj
View File

@@ -4,6 +4,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0167-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Distro.RedHat. |
| AUDIT-0167-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Distro.RedHat. |
| AUDIT-0167-A | TODO | Pending approval for changes. |
| AUDIT-0167-A | DONE | Applied audit remediations. |

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCsafParser.cs
View File

@@ -37,7 +37,7 @@ internal static class SuseCsafParser
var summary = ExtractSummary(documentElement);
var published = ParseDate(trackingElement, "initial_release_date")
?? ParseDate(trackingElement, "current_release_date")
?? DateTimeOffset.UtcNow;
?? DateTimeOffset.MinValue;
var references = new List<SuseReferenceDto>();
if (documentElement.TryGetProperty("references", out var referencesElement) &&
@@ -217,7 +217,11 @@ internal static class SuseCsafParser
}
if (dateElement.ValueKind == JsonValueKind.String &&
DateTimeOffset.TryParse(dateElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
DateTimeOffset.TryParse(
dateElement.GetString(),
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed))
{
return parsed.ToUniversalTime();
}

21
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCursor.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using StellaOps.Concelier.Documents;
@@ -32,7 +33,11 @@ internal sealed record SuseCursor(
lastModified = lastValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
lastValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -49,8 +54,8 @@ internal sealed record SuseCursor(
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString())),
["pendingDocuments"] = new DocumentArray(PendingDocuments.OrderBy(static id => id).Select(static id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.OrderBy(static id => id).Select(static id => id.ToString())),
};
if (LastModified.HasValue)
@@ -60,13 +65,14 @@ internal sealed record SuseCursor(
if (ProcessedIds.Count > 0)
{
document["processedIds"] = new DocumentArray(ProcessedIds);
document["processedIds"] = new DocumentArray(
ProcessedIds.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase));
}
if (FetchCache.Count > 0)
{
var cacheDocument = new DocumentObject();
foreach (var (key, entry) in FetchCache)
foreach (var (key, entry) in FetchCache.OrderBy(static entry => entry.Key, StringComparer.OrdinalIgnoreCase))
{
cacheDocument[key] = entry.ToDocumentObject();
}
@@ -78,10 +84,10 @@ internal sealed record SuseCursor(
}
public SuseCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingDocuments = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public SuseCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
=> this with { PendingMappings = ids?.Distinct().OrderBy(static id => id).ToArray() ?? EmptyGuidList };
public SuseCursor WithFetchCache(IDictionary<string, SuseFetchCacheEntry>? cache)
{
@@ -100,6 +106,7 @@ internal sealed record SuseCursor(
ProcessedIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase)
.ToArray() ?? EmptyStringList
};

7
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseFetchCacheEntry.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Globalization;
using StellaOps.Concelier.Documents;
using LegacyContracts = StellaOps.Concelier.Storage;
using StorageContracts = StellaOps.Concelier.Storage.Contracts;
@@ -35,7 +36,11 @@ internal sealed record SuseFetchCacheEntry(string? ETag, DateTimeOffset? LastMod
lastModified = modifiedValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.String when DateTimeOffset.TryParse(
modifiedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/StellaOps.Concelier.Connector.Distro.Suse.csproj
View File

@@ -5,6 +5,7 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

85
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/SuseConnector.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
@@ -31,6 +32,7 @@ public sealed class SuseConnector : IFeedConnector
new EventId(1, "SuseMapped"),
"SUSE advisory {AdvisoryId} mapped with {AffectedCount} affected packages");
private const string DtoSchemaVersion = "suse.csaf.v1";
private readonly SourceFetchService _fetchService;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly IDocumentStore _documentStore;
@@ -186,6 +188,13 @@ public sealed class SuseConnector : IFeedConnector
{
cancellationToken.ThrowIfCancellationRequested();
if (cursor.LastModified.HasValue
&& record.ModifiedAt == cursor.LastModified.Value
&& processedIds.Contains(record.FileName))
{
continue;
}
var detailUri = new Uri(_options.AdvisoryBaseUri, record.FileName);
var cacheKey = detailUri.AbsoluteUri;
touchedResources.Add(cacheKey);
@@ -237,6 +246,18 @@ public sealed class SuseConnector : IFeedConnector
}
}
if (record.ModifiedAt > maxModified)
{
maxModified = record.ModifiedAt;
processedUpdated = true;
currentWindowIds.Clear();
currentWindowIds.Add(record.FileName);
}
else if (record.ModifiedAt == maxModified)
{
currentWindowIds.Add(record.FileName);
}
continue;
}
@@ -248,13 +269,18 @@ public sealed class SuseConnector : IFeedConnector
fetchCache[cacheKey] = SuseFetchCacheEntry.FromDocument(result.Document);
pendingDocuments.Add(result.Document.Id);
pendingMappings.Remove(result.Document.Id);
currentWindowIds.Add(record.FileName);
if (record.ModifiedAt > maxModified)
{
maxModified = record.ModifiedAt;
processedUpdated = true;
currentWindowIds.Clear();
currentWindowIds.Add(record.FileName);
}
else if (record.ModifiedAt == maxModified)
{
currentWindowIds.Add(record.FileName);
}
}
}
@@ -346,7 +372,14 @@ public sealed class SuseConnector : IFeedConnector
await _documentStore.UpsertAsync(updatedDocument, cancellationToken).ConfigureAwait(false);
var payload = ToDocument(dto);
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "suse.csaf.v1", payload, _timeProvider.GetUtcNow());
var dtoRecord = new DtoRecord(
CreateDeterministicGuid($"suse:dto:{document.Id}:{DtoSchemaVersion}"),
document.Id,
SourceName,
DtoSchemaVersion,
payload,
_timeProvider.GetUtcNow(),
SchemaVersion: DtoSchemaVersion);
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
@@ -402,19 +435,28 @@ public sealed class SuseConnector : IFeedConnector
continue;
}
var advisory = SuseMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
try
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
var advisory = SuseMapper.Map(dto, document, _timeProvider.GetUtcNow());
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
{
var rawPayloadJson = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson });
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
}
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to map SUSE advisory for document {DocumentId}", documentId);
await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null);
}
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
@@ -511,10 +553,14 @@ public sealed class SuseConnector : IFeedConnector
? publishedValue.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => DateTimeOffset.UtcNow
DocumentType.String when DateTimeOffset.TryParse(
publishedValue.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed) => parsed.ToUniversalTime(),
_ => DateTimeOffset.MinValue
}
: DateTimeOffset.UtcNow;
: DateTimeOffset.MinValue;
var cves = document.TryGetValue("cves", out var cveArray) && cveArray is DocumentArray cveArr
? cveArr.OfType<DocumentValue>()
@@ -522,6 +568,7 @@ public sealed class SuseConnector : IFeedConnector
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value!)
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.ToArray()
: Array.Empty<string>();
@@ -665,4 +712,12 @@ public sealed class SuseConnector : IFeedConnector
}
}
}
private static Guid CreateDeterministicGuid(string value)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value ?? string.Empty));
bytes[6] = (byte)((bytes[6] & 0x0F) | 0x50);
bytes[8] = (byte)((bytes[8] & 0x3F) | 0x80);
return new Guid(bytes.AsSpan(0, 16));
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Suse/TASKS.md
View File

@@ -7,5 +7,5 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0169-M | DONE | Maintainability audit for StellaOps.Concelier.Connector.Distro.Suse. |
| AUDIT-0169-T | DONE | Test coverage audit for StellaOps.Concelier.Connector.Distro.Suse. |
| AUDIT-0169-A | TODO | Pending approval for changes. |
| AUDIT-0169-A | DONE | Applied audit remediations. |
| CICD-VAL-SMOKE-001 | DOING | Smoke validation: trim CSAF product IDs to preserve package mapping. |

20
src/Concelier/__Tests/StellaOps.Concelier.Analyzers.Tests/AGENTS.md Normal file
View File

@@ -0,0 +1,20 @@
# Concelier Analyzer Tests Charter
## Mission
Own tests for the Concelier analyzer rules.
## Responsibilities
- Keep analyzer diagnostics deterministic and scoped to Concelier connectors.
- Maintain test coverage for allowed/blocked HttpClient usage.
## Key Paths
- `ConnectorHttpClientSandboxAnalyzerTests.cs`
## Required Reading
- `docs/modules/concelier/architecture.md`
- `docs/modules/platform/architecture-overview.md`
## Working Agreement
- 1. Update task status to `DOING`/`DONE` in both corresponding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.

113
src/Concelier/__Tests/StellaOps.Concelier.Analyzers.Tests/ConnectorHttpClientSandboxAnalyzerTests.cs Normal file
View File

@@ -0,0 +1,113 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Diagnostics;
using StellaOps.TestKit;
namespace StellaOps.Concelier.Analyzers.Tests;
public sealed class ConnectorHttpClientSandboxAnalyzerTests
{
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ReportsDiagnostic_ForHttpClientInConnectorNamespace()
{
const string source = """
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Demo;
public sealed class ClientFactory
{
public HttpClient Create() => new HttpClient();
}
""";
var diagnostics = await AnalyzeAsync(source, "StellaOps.Concelier.Connector.Demo");
Assert.Contains(diagnostics, d => d.Id == ConnectorHttpClientSandboxAnalyzer.DiagnosticId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task DoesNotReportDiagnostic_OutsideConnectorNamespace()
{
const string source = """
using System.Net.Http;
namespace Sample.App;
public sealed class ClientFactory
{
public HttpClient Create() => new HttpClient();
}
""";
var diagnostics = await AnalyzeAsync(source, "Sample.App");
Assert.DoesNotContain(diagnostics, d => d.Id == ConnectorHttpClientSandboxAnalyzer.DiagnosticId);
}
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("StellaOps.Concelier.Connector.Demo.Tests")]
[InlineData("StellaOps.Concelier.Connector.Demo.Test")]
[InlineData("StellaOps.Concelier.Connector.Demo.Testing")]
public async Task DoesNotReportDiagnostic_InTestAssemblies(string assemblyName)
{
const string source = """
using System.Net.Http;
namespace StellaOps.Concelier.Connector.Demo;
public sealed class ClientFactory
{
public HttpClient Create() => new HttpClient();
}
""";
var diagnostics = await AnalyzeAsync(source, assemblyName);
Assert.DoesNotContain(diagnostics, d => d.Id == ConnectorHttpClientSandboxAnalyzer.DiagnosticId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task DoesNotReportDiagnostic_ForOtherTypes()
{
const string source = """
namespace StellaOps.Concelier.Connector.Demo;
public sealed class ClientFactory
{
public object Create() => new object();
}
""";
var diagnostics = await AnalyzeAsync(source, "StellaOps.Concelier.Connector.Demo");
Assert.DoesNotContain(diagnostics, d => d.Id == ConnectorHttpClientSandboxAnalyzer.DiagnosticId);
}
private static async Task<ImmutableArray<Diagnostic>> AnalyzeAsync(string source, string assemblyName)
{
var compilation = CSharpCompilation.Create(
assemblyName,
new[] { CSharpSyntaxTree.ParseText(source) },
CreateMetadataReferences(),
new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary));
var analyzer = new ConnectorHttpClientSandboxAnalyzer();
var compilationWithAnalyzers = compilation.WithAnalyzers(ImmutableArray.Create<DiagnosticAnalyzer>(analyzer));
return await compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync();
}
private static IEnumerable<MetadataReference> CreateMetadataReferences()
{
yield return MetadataReference.CreateFromFile(typeof(object).GetTypeInfo().Assembly.Location);
yield return MetadataReference.CreateFromFile(typeof(Enumerable).GetTypeInfo().Assembly.Location);
yield return MetadataReference.CreateFromFile(typeof(HttpClient).GetTypeInfo().Assembly.Location);
}
}

19
src/Concelier/__Tests/StellaOps.Concelier.Analyzers.Tests/StellaOps.Concelier.Analyzers.Tests.csproj Normal file
View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\__Analyzers\StellaOps.Concelier.Analyzers\StellaOps.Concelier.Analyzers.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
</ItemGroup>
</Project>

8
src/Concelier/__Tests/StellaOps.Concelier.Analyzers.Tests/TASKS.md Normal file
View File

@@ -0,0 +1,8 @@
# Concelier Analyzer Tests Task Board
This board mirrors active sprint tasks for this module.
Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md`.
| Task ID | Status | Notes |
| --- | --- | --- |
| AUDIT-0144-A | DONE | Tests for StellaOps.Concelier.Analyzers. |

63
src/Concelier/__Tests/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs
View File

@@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Acsc;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
@@ -120,7 +121,65 @@ public sealed class AcscConnectorFetchTests
});
}
private async Task<ConnectorTestHarness> BuildHarnessAsync(bool preferRelay)
[Fact]
public async Task ProbeAsync_HeadNotAllowedFallsBackToGetAndPrefersDirect()
{
await using var harness = await BuildHarnessAsync(preferRelay: true);
var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
var cursor = AcscCursor.Empty.WithPreferredEndpoint(AcscEndpointPreference.Relay);
await stateRepository.UpdateCursorAsync(
AcscConnectorPlugin.SourceName,
cursor.ToDocumentObject(),
harness.TimeProvider.GetUtcNow(),
CancellationToken.None);
harness.Handler.AddResponse(HttpMethod.Head, AlertsDirectUri, _ => new HttpResponseMessage(HttpStatusCode.MethodNotAllowed));
harness.Handler.AddResponse(HttpMethod.Get, AlertsDirectUri, _ => new HttpResponseMessage(HttpStatusCode.OK));
var connector = harness.ServiceProvider.GetRequiredService<AcscConnector>();
await connector.ProbeAsync(CancellationToken.None);
var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.Equal("Direct", state!.Cursor.GetValue("preferredEndpoint").AsString);
Assert.Collection(harness.Handler.Requests,
request =>
{
Assert.Equal(HttpMethod.Head, request.Method);
Assert.Equal(AlertsDirectUri, request.Uri);
},
request =>
{
Assert.Equal(HttpMethod.Get, request.Method);
Assert.Equal(AlertsDirectUri, request.Uri);
});
}
[Fact]
public async Task ProbeAsync_RelayNotConfiguredForcesDirectPreference()
{
await using var harness = await BuildHarnessAsync(preferRelay: true, includeRelay: false);
var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
var cursor = AcscCursor.Empty.WithPreferredEndpoint(AcscEndpointPreference.Relay);
await stateRepository.UpdateCursorAsync(
AcscConnectorPlugin.SourceName,
cursor.ToDocumentObject(),
harness.TimeProvider.GetUtcNow(),
CancellationToken.None);
var connector = harness.ServiceProvider.GetRequiredService<AcscConnector>();
await connector.ProbeAsync(CancellationToken.None);
var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None);
Assert.NotNull(state);
Assert.Equal("Direct", state!.Cursor.GetValue("preferredEndpoint").AsString);
Assert.Empty(harness.Handler.Requests);
}
private async Task<ConnectorTestHarness> BuildHarnessAsync(bool preferRelay, bool includeRelay = true)
{
var initialTime = new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero);
var harness = new ConnectorTestHarness(_fixture, initialTime, AcscOptions.HttpClientName);
@@ -129,7 +188,7 @@ public sealed class AcscConnectorFetchTests
services.AddAcscConnector(options =>
{
options.BaseEndpoint = BaseEndpoint;
options.RelayEndpoint = RelayEndpoint;
options.RelayEndpoint = includeRelay ? RelayEndpoint : null;
options.EnableRelayFallback = true;
options.PreferRelayByDefault = preferRelay;
options.ForceRelay = false;

15
src/Concelier/__Tests/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorParseTests.cs
View File

@@ -8,7 +8,10 @@ using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Acsc;
using StellaOps.Concelier.Connector.Acsc.Configuration;
using StellaOps.Concelier.Connector.Acsc.Internal;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.Html;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Common.Testing;
using StellaOps.Concelier.Storage;
@@ -46,6 +49,14 @@ public sealed class AcscConnectorParseTests
var document = await documentStore.FindBySourceAndUriAsync(AcscConnectorPlugin.SourceName, feedUri.ToString(), CancellationToken.None);
Assert.NotNull(document);
var rawStorage = harness.ServiceProvider.GetRequiredService<RawDocumentStorage>();
var rawBytes = await rawStorage.DownloadAsync(document!.PayloadId!.Value, CancellationToken.None);
Assert.NotEmpty(rawBytes);
var rawText = Encoding.UTF8.GetString(rawBytes);
Assert.Contains("<rss", rawText, StringComparison.OrdinalIgnoreCase);
var sanityDto = AcscFeedParser.Parse(rawBytes, "alerts", new DateTimeOffset(2025, 10, 12, 6, 0, 0, TimeSpan.Zero), new HtmlContentSanitizer());
Assert.NotEmpty(sanityDto.Entries);
await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None);
var refreshed = await documentStore.FindAsync(document!.Id, CancellationToken.None);
@@ -60,7 +71,9 @@ public sealed class AcscConnectorParseTests
var payload = dtoRecord.Payload;
Assert.NotNull(payload);
Assert.Equal("alerts", payload.GetValue("feedSlug").AsString);
Assert.Single(payload.GetValue("entries").AsDocumentArray);
var entriesValue = payload.GetValue("entries");
Assert.Equal(DocumentType.Array, entriesValue.DocumentType);
Assert.Single(entriesValue.AsDocumentArray);
var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>();
var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None);

Some files were not shown because too many files have changed in this diff Show More