stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

save progress

Browse Source
This commit is contained in:
StellaOps Bot
2026-01-03 11:02:24 +02:00
parent ca578801fd
commit 83c37243e0
446 changed files with 22798 additions and 4031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/AdvisoryCacheKeys.cs
View File

@@ -5,6 +5,7 @@
// Description: Key schema for Concelier Valkey cache
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Concelier.Cache.Valkey;
@@ -15,13 +16,13 @@ namespace StellaOps.Concelier.Cache.Valkey;
/// <remarks>
/// Key Schema:
/// <code>
/// advisory:{merge_hash} → JSON(CanonicalAdvisory) - TTL based on interest_score
/// rank:hot → ZSET { merge_hash: interest_score } - max 10,000 entries
/// by:purl:{normalized_purl} → SET { merge_hash, ... } - TTL 24h
/// by:cve:{cve_id} → STRING merge_hash - TTL 24h
/// cache:stats:hits → INCR counter
/// cache:stats:misses → INCR counter
/// cache:warmup:last → STRING ISO8601 timestamp
/// advisory:{merge_hash} -> JSON(CanonicalAdvisory) - TTL based on interest_score
/// rank:hot -> ZSET { merge_hash: interest_score } - max 10,000 entries
/// by:purl:{normalized_purl} -> SET { merge_hash, ... } - TTL 24h
/// by:cve:{cve_id} -> STRING merge_hash - TTL 24h
/// cache:stats:hits -> INCR counter
/// cache:stats:misses -> INCR counter
/// cache:warmup:last -> STRING ISO8601 timestamp
/// </code>
/// </remarks>
public static class AdvisoryCacheKeys
@@ -30,6 +31,7 @@ public static class AdvisoryCacheKeys
/// Default key prefix for all cache keys.
/// </summary>
public const string DefaultPrefix = "concelier:";
public const int MaxPurlKeyLength = 500;
/// <summary>
/// Key for advisory by merge hash.
@@ -158,14 +160,20 @@ public static class AdvisoryCacheKeys
}
}
// Truncate if too long (Redis keys can be up to 512MB, but we want reasonable sizes)
const int MaxKeyLength = 500;
if (sb.Length > MaxKeyLength)
var normalizedKey = sb.ToString();
if (normalizedKey.Length <= MaxPurlKeyLength)
{
return sb.ToString(0, MaxKeyLength);
return normalizedKey;
}
return sb.ToString();
var hash = ComputeHash(normalizedKey);
var prefixLength = Math.Max(0, MaxPurlKeyLength - hash.Length - 1);
if (prefixLength == 0)
{
return hash;
}
return normalizedKey[..prefixLength] + "-" + hash;
}
/// <summary>
@@ -215,4 +223,25 @@ public static class AdvisoryCacheKeys
}
return null;
}
private static string ComputeHash(string value)
{
using var sha = SHA256.Create();
var bytes = Encoding.UTF8.GetBytes(value);
var hash = sha.ComputeHash(bytes);
return ToLowerHex(hash);
}
private static string ToLowerHex(byte[] bytes)
{
const string hex = "0123456789abcdef";
var chars = new char[bytes.Length * 2];
for (var i = 0; i < bytes.Length; i++)
{
var b = bytes[i];
chars[i * 2] = hex[b >> 4];
chars[i * 2 + 1] = hex[b & 0xF];
}
return new string(chars);
}
}

22
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/CacheWarmupHostedService.cs
View File

@@ -42,8 +42,12 @@ public sealed class CacheWarmupHostedService : BackgroundService
return;
}
// Wait a short time for the application to fully start
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
var delay = ResolveWarmupDelay(_options);
if (delay > TimeSpan.Zero)
{
_logger?.LogDebug("Cache warmup delay set to {Delay}", delay);
await Task.Delay(delay, stoppingToken).ConfigureAwait(false);
}
_logger?.LogInformation("Starting cache warmup with limit {Limit}", _options.WarmupLimit);
@@ -61,4 +65,18 @@ public sealed class CacheWarmupHostedService : BackgroundService
_logger?.LogError(ex, "Cache warmup failed");
}
}
private static TimeSpan ResolveWarmupDelay(ConcelierCacheOptions options)
{
var delay = options.WarmupDelay;
var jitter = options.WarmupDelayJitter;
if (jitter <= TimeSpan.Zero)
{
return delay;
}
var jitterMillis = Random.Shared.NextDouble() * jitter.TotalMilliseconds;
return delay + TimeSpan.FromMilliseconds(jitterMillis);
}
}

35
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheConnectionFactory.cs
View File

@@ -38,8 +38,7 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
{
_options = options.Value;
_logger = logger;
_connectionFactory = connectionFactory ??
(config => Task.FromResult<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(config)));
_connectionFactory = connectionFactory ?? DefaultConnectionFactory;
}
/// <summary>
@@ -95,7 +94,8 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
_logger?.LogDebug("Connecting to Valkey at {Endpoint} (database {Database})",
_options.ConnectionString, _options.Database);
_connection = await _connectionFactory(config).ConfigureAwait(false);
_connection = await WaitWithCancellationAsync(_connectionFactory(config), cancellationToken)
.ConfigureAwait(false);
_logger?.LogInformation("Connected to Valkey for Concelier cache");
}
@@ -170,4 +170,33 @@ public sealed class ConcelierCacheConnectionFactory : IAsyncDisposable
_connectionLock.Dispose();
}
private static async Task<T> WaitWithCancellationAsync<T>(Task<T> task, CancellationToken cancellationToken)
{
if (task.IsCompleted || !cancellationToken.CanBeCanceled)
{
return await task.ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(cancellationToken);
}
var cancellationSignal = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
using var registration = cancellationToken.Register(
static state => ((TaskCompletionSource<bool>)state!).TrySetResult(true),
cancellationSignal);
var completed = await Task.WhenAny(task, cancellationSignal.Task).ConfigureAwait(false);
if (completed == cancellationSignal.Task)
{
throw new OperationCanceledException(cancellationToken);
}
return await task.ConfigureAwait(false);
}
private static async Task<IConnectionMultiplexer> DefaultConnectionFactory(ConfigurationOptions options)
=> await ConnectionMultiplexer.ConnectAsync(options).ConfigureAwait(false);
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheMetrics.cs
View File

@@ -144,7 +144,6 @@ public sealed class ConcelierCacheMetrics : IDisposable
public void Dispose()
{
_meter.Dispose();
ActivitySource.Dispose();
}
}

30
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ConcelierCacheOptions.cs
View File

@@ -81,6 +81,16 @@ public sealed class ConcelierCacheOptions
/// Number of advisories to preload during warmup.
/// </summary>
public int WarmupLimit { get; set; } = 1000;
/// <summary>
/// Delay before warmup begins.
/// </summary>
public TimeSpan WarmupDelay { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Optional jitter added to the warmup delay.
/// </summary>
public TimeSpan WarmupDelayJitter { get; set; } = TimeSpan.Zero;
}
/// <summary>
@@ -130,16 +140,24 @@ public sealed class CacheTtlPolicy
/// <returns>TTL for the advisory.</returns>
public TimeSpan GetTtl(double? score)
{
if (!score.HasValue)
if (!score.HasValue || double.IsNaN(score.Value))
{
return LowScoreTtl;
}
return score.Value switch
var highThreshold = HighScoreThreshold;
var mediumThreshold = Math.Min(MediumScoreThreshold, highThreshold);
if (score.Value >= highThreshold)
{
>= 0.7 => HighScoreTtl, // High interest: 24h
>= 0.4 => MediumScoreTtl, // Medium interest: 4h
_ => LowScoreTtl // Low interest: 1h
};
return HighScoreTtl;
}
if (score.Value >= mediumThreshold)
{
return MediumScoreTtl;
}
return LowScoreTtl;
}
}

43
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ServiceCollectionExtensions.cs
View File

@@ -101,14 +101,26 @@ public static class ServiceCollectionExtensions
existingDescriptor.ImplementationType,
existingDescriptor.ImplementationType,
existingDescriptor.Lifetime));
services.Add(new ServiceDescriptor(
typeof(ICanonicalAdvisoryServiceInner),
existingDescriptor.ImplementationType,
existingDescriptor.Lifetime));
}
else if (existingDescriptor.ImplementationFactory is not null)
{
services.Add(new ServiceDescriptor(
typeof(StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService),
sp => existingDescriptor.ImplementationFactory(sp),
typeof(ICanonicalAdvisoryServiceInner),
sp => (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationFactory(sp),
existingDescriptor.Lifetime));
}
else if (existingDescriptor.ImplementationInstance is not null)
{
services.Add(new ServiceDescriptor(
typeof(ICanonicalAdvisoryServiceInner),
existingDescriptor.ImplementationInstance));
}
// Register the decorator as the new ICanonicalAdvisoryService
services.Add(new ServiceDescriptor(
@@ -116,28 +128,7 @@ public static class ServiceCollectionExtensions
sp =>
{
// Resolve the inner service (the original implementation)
StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService innerService;
if (existingDescriptor.ImplementationType is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
sp.GetRequiredService(existingDescriptor.ImplementationType);
}
else if (existingDescriptor.ImplementationFactory is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationFactory(sp);
}
else if (existingDescriptor.ImplementationInstance is not null)
{
innerService = (StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService)
existingDescriptor.ImplementationInstance;
}
else
{
throw new InvalidOperationException(
"Unable to resolve inner ICanonicalAdvisoryService for decorator.");
}
var innerService = sp.GetRequiredService<ICanonicalAdvisoryServiceInner>();
var cache = sp.GetRequiredService<IAdvisoryCacheService>();
var logger = sp.GetRequiredService<Microsoft.Extensions.Logging.ILogger<ValkeyCanonicalAdvisoryService>>();
@@ -166,4 +157,8 @@ public static class ServiceCollectionExtensions
services.AddValkeyCachingDecorator();
return services;
}
internal interface ICanonicalAdvisoryServiceInner : StellaOps.Concelier.Core.Canonical.ICanonicalAdvisoryService
{
}
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/TASKS.md
View File

@@ -7,4 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.
| --- | --- | --- |
| AUDIT-0145-M | DONE | Maintainability audit for StellaOps.Concelier.Cache.Valkey. |
| AUDIT-0145-T | DONE | Test coverage audit for StellaOps.Concelier.Cache.Valkey. |
| AUDIT-0145-A | TODO | Pending approval for changes. |
| AUDIT-0145-A | DOING | Applying cache hardening + tests. |

101
src/Concelier/__Libraries/StellaOps.Concelier.Cache.Valkey/ValkeyAdvisoryCacheService.cs
View File

@@ -22,6 +22,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
private readonly ConcelierCacheConnectionFactory _connectionFactory;
private readonly ConcelierCacheOptions _options;
private readonly ConcelierCacheMetrics? _metrics;
private readonly ILogger<ValkeyAdvisoryCacheService>? _logger;
private static readonly JsonSerializerOptions JsonOptions = new()
@@ -36,10 +37,12 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
public ValkeyAdvisoryCacheService(
ConcelierCacheConnectionFactory connectionFactory,
IOptions<ConcelierCacheOptions> options,
ConcelierCacheMetrics? metrics = null,
ILogger<ValkeyAdvisoryCacheService>? logger = null)
{
_connectionFactory = connectionFactory;
_options = options.Value;
_metrics = metrics;
_logger = logger;
}
@@ -51,6 +54,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return null;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -60,11 +64,13 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
if (cached.HasValue)
{
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsHits(_options.KeyPrefix)).ConfigureAwait(false);
_metrics?.RecordHit();
_logger?.LogDebug("Cache hit for advisory {MergeHash}", mergeHash);
return JsonSerializer.Deserialize<CanonicalAdvisory>((string)cached!, JsonOptions);
}
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsMisses(_options.KeyPrefix)).ConfigureAwait(false);
_metrics?.RecordMiss();
_logger?.LogDebug("Cache miss for advisory {MergeHash}", mergeHash);
return null;
}
@@ -73,6 +79,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisory {MergeHash} from cache", mergeHash);
return null;
}
finally
{
StopTiming(sw, "get");
}
}
/// <inheritdoc />
@@ -83,6 +93,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return [];
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -116,6 +127,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisories for PURL {Purl}", purl);
return [];
}
finally
{
StopTiming(sw, "get-by-purl");
}
}
/// <inheritdoc />
@@ -126,6 +141,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return null;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -144,6 +160,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get advisory for CVE {Cve}", cve);
return null;
}
finally
{
StopTiming(sw, "get-by-cve");
}
}
/// <inheritdoc />
@@ -154,6 +174,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return [];
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -189,6 +210,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get hot advisories");
return [];
}
finally
{
StopTiming(sw, "get-hot");
}
}
/// <inheritdoc />
@@ -199,6 +224,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -232,6 +258,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to cache advisory {MergeHash}", advisory.MergeHash);
}
finally
{
StopTiming(sw, "set");
}
}
/// <inheritdoc />
@@ -242,6 +272,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -254,12 +285,17 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
await db.SortedSetRemoveAsync(hotKey, mergeHash).ConfigureAwait(false);
_metrics?.RecordEviction("invalidate");
_logger?.LogDebug("Invalidated advisory {MergeHash}", mergeHash);
}
catch (Exception ex)
{
_logger?.LogWarning(ex, "Failed to invalidate advisory {MergeHash}", mergeHash);
}
finally
{
StopTiming(sw, "invalidate");
}
}
/// <inheritdoc />
@@ -270,6 +306,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -283,12 +320,19 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
if (currentSize > _options.MaxHotSetSize)
{
// Remove lowest scoring entries
await db.SortedSetRemoveRangeByRankAsync(
var removed = await db.SortedSetRemoveRangeByRankAsync(
hotKey,
start: 0,
stop: currentSize - _options.MaxHotSetSize - 1).ConfigureAwait(false);
if (removed > 0)
{
_metrics?.RecordEviction("hotset-trim");
}
}
_metrics?.UpdateHotSetSize(Math.Min(currentSize, _options.MaxHotSetSize));
// Update advisory TTL if cached
var advisoryKey = AdvisoryCacheKeys.Advisory(mergeHash, _options.KeyPrefix);
if (await db.KeyExistsAsync(advisoryKey).ConfigureAwait(false))
@@ -303,6 +347,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to update score for {MergeHash}", mergeHash);
}
finally
{
StopTiming(sw, "update-score");
}
}
/// <inheritdoc />
@@ -313,6 +361,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -327,6 +376,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to index PURL {Purl}", purl);
}
finally
{
StopTiming(sw, "index-purl");
}
}
/// <inheritdoc />
@@ -337,6 +390,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -350,6 +404,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to unindex PURL {Purl}", purl);
}
finally
{
StopTiming(sw, "unindex-purl");
}
}
/// <inheritdoc />
@@ -360,6 +418,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return;
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -373,6 +432,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Failed to index CVE {Cve}", cve);
}
finally
{
StopTiming(sw, "index-cve");
}
}
/// <inheritdoc />
@@ -424,6 +487,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
{
_logger?.LogWarning(ex, "Cache warmup failed after {Elapsed}ms", sw.ElapsedMilliseconds);
}
finally
{
StopTiming(sw, "warmup");
}
}
/// <inheritdoc />
@@ -434,6 +501,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return new CacheStatistics { IsHealthy = false };
}
var sw = StartTiming();
try
{
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
@@ -446,6 +514,7 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
var hits = (long)(await db.StringGetAsync(hitsKey).ConfigureAwait(false));
var misses = (long)(await db.StringGetAsync(missesKey).ConfigureAwait(false));
var hotSetSize = await db.SortedSetLengthAsync(hotKey).ConfigureAwait(false);
_metrics?.UpdateHotSetSize(hotSetSize);
DateTimeOffset? lastWarmup = null;
var warmupStr = await db.StringGetAsync(warmupKey).ConfigureAwait(false);
@@ -478,6 +547,10 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
_logger?.LogWarning(ex, "Failed to get cache statistics");
return new CacheStatistics { IsHealthy = false };
}
finally
{
StopTiming(sw, "stats");
}
}
/// <inheritdoc />
@@ -488,6 +561,30 @@ public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
return false;
}
return await _connectionFactory.PingAsync(cancellationToken).ConfigureAwait(false);
var sw = StartTiming();
try
{
return await _connectionFactory.PingAsync(cancellationToken).ConfigureAwait(false);
}
finally
{
StopTiming(sw, "health");
}
}
private Stopwatch? StartTiming()
{
return _metrics is null ? null : Stopwatch.StartNew();
}
private void StopTiming(Stopwatch? sw, string operation)
{
if (sw is null || _metrics is null)
{
return;
}
sw.Stop();
_metrics.RecordLatency(sw.Elapsed.TotalMilliseconds, operation);
}
}