591 lines
19 KiB
C#
591 lines
19 KiB
C#
// -----------------------------------------------------------------------------
|
|
// ValkeyAdvisoryCacheService.cs
|
|
// Sprint: SPRINT_8200_0013_0001_GW_valkey_advisory_cache
|
|
// Task: VCACHE-8200-011 to VCACHE-8200-016
|
|
// Description: Valkey implementation of advisory cache service
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Diagnostics;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using StackExchange.Redis;
|
|
using StellaOps.Concelier.Core.Canonical;
|
|
|
|
namespace StellaOps.Concelier.Cache.Valkey;
|
|
|
|
/// <summary>
|
|
/// Valkey-based implementation of the advisory cache service.
|
|
/// Provides read-through caching with TTL based on interest scores.
|
|
/// </summary>
|
|
public sealed class ValkeyAdvisoryCacheService : IAdvisoryCacheService
|
|
{
|
|
private readonly ConcelierCacheConnectionFactory _connectionFactory;
|
|
private readonly ConcelierCacheOptions _options;
|
|
private readonly ConcelierCacheMetrics? _metrics;
|
|
private readonly ILogger<ValkeyAdvisoryCacheService>? _logger;
|
|
|
|
private static readonly JsonSerializerOptions JsonOptions = new()
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
|
WriteIndented = false
|
|
};
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of <see cref="ValkeyAdvisoryCacheService"/>.
|
|
/// </summary>
|
|
public ValkeyAdvisoryCacheService(
|
|
ConcelierCacheConnectionFactory connectionFactory,
|
|
IOptions<ConcelierCacheOptions> options,
|
|
ConcelierCacheMetrics? metrics = null,
|
|
ILogger<ValkeyAdvisoryCacheService>? logger = null)
|
|
{
|
|
_connectionFactory = connectionFactory;
|
|
_options = options.Value;
|
|
_metrics = metrics;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<CanonicalAdvisory?> GetAsync(string mergeHash, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var key = AdvisoryCacheKeys.Advisory(mergeHash, _options.KeyPrefix);
|
|
|
|
var cached = await db.StringGetAsync(key).ConfigureAwait(false);
|
|
if (cached.HasValue)
|
|
{
|
|
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsHits(_options.KeyPrefix)).ConfigureAwait(false);
|
|
_metrics?.RecordHit();
|
|
_logger?.LogDebug("Cache hit for advisory {MergeHash}", mergeHash);
|
|
return JsonSerializer.Deserialize<CanonicalAdvisory>((string)cached!, JsonOptions);
|
|
}
|
|
|
|
await db.StringIncrementAsync(AdvisoryCacheKeys.StatsMisses(_options.KeyPrefix)).ConfigureAwait(false);
|
|
_metrics?.RecordMiss();
|
|
_logger?.LogDebug("Cache miss for advisory {MergeHash}", mergeHash);
|
|
return null;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to get advisory {MergeHash} from cache", mergeHash);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "get");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IReadOnlyList<CanonicalAdvisory>> GetByPurlAsync(string purl, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var indexKey = AdvisoryCacheKeys.ByPurl(purl, _options.KeyPrefix);
|
|
|
|
// Get all merge hashes for this PURL
|
|
var mergeHashes = await db.SetMembersAsync(indexKey).ConfigureAwait(false);
|
|
if (mergeHashes.Length == 0)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
// Refresh TTL on access
|
|
await db.KeyExpireAsync(indexKey, _options.TtlPolicy.PurlIndexTtl).ConfigureAwait(false);
|
|
|
|
// Fetch all advisories
|
|
var results = new List<CanonicalAdvisory>(mergeHashes.Length);
|
|
foreach (var hash in mergeHashes)
|
|
{
|
|
var advisory = await GetAsync(hash!, cancellationToken).ConfigureAwait(false);
|
|
if (advisory is not null)
|
|
{
|
|
results.Add(advisory);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to get advisories for PURL {Purl}", purl);
|
|
return [];
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "get-by-purl");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<CanonicalAdvisory?> GetByCveAsync(string cve, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var mappingKey = AdvisoryCacheKeys.ByCve(cve, _options.KeyPrefix);
|
|
|
|
var mergeHash = await db.StringGetAsync(mappingKey).ConfigureAwait(false);
|
|
if (!mergeHash.HasValue)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return await GetAsync(mergeHash!, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to get advisory for CVE {Cve}", cve);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "get-by-cve");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IReadOnlyList<CanonicalAdvisory>> GetHotAsync(int limit = 100, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
|
|
|
|
// Get top N merge hashes by score (descending)
|
|
var entries = await db.SortedSetRangeByRankAsync(
|
|
hotKey,
|
|
start: 0,
|
|
stop: limit - 1,
|
|
order: Order.Descending).ConfigureAwait(false);
|
|
|
|
if (entries.Length == 0)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
// Fetch all advisories
|
|
var results = new List<CanonicalAdvisory>(entries.Length);
|
|
foreach (var mergeHash in entries)
|
|
{
|
|
var advisory = await GetAsync(mergeHash!, cancellationToken).ConfigureAwait(false);
|
|
if (advisory is not null)
|
|
{
|
|
results.Add(advisory);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to get hot advisories");
|
|
return [];
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "get-hot");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task SetAsync(CanonicalAdvisory advisory, double? interestScore = null, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var key = AdvisoryCacheKeys.Advisory(advisory.MergeHash, _options.KeyPrefix);
|
|
|
|
var json = JsonSerializer.Serialize(advisory, JsonOptions);
|
|
var ttl = _options.TtlPolicy.GetTtl(interestScore);
|
|
|
|
await db.StringSetAsync(key, json, ttl).ConfigureAwait(false);
|
|
_logger?.LogDebug("Cached advisory {MergeHash} with TTL {Ttl}", advisory.MergeHash, ttl);
|
|
|
|
// Update hot set if score provided
|
|
if (interestScore.HasValue)
|
|
{
|
|
await UpdateScoreAsync(advisory.MergeHash, interestScore.Value, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
// Index CVE mapping
|
|
if (!string.IsNullOrWhiteSpace(advisory.Cve))
|
|
{
|
|
await IndexCveAsync(advisory.Cve, advisory.MergeHash, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
// Index by PURL (affects key)
|
|
if (!string.IsNullOrWhiteSpace(advisory.AffectsKey))
|
|
{
|
|
await IndexPurlAsync(advisory.AffectsKey, advisory.MergeHash, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to cache advisory {MergeHash}", advisory.MergeHash);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "set");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task InvalidateAsync(string mergeHash, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
// Remove from advisory cache
|
|
var key = AdvisoryCacheKeys.Advisory(mergeHash, _options.KeyPrefix);
|
|
await db.KeyDeleteAsync(key).ConfigureAwait(false);
|
|
|
|
// Remove from hot set
|
|
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
|
|
await db.SortedSetRemoveAsync(hotKey, mergeHash).ConfigureAwait(false);
|
|
|
|
_metrics?.RecordEviction("invalidate");
|
|
_logger?.LogDebug("Invalidated advisory {MergeHash}", mergeHash);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to invalidate advisory {MergeHash}", mergeHash);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "invalidate");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task UpdateScoreAsync(string mergeHash, double score, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
|
|
|
|
// Add/update in hot set
|
|
await db.SortedSetAddAsync(hotKey, mergeHash, score).ConfigureAwait(false);
|
|
|
|
// Trim to max size
|
|
var currentSize = await db.SortedSetLengthAsync(hotKey).ConfigureAwait(false);
|
|
if (currentSize > _options.MaxHotSetSize)
|
|
{
|
|
// Remove lowest scoring entries
|
|
var removed = await db.SortedSetRemoveRangeByRankAsync(
|
|
hotKey,
|
|
start: 0,
|
|
stop: currentSize - _options.MaxHotSetSize - 1).ConfigureAwait(false);
|
|
|
|
if (removed > 0)
|
|
{
|
|
_metrics?.RecordEviction("hotset-trim");
|
|
}
|
|
}
|
|
|
|
_metrics?.UpdateHotSetSize(Math.Min(currentSize, _options.MaxHotSetSize));
|
|
|
|
// Update advisory TTL if cached
|
|
var advisoryKey = AdvisoryCacheKeys.Advisory(mergeHash, _options.KeyPrefix);
|
|
if (await db.KeyExistsAsync(advisoryKey).ConfigureAwait(false))
|
|
{
|
|
var ttl = _options.TtlPolicy.GetTtl(score);
|
|
await db.KeyExpireAsync(advisoryKey, ttl).ConfigureAwait(false);
|
|
}
|
|
|
|
_logger?.LogDebug("Updated score for {MergeHash} to {Score}", mergeHash, score);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to update score for {MergeHash}", mergeHash);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "update-score");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task IndexPurlAsync(string purl, string mergeHash, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var indexKey = AdvisoryCacheKeys.ByPurl(purl, _options.KeyPrefix);
|
|
|
|
await db.SetAddAsync(indexKey, mergeHash).ConfigureAwait(false);
|
|
await db.KeyExpireAsync(indexKey, _options.TtlPolicy.PurlIndexTtl).ConfigureAwait(false);
|
|
|
|
_logger?.LogDebug("Indexed PURL {Purl} -> {MergeHash}", purl, mergeHash);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to index PURL {Purl}", purl);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "index-purl");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task UnindexPurlAsync(string purl, string mergeHash, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var indexKey = AdvisoryCacheKeys.ByPurl(purl, _options.KeyPrefix);
|
|
|
|
await db.SetRemoveAsync(indexKey, mergeHash).ConfigureAwait(false);
|
|
|
|
_logger?.LogDebug("Unindexed PURL {Purl} -> {MergeHash}", purl, mergeHash);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to unindex PURL {Purl}", purl);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "unindex-purl");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task IndexCveAsync(string cve, string mergeHash, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
var mappingKey = AdvisoryCacheKeys.ByCve(cve, _options.KeyPrefix);
|
|
|
|
await db.StringSetAsync(mappingKey, mergeHash, _options.TtlPolicy.CveMappingTtl).ConfigureAwait(false);
|
|
|
|
_logger?.LogDebug("Indexed CVE {Cve} -> {MergeHash}", cve, mergeHash);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to index CVE {Cve}", cve);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "index-cve");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task WarmupAsync(int limit = 1000, CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled || !_options.EnableWarmup)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sw = Stopwatch.StartNew();
|
|
_logger?.LogInformation("Starting cache warmup (limit: {Limit})", limit);
|
|
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
// Try to acquire warmup lock (prevent concurrent warmups)
|
|
var lockKey = AdvisoryCacheKeys.WarmupLock(_options.KeyPrefix);
|
|
var lockAcquired = await db.StringSetAsync(
|
|
lockKey,
|
|
"warming",
|
|
TimeSpan.FromMinutes(10),
|
|
When.NotExists).ConfigureAwait(false);
|
|
|
|
if (!lockAcquired)
|
|
{
|
|
_logger?.LogDebug("Warmup already in progress, skipping");
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
// Record warmup timestamp
|
|
var warmupKey = AdvisoryCacheKeys.WarmupLast(_options.KeyPrefix);
|
|
await db.StringSetAsync(warmupKey, DateTimeOffset.UtcNow.ToString("o")).ConfigureAwait(false);
|
|
|
|
// Note: Actual warmup would load from ICanonicalAdvisoryStore
|
|
// This is a placeholder - the actual implementation would be in the integration layer
|
|
_logger?.LogInformation("Cache warmup completed in {Elapsed}ms", sw.ElapsedMilliseconds);
|
|
}
|
|
finally
|
|
{
|
|
// Release lock
|
|
await db.KeyDeleteAsync(lockKey).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Cache warmup failed after {Elapsed}ms", sw.ElapsedMilliseconds);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "warmup");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<CacheStatistics> GetStatisticsAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return new CacheStatistics { IsHealthy = false };
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
var db = await _connectionFactory.GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
var hitsKey = AdvisoryCacheKeys.StatsHits(_options.KeyPrefix);
|
|
var missesKey = AdvisoryCacheKeys.StatsMisses(_options.KeyPrefix);
|
|
var hotKey = AdvisoryCacheKeys.HotSet(_options.KeyPrefix);
|
|
var warmupKey = AdvisoryCacheKeys.WarmupLast(_options.KeyPrefix);
|
|
|
|
var hits = (long)(await db.StringGetAsync(hitsKey).ConfigureAwait(false));
|
|
var misses = (long)(await db.StringGetAsync(missesKey).ConfigureAwait(false));
|
|
var hotSetSize = await db.SortedSetLengthAsync(hotKey).ConfigureAwait(false);
|
|
_metrics?.UpdateHotSetSize(hotSetSize);
|
|
|
|
DateTimeOffset? lastWarmup = null;
|
|
var warmupStr = await db.StringGetAsync(warmupKey).ConfigureAwait(false);
|
|
if (warmupStr.HasValue && DateTimeOffset.TryParse(warmupStr, out var parsed))
|
|
{
|
|
lastWarmup = parsed;
|
|
}
|
|
|
|
// Get server info
|
|
var connection = await _connectionFactory.GetConnectionAsync(cancellationToken).ConfigureAwait(false);
|
|
var server = connection.GetServer(connection.GetEndPoints().First());
|
|
var info = (await server.InfoAsync().ConfigureAwait(false))
|
|
.FirstOrDefault(g => g.Key == "Server")?
|
|
.FirstOrDefault(e => e.Key == "redis_version")
|
|
.Value;
|
|
|
|
return new CacheStatistics
|
|
{
|
|
Hits = hits,
|
|
Misses = misses,
|
|
HotSetSize = hotSetSize,
|
|
TotalCachedAdvisories = hotSetSize, // Approximation
|
|
LastWarmup = lastWarmup,
|
|
IsHealthy = true,
|
|
ServerInfo = info
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger?.LogWarning(ex, "Failed to get cache statistics");
|
|
return new CacheStatistics { IsHealthy = false };
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "stats");
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<bool> IsHealthyAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
if (!_options.Enabled)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var sw = StartTiming();
|
|
try
|
|
{
|
|
return await _connectionFactory.PingAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
StopTiming(sw, "health");
|
|
}
|
|
}
|
|
|
|
private Stopwatch? StartTiming()
|
|
{
|
|
return _metrics is null ? null : Stopwatch.StartNew();
|
|
}
|
|
|
|
private void StopTiming(Stopwatch? sw, string operation)
|
|
{
|
|
if (sw is null || _metrics is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
sw.Stop();
|
|
_metrics.RecordLatency(sw.Elapsed.TotalMilliseconds, operation);
|
|
}
|
|
}
|