Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,279 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StackExchange.Redis;
|
||||
using StellaOps.BinaryIndex.Contracts.Resolution;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Cache;
|
||||
|
||||
/// <summary>
|
||||
/// Caching service for binary resolution results.
|
||||
/// Uses Valkey/Redis for high-performance caching with configurable TTLs.
|
||||
/// </summary>
|
||||
public interface IResolutionCacheService
|
||||
{
|
||||
/// <summary>
|
||||
/// Get cached resolution status.
|
||||
/// </summary>
|
||||
/// <param name="cacheKey">The cache key.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Cached resolution if found, null otherwise.</returns>
|
||||
Task<CachedResolution?> GetAsync(string cacheKey, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Cache resolution result.
|
||||
/// </summary>
|
||||
/// <param name="cacheKey">The cache key.</param>
|
||||
/// <param name="result">The resolution result to cache.</param>
|
||||
/// <param name="ttl">Time-to-live for the cache entry.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
Task SetAsync(string cacheKey, CachedResolution result, TimeSpan ttl, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Invalidate cache entries by pattern.
|
||||
/// </summary>
|
||||
/// <param name="pattern">Redis pattern (e.g., "resolution:*:debian:*").</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
Task InvalidateByPatternAsync(string pattern, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Generate cache key from resolution request.
|
||||
/// </summary>
|
||||
/// <param name="request">The resolution request.</param>
|
||||
/// <returns>Deterministic cache key.</returns>
|
||||
string GenerateCacheKey(VulnResolutionRequest request);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cached resolution entry.
|
||||
/// </summary>
|
||||
public sealed record CachedResolution
|
||||
{
|
||||
/// <summary>Resolution status.</summary>
|
||||
public required ResolutionStatus Status { get; init; }
|
||||
|
||||
/// <summary>Fixed version if applicable.</summary>
|
||||
public string? FixedVersion { get; init; }
|
||||
|
||||
/// <summary>Reference to evidence record.</summary>
|
||||
public string? EvidenceRef { get; init; }
|
||||
|
||||
/// <summary>When this entry was cached.</summary>
|
||||
public DateTimeOffset CachedAt { get; init; }
|
||||
|
||||
/// <summary>Version key for invalidation.</summary>
|
||||
public string? VersionKey { get; init; }
|
||||
|
||||
/// <summary>Confidence score.</summary>
|
||||
public decimal Confidence { get; init; }
|
||||
|
||||
/// <summary>Match type used.</summary>
|
||||
public string? MatchType { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for resolution caching.
|
||||
/// </summary>
|
||||
public sealed class ResolutionCacheOptions
|
||||
{
|
||||
/// <summary>Configuration section name.</summary>
|
||||
public const string SectionName = "ResolutionCache";
|
||||
|
||||
/// <summary>TTL for fixed (high confidence) results.</summary>
|
||||
public TimeSpan FixedTtl { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>TTL for vulnerable results.</summary>
|
||||
public TimeSpan VulnerableTtl { get; set; } = TimeSpan.FromHours(4);
|
||||
|
||||
/// <summary>TTL for unknown results.</summary>
|
||||
public TimeSpan UnknownTtl { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>Cache key prefix.</summary>
|
||||
public string KeyPrefix { get; set; } = "resolution";
|
||||
|
||||
/// <summary>Enable probabilistic early expiry to prevent stampedes.</summary>
|
||||
public bool EnableEarlyExpiry { get; set; } = true;
|
||||
|
||||
/// <summary>Early expiry factor (0.0-1.0).</summary>
|
||||
public double EarlyExpiryFactor { get; set; } = 0.1;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Valkey/Redis implementation of resolution caching.
|
||||
/// </summary>
|
||||
public sealed class ResolutionCacheService : IResolutionCacheService
|
||||
{
|
||||
private readonly IConnectionMultiplexer _redis;
|
||||
private readonly ResolutionCacheOptions _options;
|
||||
private readonly ILogger<ResolutionCacheService> _logger;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public ResolutionCacheService(
|
||||
IConnectionMultiplexer redis,
|
||||
IOptions<ResolutionCacheOptions> options,
|
||||
ILogger<ResolutionCacheService> logger)
|
||||
{
|
||||
_redis = redis ?? throw new ArgumentNullException(nameof(redis));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<CachedResolution?> GetAsync(string cacheKey, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var db = _redis.GetDatabase();
|
||||
var value = await db.StringGetAsync(cacheKey);
|
||||
|
||||
if (value.IsNullOrEmpty)
|
||||
{
|
||||
_logger.LogDebug("Cache miss for key {CacheKey}", cacheKey);
|
||||
return null;
|
||||
}
|
||||
|
||||
var cached = JsonSerializer.Deserialize<CachedResolution>(value.ToString(), _jsonOptions);
|
||||
|
||||
// Check for probabilistic early expiry
|
||||
if (_options.EnableEarlyExpiry && cached is not null)
|
||||
{
|
||||
var ttl = await db.KeyTimeToLiveAsync(cacheKey);
|
||||
if (ShouldExpireEarly(ttl))
|
||||
{
|
||||
_logger.LogDebug("Early expiry triggered for key {CacheKey}", cacheKey);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Cache hit for key {CacheKey}", cacheKey);
|
||||
return cached;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to get cache entry for key {CacheKey}", cacheKey);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task SetAsync(string cacheKey, CachedResolution result, TimeSpan ttl, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var db = _redis.GetDatabase();
|
||||
var value = JsonSerializer.Serialize(result, _jsonOptions);
|
||||
|
||||
await db.StringSetAsync(cacheKey, value, ttl);
|
||||
_logger.LogDebug("Cached resolution for key {CacheKey} with TTL {Ttl}", cacheKey, ttl);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to cache resolution for key {CacheKey}", cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InvalidateByPatternAsync(string pattern, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var server = _redis.GetServer(_redis.GetEndPoints().First());
|
||||
var db = _redis.GetDatabase();
|
||||
|
||||
var keys = server.Keys(pattern: pattern).ToArray();
|
||||
|
||||
if (keys.Length > 0)
|
||||
{
|
||||
await db.KeyDeleteAsync(keys);
|
||||
_logger.LogInformation("Invalidated {Count} cache entries matching pattern {Pattern}",
|
||||
keys.Length, pattern);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to invalidate cache entries matching pattern {Pattern}", pattern);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string GenerateCacheKey(VulnResolutionRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
// Build deterministic cache key
|
||||
// Format: resolution:{algorithm}:{hash}:{cve_id_or_all}
|
||||
var algorithm = DetermineAlgorithm(request);
|
||||
var hash = ComputeIdentityHash(request);
|
||||
var cveId = request.CveId ?? "all";
|
||||
|
||||
return $"{_options.KeyPrefix}:{algorithm}:{hash}:{cveId}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get appropriate TTL based on resolution status.
|
||||
/// </summary>
|
||||
public TimeSpan GetTtlForStatus(ResolutionStatus status)
|
||||
{
|
||||
return status switch
|
||||
{
|
||||
ResolutionStatus.Fixed => _options.FixedTtl,
|
||||
ResolutionStatus.Vulnerable => _options.VulnerableTtl,
|
||||
ResolutionStatus.NotAffected => _options.FixedTtl,
|
||||
_ => _options.UnknownTtl
|
||||
};
|
||||
}
|
||||
|
||||
private static string DetermineAlgorithm(VulnResolutionRequest request)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(request.BuildId))
|
||||
return "build_id";
|
||||
if (!string.IsNullOrEmpty(request.Fingerprint))
|
||||
return request.FingerprintAlgorithm ?? "combined";
|
||||
if (request.Hashes?.TextSha256 != null)
|
||||
return "text_sha256";
|
||||
if (request.Hashes?.FileSha256 != null)
|
||||
return "file_sha256";
|
||||
return "package";
|
||||
}
|
||||
|
||||
private static string ComputeIdentityHash(VulnResolutionRequest request)
|
||||
{
|
||||
// Use the most specific identifier available
|
||||
if (!string.IsNullOrEmpty(request.BuildId))
|
||||
return request.BuildId;
|
||||
if (!string.IsNullOrEmpty(request.Fingerprint))
|
||||
return ComputeShortHash(request.Fingerprint);
|
||||
if (request.Hashes?.TextSha256 != null)
|
||||
return request.Hashes.TextSha256;
|
||||
if (request.Hashes?.FileSha256 != null)
|
||||
return request.Hashes.FileSha256;
|
||||
|
||||
// Fall back to package + distro
|
||||
var key = $"{request.Package}:{request.DistroRelease ?? "unknown"}";
|
||||
return ComputeShortHash(key);
|
||||
}
|
||||
|
||||
private static string ComputeShortHash(string input)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(input);
|
||||
var hash = System.Security.Cryptography.SHA256.HashData(bytes);
|
||||
return Convert.ToHexStringLower(hash)[..16];
|
||||
}
|
||||
|
||||
private bool ShouldExpireEarly(TimeSpan? remainingTtl)
|
||||
{
|
||||
if (!remainingTtl.HasValue || remainingTtl.Value <= TimeSpan.Zero)
|
||||
return true;
|
||||
|
||||
// Probabilistic early expiry using exponential decay
|
||||
var random = Random.Shared.NextDouble();
|
||||
var threshold = _options.EarlyExpiryFactor * Math.Exp(-remainingTtl.Value.TotalSeconds / 3600);
|
||||
|
||||
return random < threshold;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user