Restructure solution layout by module
This commit is contained in:
15
src/Scanner/__Libraries/StellaOps.Scanner.Cache/AGENTS.md
Normal file
15
src/Scanner/__Libraries/StellaOps.Scanner.Cache/AGENTS.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# StellaOps.Scanner.Cache — Agent Charter
|
||||
|
||||
## Mission
|
||||
Provide deterministic, offline-friendly caching primitives for scanner layers and file content so warm scans complete in <5 s and cache reuse remains reproducible across deployments.
|
||||
|
||||
## Responsibilities
|
||||
- Implement layer cache keyed by layer digest, retaining analyzer metadata and provenance per architecture §3.3.
|
||||
- Deliver file content-addressable storage (CAS) with deduplication, TTL enforcement, and offline import/export hooks.
|
||||
- Expose structured metrics, health probes, and configuration toggles for cache sizing, eviction, and warm/cold thresholds.
|
||||
- Coordinate invalidation workflows (layer purge, TTL expiry, diff invalidation) while keeping deterministic logs and telemetry.
|
||||
|
||||
## Interfaces & Dependencies
|
||||
- Relies on `StackExchange.Redis` via `StellaOps.DependencyInjection` bindings for cache state.
|
||||
- Coordinates with `StellaOps.Scanner.Storage` object store when persisting immutable artifacts.
|
||||
- Targets `net10.0` preview SDK and follows scanner coding standards from `docs/18_CODING_STANDARDS.md`.
|
||||
@@ -0,0 +1,48 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface IFileContentAddressableStore
|
||||
{
|
||||
ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record FileCasEntry(
|
||||
string Sha256,
|
||||
long SizeBytes,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
string RelativePath);
|
||||
|
||||
public sealed class FileCasPutRequest
|
||||
{
|
||||
public string Sha256 { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public FileCasPutRequest(string sha256, Stream content, bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
throw new ArgumentException("SHA-256 identifier must be provided.", nameof(sha256));
|
||||
}
|
||||
|
||||
Sha256 = sha256;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface ILayerCacheStore
|
||||
{
|
||||
ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Represents cached metadata for a single layer digest.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheEntry(
|
||||
string LayerDigest,
|
||||
string Architecture,
|
||||
string MediaType,
|
||||
DateTimeOffset CachedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
long TotalSizeBytes,
|
||||
IReadOnlyDictionary<string, LayerCacheArtifactReference> Artifacts,
|
||||
IReadOnlyDictionary<string, string> Metadata)
|
||||
{
|
||||
public bool IsExpired(DateTimeOffset utcNow, TimeSpan ttl)
|
||||
=> utcNow - CachedAt >= ttl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Points to a cached artifact stored on disk.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheArtifactReference(
|
||||
string Name,
|
||||
string RelativePath,
|
||||
string ContentType,
|
||||
long SizeBytes,
|
||||
bool IsImmutable = false);
|
||||
@@ -0,0 +1,93 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Describes layer cache content to be stored.
|
||||
/// </summary>
|
||||
public sealed class LayerCachePutRequest
|
||||
{
|
||||
public string LayerDigest { get; }
|
||||
|
||||
public string Architecture { get; }
|
||||
|
||||
public string MediaType { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; }
|
||||
|
||||
public IReadOnlyList<LayerCacheArtifactContent> Artifacts { get; }
|
||||
|
||||
public LayerCachePutRequest(
|
||||
string layerDigest,
|
||||
string architecture,
|
||||
string mediaType,
|
||||
IReadOnlyDictionary<string, string> metadata,
|
||||
IReadOnlyList<LayerCacheArtifactContent> artifacts)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(layerDigest))
|
||||
{
|
||||
throw new ArgumentException("Layer digest must be provided.", nameof(layerDigest));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(architecture))
|
||||
{
|
||||
throw new ArgumentException("Architecture must be provided.", nameof(architecture));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mediaType))
|
||||
{
|
||||
throw new ArgumentException("Media type must be provided.", nameof(mediaType));
|
||||
}
|
||||
|
||||
Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
|
||||
Artifacts = artifacts ?? throw new ArgumentNullException(nameof(artifacts));
|
||||
if (artifacts.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one artifact must be supplied.", nameof(artifacts));
|
||||
}
|
||||
|
||||
LayerDigest = layerDigest;
|
||||
Architecture = architecture;
|
||||
MediaType = mediaType;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stream payload for a cached artifact.
|
||||
/// </summary>
|
||||
public sealed class LayerCacheArtifactContent
|
||||
{
|
||||
public string Name { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public string ContentType { get; }
|
||||
|
||||
public bool Immutable { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public LayerCacheArtifactContent(
|
||||
string name,
|
||||
Stream content,
|
||||
string contentType,
|
||||
bool immutable = false,
|
||||
bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
throw new ArgumentException("Artifact name must be provided.", nameof(name));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
throw new ArgumentException("Content type must be provided.", nameof(contentType));
|
||||
}
|
||||
|
||||
Name = name;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
ContentType = contentType;
|
||||
Immutable = immutable;
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
public sealed class FileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
private const string ContentFileName = "content.bin";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<FileContentAddressableStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public FileContentAddressableStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<FileContentAddressableStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entryDirectory = GetEntryDirectory(sha256);
|
||||
var metadataPath = Path.Combine(entryDirectory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasHit(sha256);
|
||||
|
||||
return new FileCasEntry(
|
||||
metadata.Sha256,
|
||||
metadata.SizeBytes,
|
||||
metadata.CreatedAt,
|
||||
metadata.LastAccessed,
|
||||
GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var sha = request.Sha256;
|
||||
var directory = GetEntryDirectory(sha);
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var contentPath = Path.Combine(directory, ContentFileName);
|
||||
await using (var destination = new FileStream(contentPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await request.Content.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!request.LeaveOpen)
|
||||
{
|
||||
request.Content.Dispose();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var sizeBytes = new FileInfo(contentPath).Length;
|
||||
var metadata = new FileCasMetadata
|
||||
{
|
||||
Sha256 = NormalizeHash(sha),
|
||||
CreatedAt = now,
|
||||
LastAccessed = now,
|
||||
SizeBytes = sizeBytes
|
||||
};
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasBytes(sizeBytes);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Stored CAS entry {Sha256} ({SizeBytes} bytes)", sha, sizeBytes);
|
||||
return new FileCasEntry(metadata.Sha256, metadata.SizeBytes, metadata.CreatedAt, metadata.LastAccessed, GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetEntryDirectory(sha256);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(metadata.Sha256);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} CAS entries due to TTL", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(destinationDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Directory.CreateDirectory(destinationDirectory);
|
||||
var exported = 0;
|
||||
|
||||
foreach (var entryDirectory in EnumerateEntryDirectories())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var hash = Path.GetFileName(entryDirectory);
|
||||
if (hash is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = Path.Combine(destinationDirectory, hash);
|
||||
if (Directory.Exists(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
CopyDirectory(entryDirectory, target);
|
||||
exported++;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Exported {Count} CAS entries to {Destination}", exported, destinationDirectory);
|
||||
return exported;
|
||||
}
|
||||
|
||||
public async Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sourceDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!Directory.Exists(sourceDirectory))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var imported = 0;
|
||||
foreach (var directory in Directory.EnumerateDirectories(sourceDirectory))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var destination = GetEntryDirectory(metadata.Sha256);
|
||||
if (Directory.Exists(destination))
|
||||
{
|
||||
// Only overwrite if the source is newer.
|
||||
var existingMetadataPath = Path.Combine(destination, MetadataFileName);
|
||||
var existing = await ReadMetadataAsync(existingMetadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && existing.CreatedAt >= metadata.CreatedAt)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(destination).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
CopyDirectory(directory, destination);
|
||||
imported++;
|
||||
}
|
||||
|
||||
if (imported > 0)
|
||||
{
|
||||
_logger.LogInformation("Imported {Count} CAS entries from {Source}", imported, sourceDirectory);
|
||||
}
|
||||
|
||||
return imported;
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(FileCasMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var target = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= target)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(entry.Directory).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordFileCasEviction(entry.Metadata.Sha256);
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
{
|
||||
_logger.LogInformation("Compacted CAS store, removed {Count} entries", removed);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.FileCasDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.FileCasDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateEntryDirectories()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var directory in Directory.EnumerateDirectories(_options.FileCasDirectoryPath, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (File.Exists(Path.Combine(directory, MetadataFileName)))
|
||||
{
|
||||
yield return directory;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FileCasMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<FileCasMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read CAS metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, FileCasMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private static string GetRelativeContentPath(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(NormalizedPrefix(normalized, 0, 2), NormalizedPrefix(normalized, 2, 2), normalized, ContentFileName);
|
||||
}
|
||||
|
||||
private string GetEntryDirectory(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(
|
||||
_options.FileCasDirectoryPath,
|
||||
NormalizedPrefix(normalized, 0, 2),
|
||||
NormalizedPrefix(normalized, 2, 2),
|
||||
normalized);
|
||||
}
|
||||
|
||||
private static string NormalizeHash(string sha256)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = sha256.Contains(':', StringComparison.Ordinal) ? sha256[(sha256.IndexOf(':') + 1)..] : sha256;
|
||||
return hash.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizedPrefix(string hash, int offset, int length)
|
||||
{
|
||||
if (hash.Length <= offset)
|
||||
{
|
||||
return "00";
|
||||
}
|
||||
|
||||
if (hash.Length < offset + length)
|
||||
{
|
||||
length = hash.Length - offset;
|
||||
}
|
||||
|
||||
return hash.Substring(offset, length);
|
||||
}
|
||||
|
||||
private bool IsExpired(FileCasMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CreatedAt >= _options.FileTtl;
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string sourceDir, string destDir)
|
||||
{
|
||||
Directory.CreateDirectory(destDir);
|
||||
foreach (var file in Directory.EnumerateFiles(sourceDir, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(sourceDir, file);
|
||||
var destination = Path.Combine(destDir, relative);
|
||||
var parent = Path.GetDirectoryName(destination);
|
||||
if (!string.IsNullOrEmpty(parent))
|
||||
{
|
||||
Directory.CreateDirectory(parent);
|
||||
}
|
||||
File.Copy(file, destination, overwrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
private Task RemoveDirectoryAsync(string directory)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete CAS directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FileCasMetadata
|
||||
{
|
||||
public string Sha256 { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
internal sealed class NullFileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
public ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<FileCasEntry?>(null);
|
||||
|
||||
public Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
=> Task.FromException<FileCasEntry>(new InvalidOperationException("File CAS is disabled via configuration."));
|
||||
|
||||
public Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(false);
|
||||
|
||||
public Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.LayerCache;
|
||||
|
||||
public sealed class LayerCacheStore : ILayerCacheStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<LayerCacheStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public LayerCacheStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<LayerCacheStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogTrace("Layer cache miss: directory {Directory} not found for {LayerDigest}", directory, layerDigest);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache metadata missing at {Path} for {LayerDigest}; removing directory", metadataPath, layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
_logger.LogDebug("Layer cache entry {LayerDigest} expired at {Expiration}", metadata.LayerDigest, metadata.CachedAt + _options.LayerTtl);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerHit(layerDigest);
|
||||
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = request.LayerDigest;
|
||||
var directory = GetLayerDirectory(digest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogDebug("Replacing existing layer cache entry for {LayerDigest}", digest);
|
||||
await RemoveInternalAsync(directory, digest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var artifactMetadata = new Dictionary<string, LayerCacheArtifactMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
long totalSize = 0;
|
||||
|
||||
foreach (var artifact in request.Artifacts)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var fileName = SanitizeArtifactName(artifact.Name);
|
||||
var relativePath = Path.Combine("artifacts", fileName);
|
||||
var artifactDirectory = Path.Combine(directory, "artifacts");
|
||||
Directory.CreateDirectory(artifactDirectory);
|
||||
var filePath = Path.Combine(directory, relativePath);
|
||||
|
||||
await using (var target = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await artifact.Content.CopyToAsync(target, cancellationToken).ConfigureAwait(false);
|
||||
await target.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
totalSize += target.Length;
|
||||
}
|
||||
|
||||
if (!artifact.LeaveOpen)
|
||||
{
|
||||
artifact.Content.Dispose();
|
||||
}
|
||||
|
||||
var sizeBytes = new FileInfo(filePath).Length;
|
||||
|
||||
artifactMetadata[artifact.Name] = new LayerCacheArtifactMetadata
|
||||
{
|
||||
Name = artifact.Name,
|
||||
ContentType = artifact.ContentType,
|
||||
RelativePath = relativePath,
|
||||
SizeBytes = sizeBytes,
|
||||
Immutable = artifact.Immutable
|
||||
};
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var metadata = new LayerCacheMetadata
|
||||
{
|
||||
LayerDigest = digest,
|
||||
Architecture = request.Architecture,
|
||||
MediaType = request.MediaType,
|
||||
CachedAt = now,
|
||||
LastAccessed = now,
|
||||
Metadata = new Dictionary<string, string>(request.Metadata, StringComparer.Ordinal),
|
||||
Artifacts = artifactMetadata,
|
||||
SizeBytes = totalSize
|
||||
};
|
||||
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerBytes(totalSize);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Cached layer {LayerDigest} with {ArtifactCount} artifacts ({SizeBytes} bytes)", digest, artifactMetadata.Count, totalSize);
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveInternalAsync(directory, metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(metadata.LayerDigest);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} expired layer cache entries", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!metadata.Artifacts.TryGetValue(artifactName, out var artifact))
|
||||
{
|
||||
_logger.LogDebug("Layer cache artifact {Artifact} missing for {LayerDigest}", artifactName, layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var filePath = Path.Combine(directory, artifact.RelativePath);
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache file {FilePath} not found for artifact {Artifact}", filePath, artifactName);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = _timeProvider.GetUtcNow();
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(LayerCacheMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var targetBytes = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= targetBytes)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveInternalAsync(entry.Directory, entry.Metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordLayerEviction(entry.Metadata.LayerDigest);
|
||||
_logger.LogInformation("Evicted layer {LayerDigest} during compaction (remaining bytes: {Bytes})", entry.Metadata.LayerDigest, totalBytes);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.LayersDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.LayersDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.LayersDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<LayerCacheMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<LayerCacheMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load layer cache metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, LayerCacheMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private Task RemoveInternalAsync(string directory, string layerDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
_logger.LogDebug("Removed layer cache entry {LayerDigest}", layerDigest);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete layer cache directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private bool IsExpired(LayerCacheMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (metadata.CachedAt == default)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CachedAt >= _options.LayerTtl;
|
||||
}
|
||||
|
||||
private LayerCacheEntry Map(LayerCacheMetadata metadata)
|
||||
{
|
||||
var artifacts = metadata.Artifacts?.ToDictionary(
|
||||
pair => pair.Key,
|
||||
pair => new LayerCacheArtifactReference(
|
||||
pair.Value.Name,
|
||||
pair.Value.RelativePath,
|
||||
pair.Value.ContentType,
|
||||
pair.Value.SizeBytes,
|
||||
pair.Value.Immutable),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, LayerCacheArtifactReference>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
return new LayerCacheEntry(
|
||||
metadata.LayerDigest,
|
||||
metadata.Architecture,
|
||||
metadata.MediaType,
|
||||
metadata.CachedAt,
|
||||
metadata.LastAccessed,
|
||||
metadata.SizeBytes,
|
||||
artifacts,
|
||||
metadata.Metadata is null
|
||||
? new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, string>(metadata.Metadata, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
private string GetLayerDirectory(string layerDigest)
|
||||
{
|
||||
var safeDigest = SanitizeDigest(layerDigest);
|
||||
return Path.Combine(_options.LayersDirectoryPath, safeDigest);
|
||||
}
|
||||
|
||||
private static string SanitizeArtifactName(string name)
|
||||
{
|
||||
var fileName = Path.GetFileName(name);
|
||||
return string.IsNullOrWhiteSpace(fileName) ? "artifact" : fileName;
|
||||
}
|
||||
|
||||
private static string SanitizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = digest.Contains(':', StringComparison.Ordinal)
|
||||
? digest[(digest.IndexOf(':') + 1)..]
|
||||
: digest;
|
||||
|
||||
var buffer = new char[hash.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in hash)
|
||||
{
|
||||
buffer[count++] = char.IsLetterOrDigit(ch) ? char.ToLowerInvariant(ch) : '_';
|
||||
}
|
||||
|
||||
return new string(buffer, 0, count);
|
||||
}
|
||||
|
||||
private sealed class LayerCacheMetadata
|
||||
{
|
||||
public string LayerDigest { get; set; } = string.Empty;
|
||||
|
||||
public string Architecture { get; set; } = string.Empty;
|
||||
|
||||
public string MediaType { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CachedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
|
||||
public Dictionary<string, LayerCacheArtifactMetadata> Artifacts { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
}
|
||||
|
||||
private sealed class LayerCacheArtifactMetadata
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
public bool Immutable { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
public sealed class ScannerCacheMaintenanceService : BackgroundService
|
||||
{
|
||||
private readonly ILayerCacheStore _layerCache;
|
||||
private readonly IFileContentAddressableStore _fileCas;
|
||||
private readonly IOptions<ScannerCacheOptions> _options;
|
||||
private readonly ILogger<ScannerCacheMaintenanceService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ScannerCacheMaintenanceService(
|
||||
ILayerCacheStore layerCache,
|
||||
IFileContentAddressableStore fileCas,
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<ScannerCacheMaintenanceService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_layerCache = layerCache ?? throw new ArgumentNullException(nameof(layerCache));
|
||||
_fileCas = fileCas ?? throw new ArgumentNullException(nameof(fileCas));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var settings = _options.Value;
|
||||
if (!settings.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache disabled; maintenance loop will not start.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings.EnableAutoEviction)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache automatic eviction disabled by configuration.");
|
||||
return;
|
||||
}
|
||||
|
||||
var interval = settings.MaintenanceInterval > TimeSpan.Zero
|
||||
? settings.MaintenanceInterval
|
||||
: TimeSpan.FromMinutes(15);
|
||||
|
||||
_logger.LogInformation("Scanner cache maintenance loop started with interval {Interval}", interval);
|
||||
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
using var timer = new PeriodicTimer(interval, _timeProvider);
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunMaintenanceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var layerExpired = await _layerCache.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var layerCompacted = await _layerCache.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casExpired = await _fileCas.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casCompacted = await _fileCas.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Scanner cache maintenance tick complete (layers expired={LayersExpired}, layers compacted={LayersCompacted}, cas expired={CasExpired}, cas compacted={CasCompacted})",
|
||||
layerExpired,
|
||||
layerCompacted,
|
||||
casExpired,
|
||||
casCompacted);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutting down; ignore.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Scanner cache maintenance tick failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheMetrics
|
||||
{
|
||||
public const string MeterName = "StellaOps.Scanner.Cache";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName, "1.0.0");
|
||||
|
||||
private static readonly Counter<long> LayerHits = Meter.CreateCounter<long>("scanner.layer_cache_hits_total");
|
||||
private static readonly Counter<long> LayerMisses = Meter.CreateCounter<long>("scanner.layer_cache_misses_total");
|
||||
private static readonly Counter<long> LayerEvictions = Meter.CreateCounter<long>("scanner.layer_cache_evictions_total");
|
||||
private static readonly Histogram<long> LayerBytes = Meter.CreateHistogram<long>("scanner.layer_cache_bytes");
|
||||
private static readonly Counter<long> FileCasHits = Meter.CreateCounter<long>("scanner.file_cas_hits_total");
|
||||
private static readonly Counter<long> FileCasMisses = Meter.CreateCounter<long>("scanner.file_cas_misses_total");
|
||||
private static readonly Counter<long> FileCasEvictions = Meter.CreateCounter<long>("scanner.file_cas_evictions_total");
|
||||
private static readonly Histogram<long> FileCasBytes = Meter.CreateHistogram<long>("scanner.file_cas_bytes");
|
||||
|
||||
public static void RecordLayerHit(string layerDigest)
|
||||
=> LayerHits.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerMiss(string layerDigest)
|
||||
=> LayerMisses.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerEviction(string layerDigest)
|
||||
=> LayerEvictions.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerBytes(long bytes)
|
||||
=> LayerBytes.Record(bytes);
|
||||
|
||||
public static void RecordFileCasHit(string sha256)
|
||||
=> FileCasHits.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasMiss(string sha256)
|
||||
=> FileCasMisses.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasEviction(string sha256)
|
||||
=> FileCasEvictions.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasBytes(long bytes)
|
||||
=> FileCasBytes.Record(bytes);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public sealed class ScannerCacheOptions
|
||||
{
|
||||
private const long DefaultMaxBytes = 5L * 1024 * 1024 * 1024; // 5 GiB
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string RootPath { get; set; } = Path.Combine("cache", "scanner");
|
||||
|
||||
public string LayersDirectoryName { get; set; } = "layers";
|
||||
|
||||
public string FileCasDirectoryName { get; set; } = "cas";
|
||||
|
||||
public TimeSpan LayerTtl { get; set; } = TimeSpan.FromDays(45);
|
||||
|
||||
public TimeSpan FileTtl { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public long MaxBytes { get; set; } = DefaultMaxBytes;
|
||||
|
||||
public long WarmBytesThreshold { get; set; } = DefaultMaxBytes / 5; // 20 %
|
||||
|
||||
public long ColdBytesThreshold { get; set; } = (DefaultMaxBytes * 4) / 5; // 80 %
|
||||
|
||||
public bool EnableAutoEviction { get; set; } = true;
|
||||
|
||||
public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
public bool EnableFileCas { get; set; } = true;
|
||||
|
||||
public string? ImportDirectory { get; set; }
|
||||
|
||||
public string? ExportDirectory { get; set; }
|
||||
|
||||
public string LayersDirectoryPath => Path.Combine(RootPath, LayersDirectoryName);
|
||||
|
||||
public string FileCasDirectoryPath => Path.Combine(RootPath, FileCasDirectoryName);
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
using StellaOps.Scanner.Cache.FileCas;
|
||||
using StellaOps.Scanner.Cache.LayerCache;
|
||||
using StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddScannerCache(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "scanner:cache")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<ScannerCacheOptions>()
|
||||
.Bind(configuration.GetSection(sectionName))
|
||||
.Validate(options => !string.IsNullOrWhiteSpace(options.RootPath), "scanner:cache:rootPath must be configured");
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
|
||||
services.TryAddSingleton<ILayerCacheStore, LayerCacheStore>();
|
||||
services.TryAddSingleton<IFileContentAddressableStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerCacheOptions>>();
|
||||
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
|
||||
if (!options.Value.EnableFileCas)
|
||||
{
|
||||
return new NullFileContentAddressableStore();
|
||||
}
|
||||
|
||||
return new FileContentAddressableStore(
|
||||
options,
|
||||
loggerFactory.CreateLogger<FileContentAddressableStore>(),
|
||||
timeProvider);
|
||||
});
|
||||
|
||||
services.AddHostedService<ScannerCacheMaintenanceService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.7.33" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
10
src/Scanner/__Libraries/StellaOps.Scanner.Cache/TASKS.md
Normal file
10
src/Scanner/__Libraries/StellaOps.Scanner.Cache/TASKS.md
Normal file
@@ -0,0 +1,10 @@
|
||||
# Scanner Cache Task Board (Sprint 10)
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|
||||
|----|--------|----------|------------|-------------|---------------|
|
||||
| SCANNER-CACHE-10-101 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-WORKER-09-201 | Implement layer cache store keyed by layer digest with metadata retention aligned with architecture §3.3 object layout. | Layer cache API supports get/put/delete by digest; metadata persisted with deterministic serialization; warm lookup covered by tests. |
|
||||
| SCANNER-CACHE-10-102 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks for offline kit workflows. | CAS stores content by SHA-256, enforces TTL policy, import/export commands documented and exercised in tests. |
|
||||
| SCANNER-CACHE-10-103 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | Metrics counters/gauges emitted; options validated; logs include correlation IDs; configuration doc references settings. |
|
||||
| SCANNER-CACHE-10-104 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | Invalidation API implemented with deterministic eviction; tests cover TTL expiry + explicit delete; logs instrumented. |
|
||||
|
||||
> Update statuses to DONE once acceptance criteria and tests/documentation are delivered.
|
||||
Reference in New Issue
Block a user