up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -1,48 +1,48 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface IFileContentAddressableStore
|
||||
{
|
||||
ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record FileCasEntry(
|
||||
string Sha256,
|
||||
long SizeBytes,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
string RelativePath);
|
||||
|
||||
public sealed class FileCasPutRequest
|
||||
{
|
||||
public string Sha256 { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public FileCasPutRequest(string sha256, Stream content, bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
throw new ArgumentException("SHA-256 identifier must be provided.", nameof(sha256));
|
||||
}
|
||||
|
||||
Sha256 = sha256;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface IFileContentAddressableStore
|
||||
{
|
||||
ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record FileCasEntry(
|
||||
string Sha256,
|
||||
long SizeBytes,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
string RelativePath);
|
||||
|
||||
public sealed class FileCasPutRequest
|
||||
{
|
||||
public string Sha256 { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public FileCasPutRequest(string sha256, Stream content, bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
throw new ArgumentException("SHA-256 identifier must be provided.", nameof(sha256));
|
||||
}
|
||||
|
||||
Sha256 = sha256;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface ILayerCacheStore
|
||||
{
|
||||
ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
public interface ILayerCacheStore
|
||||
{
|
||||
ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default);
|
||||
|
||||
Task<int> CompactAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Represents cached metadata for a single layer digest.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheEntry(
|
||||
string LayerDigest,
|
||||
string Architecture,
|
||||
string MediaType,
|
||||
DateTimeOffset CachedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
long TotalSizeBytes,
|
||||
IReadOnlyDictionary<string, LayerCacheArtifactReference> Artifacts,
|
||||
IReadOnlyDictionary<string, string> Metadata)
|
||||
{
|
||||
public bool IsExpired(DateTimeOffset utcNow, TimeSpan ttl)
|
||||
=> utcNow - CachedAt >= ttl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Points to a cached artifact stored on disk.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheArtifactReference(
|
||||
string Name,
|
||||
string RelativePath,
|
||||
string ContentType,
|
||||
long SizeBytes,
|
||||
bool IsImmutable = false);
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Represents cached metadata for a single layer digest.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheEntry(
|
||||
string LayerDigest,
|
||||
string Architecture,
|
||||
string MediaType,
|
||||
DateTimeOffset CachedAt,
|
||||
DateTimeOffset LastAccessed,
|
||||
long TotalSizeBytes,
|
||||
IReadOnlyDictionary<string, LayerCacheArtifactReference> Artifacts,
|
||||
IReadOnlyDictionary<string, string> Metadata)
|
||||
{
|
||||
public bool IsExpired(DateTimeOffset utcNow, TimeSpan ttl)
|
||||
=> utcNow - CachedAt >= ttl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Points to a cached artifact stored on disk.
|
||||
/// </summary>
|
||||
public sealed record LayerCacheArtifactReference(
|
||||
string Name,
|
||||
string RelativePath,
|
||||
string ContentType,
|
||||
long SizeBytes,
|
||||
bool IsImmutable = false);
|
||||
|
||||
@@ -1,93 +1,93 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Describes layer cache content to be stored.
|
||||
/// </summary>
|
||||
public sealed class LayerCachePutRequest
|
||||
{
|
||||
public string LayerDigest { get; }
|
||||
|
||||
public string Architecture { get; }
|
||||
|
||||
public string MediaType { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; }
|
||||
|
||||
public IReadOnlyList<LayerCacheArtifactContent> Artifacts { get; }
|
||||
|
||||
public LayerCachePutRequest(
|
||||
string layerDigest,
|
||||
string architecture,
|
||||
string mediaType,
|
||||
IReadOnlyDictionary<string, string> metadata,
|
||||
IReadOnlyList<LayerCacheArtifactContent> artifacts)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(layerDigest))
|
||||
{
|
||||
throw new ArgumentException("Layer digest must be provided.", nameof(layerDigest));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(architecture))
|
||||
{
|
||||
throw new ArgumentException("Architecture must be provided.", nameof(architecture));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mediaType))
|
||||
{
|
||||
throw new ArgumentException("Media type must be provided.", nameof(mediaType));
|
||||
}
|
||||
|
||||
Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
|
||||
Artifacts = artifacts ?? throw new ArgumentNullException(nameof(artifacts));
|
||||
if (artifacts.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one artifact must be supplied.", nameof(artifacts));
|
||||
}
|
||||
|
||||
LayerDigest = layerDigest;
|
||||
Architecture = architecture;
|
||||
MediaType = mediaType;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stream payload for a cached artifact.
|
||||
/// </summary>
|
||||
public sealed class LayerCacheArtifactContent
|
||||
{
|
||||
public string Name { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public string ContentType { get; }
|
||||
|
||||
public bool Immutable { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public LayerCacheArtifactContent(
|
||||
string name,
|
||||
Stream content,
|
||||
string contentType,
|
||||
bool immutable = false,
|
||||
bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
throw new ArgumentException("Artifact name must be provided.", nameof(name));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
throw new ArgumentException("Content type must be provided.", nameof(contentType));
|
||||
}
|
||||
|
||||
Name = name;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
ContentType = contentType;
|
||||
Immutable = immutable;
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Describes layer cache content to be stored.
|
||||
/// </summary>
|
||||
public sealed class LayerCachePutRequest
|
||||
{
|
||||
public string LayerDigest { get; }
|
||||
|
||||
public string Architecture { get; }
|
||||
|
||||
public string MediaType { get; }
|
||||
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; }
|
||||
|
||||
public IReadOnlyList<LayerCacheArtifactContent> Artifacts { get; }
|
||||
|
||||
public LayerCachePutRequest(
|
||||
string layerDigest,
|
||||
string architecture,
|
||||
string mediaType,
|
||||
IReadOnlyDictionary<string, string> metadata,
|
||||
IReadOnlyList<LayerCacheArtifactContent> artifacts)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(layerDigest))
|
||||
{
|
||||
throw new ArgumentException("Layer digest must be provided.", nameof(layerDigest));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(architecture))
|
||||
{
|
||||
throw new ArgumentException("Architecture must be provided.", nameof(architecture));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(mediaType))
|
||||
{
|
||||
throw new ArgumentException("Media type must be provided.", nameof(mediaType));
|
||||
}
|
||||
|
||||
Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
|
||||
Artifacts = artifacts ?? throw new ArgumentNullException(nameof(artifacts));
|
||||
if (artifacts.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one artifact must be supplied.", nameof(artifacts));
|
||||
}
|
||||
|
||||
LayerDigest = layerDigest;
|
||||
Architecture = architecture;
|
||||
MediaType = mediaType;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stream payload for a cached artifact.
|
||||
/// </summary>
|
||||
public sealed class LayerCacheArtifactContent
|
||||
{
|
||||
public string Name { get; }
|
||||
|
||||
public Stream Content { get; }
|
||||
|
||||
public string ContentType { get; }
|
||||
|
||||
public bool Immutable { get; }
|
||||
|
||||
public bool LeaveOpen { get; }
|
||||
|
||||
public LayerCacheArtifactContent(
|
||||
string name,
|
||||
Stream content,
|
||||
string contentType,
|
||||
bool immutable = false,
|
||||
bool leaveOpen = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
throw new ArgumentException("Artifact name must be provided.", nameof(name));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
throw new ArgumentException("Content type must be provided.", nameof(contentType));
|
||||
}
|
||||
|
||||
Name = name;
|
||||
Content = content ?? throw new ArgumentNullException(nameof(content));
|
||||
ContentType = contentType;
|
||||
Immutable = immutable;
|
||||
LeaveOpen = leaveOpen;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,481 +1,481 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
public sealed class FileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
private const string ContentFileName = "content.bin";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<FileContentAddressableStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public FileContentAddressableStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<FileContentAddressableStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entryDirectory = GetEntryDirectory(sha256);
|
||||
var metadataPath = Path.Combine(entryDirectory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasHit(sha256);
|
||||
|
||||
return new FileCasEntry(
|
||||
metadata.Sha256,
|
||||
metadata.SizeBytes,
|
||||
metadata.CreatedAt,
|
||||
metadata.LastAccessed,
|
||||
GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var sha = request.Sha256;
|
||||
var directory = GetEntryDirectory(sha);
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var contentPath = Path.Combine(directory, ContentFileName);
|
||||
await using (var destination = new FileStream(contentPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await request.Content.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!request.LeaveOpen)
|
||||
{
|
||||
request.Content.Dispose();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var sizeBytes = new FileInfo(contentPath).Length;
|
||||
var metadata = new FileCasMetadata
|
||||
{
|
||||
Sha256 = NormalizeHash(sha),
|
||||
CreatedAt = now,
|
||||
LastAccessed = now,
|
||||
SizeBytes = sizeBytes
|
||||
};
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasBytes(sizeBytes);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Stored CAS entry {Sha256} ({SizeBytes} bytes)", sha, sizeBytes);
|
||||
return new FileCasEntry(metadata.Sha256, metadata.SizeBytes, metadata.CreatedAt, metadata.LastAccessed, GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetEntryDirectory(sha256);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(metadata.Sha256);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} CAS entries due to TTL", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(destinationDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Directory.CreateDirectory(destinationDirectory);
|
||||
var exported = 0;
|
||||
|
||||
foreach (var entryDirectory in EnumerateEntryDirectories())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var hash = Path.GetFileName(entryDirectory);
|
||||
if (hash is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = Path.Combine(destinationDirectory, hash);
|
||||
if (Directory.Exists(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
CopyDirectory(entryDirectory, target);
|
||||
exported++;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Exported {Count} CAS entries to {Destination}", exported, destinationDirectory);
|
||||
return exported;
|
||||
}
|
||||
|
||||
public async Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sourceDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!Directory.Exists(sourceDirectory))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var imported = 0;
|
||||
foreach (var directory in Directory.EnumerateDirectories(sourceDirectory))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var destination = GetEntryDirectory(metadata.Sha256);
|
||||
if (Directory.Exists(destination))
|
||||
{
|
||||
// Only overwrite if the source is newer.
|
||||
var existingMetadataPath = Path.Combine(destination, MetadataFileName);
|
||||
var existing = await ReadMetadataAsync(existingMetadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && existing.CreatedAt >= metadata.CreatedAt)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(destination).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
CopyDirectory(directory, destination);
|
||||
imported++;
|
||||
}
|
||||
|
||||
if (imported > 0)
|
||||
{
|
||||
_logger.LogInformation("Imported {Count} CAS entries from {Source}", imported, sourceDirectory);
|
||||
}
|
||||
|
||||
return imported;
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(FileCasMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var target = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= target)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(entry.Directory).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordFileCasEviction(entry.Metadata.Sha256);
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
{
|
||||
_logger.LogInformation("Compacted CAS store, removed {Count} entries", removed);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.FileCasDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.FileCasDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateEntryDirectories()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var directory in Directory.EnumerateDirectories(_options.FileCasDirectoryPath, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (File.Exists(Path.Combine(directory, MetadataFileName)))
|
||||
{
|
||||
yield return directory;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FileCasMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<FileCasMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read CAS metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, FileCasMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private static string GetRelativeContentPath(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(NormalizedPrefix(normalized, 0, 2), NormalizedPrefix(normalized, 2, 2), normalized, ContentFileName);
|
||||
}
|
||||
|
||||
private string GetEntryDirectory(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(
|
||||
_options.FileCasDirectoryPath,
|
||||
NormalizedPrefix(normalized, 0, 2),
|
||||
NormalizedPrefix(normalized, 2, 2),
|
||||
normalized);
|
||||
}
|
||||
|
||||
private static string NormalizeHash(string sha256)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = sha256.Contains(':', StringComparison.Ordinal) ? sha256[(sha256.IndexOf(':') + 1)..] : sha256;
|
||||
return hash.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizedPrefix(string hash, int offset, int length)
|
||||
{
|
||||
if (hash.Length <= offset)
|
||||
{
|
||||
return "00";
|
||||
}
|
||||
|
||||
if (hash.Length < offset + length)
|
||||
{
|
||||
length = hash.Length - offset;
|
||||
}
|
||||
|
||||
return hash.Substring(offset, length);
|
||||
}
|
||||
|
||||
private bool IsExpired(FileCasMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CreatedAt >= _options.FileTtl;
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string sourceDir, string destDir)
|
||||
{
|
||||
Directory.CreateDirectory(destDir);
|
||||
foreach (var file in Directory.EnumerateFiles(sourceDir, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(sourceDir, file);
|
||||
var destination = Path.Combine(destDir, relative);
|
||||
var parent = Path.GetDirectoryName(destination);
|
||||
if (!string.IsNullOrEmpty(parent))
|
||||
{
|
||||
Directory.CreateDirectory(parent);
|
||||
}
|
||||
File.Copy(file, destination, overwrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
private Task RemoveDirectoryAsync(string directory)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete CAS directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FileCasMetadata
|
||||
{
|
||||
public string Sha256 { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
}
|
||||
}
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
public sealed class FileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
private const string ContentFileName = "content.bin";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<FileContentAddressableStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public FileContentAddressableStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<FileContentAddressableStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entryDirectory = GetEntryDirectory(sha256);
|
||||
var metadataPath = Path.Combine(entryDirectory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasMiss(sha256);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasHit(sha256);
|
||||
|
||||
return new FileCasEntry(
|
||||
metadata.Sha256,
|
||||
metadata.SizeBytes,
|
||||
metadata.CreatedAt,
|
||||
metadata.LastAccessed,
|
||||
GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var sha = request.Sha256;
|
||||
var directory = GetEntryDirectory(sha);
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var contentPath = Path.Combine(directory, ContentFileName);
|
||||
await using (var destination = new FileStream(contentPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await request.Content.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
await destination.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!request.LeaveOpen)
|
||||
{
|
||||
request.Content.Dispose();
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var sizeBytes = new FileInfo(contentPath).Length;
|
||||
var metadata = new FileCasMetadata
|
||||
{
|
||||
Sha256 = NormalizeHash(sha),
|
||||
CreatedAt = now,
|
||||
LastAccessed = now,
|
||||
SizeBytes = sizeBytes
|
||||
};
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasBytes(sizeBytes);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Stored CAS entry {Sha256} ({SizeBytes} bytes)", sha, sizeBytes);
|
||||
return new FileCasEntry(metadata.Sha256, metadata.SizeBytes, metadata.CreatedAt, metadata.LastAccessed, GetRelativeContentPath(metadata.Sha256));
|
||||
}
|
||||
|
||||
public async Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetEntryDirectory(sha256);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(sha256);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveDirectoryAsync(directory).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordFileCasEviction(metadata.Sha256);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} CAS entries due to TTL", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(destinationDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Directory.CreateDirectory(destinationDirectory);
|
||||
var exported = 0;
|
||||
|
||||
foreach (var entryDirectory in EnumerateEntryDirectories())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var hash = Path.GetFileName(entryDirectory);
|
||||
if (hash is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var target = Path.Combine(destinationDirectory, hash);
|
||||
if (Directory.Exists(target))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
CopyDirectory(entryDirectory, target);
|
||||
exported++;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Exported {Count} CAS entries to {Destination}", exported, destinationDirectory);
|
||||
return exported;
|
||||
}
|
||||
|
||||
public async Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sourceDirectory);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!Directory.Exists(sourceDirectory))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var imported = 0;
|
||||
foreach (var directory in Directory.EnumerateDirectories(sourceDirectory))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var destination = GetEntryDirectory(metadata.Sha256);
|
||||
if (Directory.Exists(destination))
|
||||
{
|
||||
// Only overwrite if the source is newer.
|
||||
var existingMetadataPath = Path.Combine(destination, MetadataFileName);
|
||||
var existing = await ReadMetadataAsync(existingMetadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null && existing.CreatedAt >= metadata.CreatedAt)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(destination).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
CopyDirectory(directory, destination);
|
||||
imported++;
|
||||
}
|
||||
|
||||
if (imported > 0)
|
||||
{
|
||||
_logger.LogInformation("Imported {Count} CAS entries from {Source}", imported, sourceDirectory);
|
||||
}
|
||||
|
||||
return imported;
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(FileCasMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var target = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= target)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveDirectoryAsync(entry.Directory).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordFileCasEviction(entry.Metadata.Sha256);
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
{
|
||||
_logger.LogInformation("Compacted CAS store, removed {Count} entries", removed);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.FileCasDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.FileCasDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateEntryDirectories()
|
||||
{
|
||||
if (!Directory.Exists(_options.FileCasDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var directory in Directory.EnumerateDirectories(_options.FileCasDirectoryPath, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (File.Exists(Path.Combine(directory, MetadataFileName)))
|
||||
{
|
||||
yield return directory;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FileCasMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<FileCasMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read CAS metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, FileCasMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private static string GetRelativeContentPath(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(NormalizedPrefix(normalized, 0, 2), NormalizedPrefix(normalized, 2, 2), normalized, ContentFileName);
|
||||
}
|
||||
|
||||
private string GetEntryDirectory(string sha256)
|
||||
{
|
||||
var normalized = NormalizeHash(sha256);
|
||||
return Path.Combine(
|
||||
_options.FileCasDirectoryPath,
|
||||
NormalizedPrefix(normalized, 0, 2),
|
||||
NormalizedPrefix(normalized, 2, 2),
|
||||
normalized);
|
||||
}
|
||||
|
||||
private static string NormalizeHash(string sha256)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sha256))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = sha256.Contains(':', StringComparison.Ordinal) ? sha256[(sha256.IndexOf(':') + 1)..] : sha256;
|
||||
return hash.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizedPrefix(string hash, int offset, int length)
|
||||
{
|
||||
if (hash.Length <= offset)
|
||||
{
|
||||
return "00";
|
||||
}
|
||||
|
||||
if (hash.Length < offset + length)
|
||||
{
|
||||
length = hash.Length - offset;
|
||||
}
|
||||
|
||||
return hash.Substring(offset, length);
|
||||
}
|
||||
|
||||
private bool IsExpired(FileCasMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.FileTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CreatedAt >= _options.FileTtl;
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string sourceDir, string destDir)
|
||||
{
|
||||
Directory.CreateDirectory(destDir);
|
||||
foreach (var file in Directory.EnumerateFiles(sourceDir, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(sourceDir, file);
|
||||
var destination = Path.Combine(destDir, relative);
|
||||
var parent = Path.GetDirectoryName(destination);
|
||||
if (!string.IsNullOrEmpty(parent))
|
||||
{
|
||||
Directory.CreateDirectory(parent);
|
||||
}
|
||||
File.Copy(file, destination, overwrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
private Task RemoveDirectoryAsync(string directory)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete CAS directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FileCasMetadata
|
||||
{
|
||||
public string Sha256 { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,27 @@
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
internal sealed class NullFileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
public ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<FileCasEntry?>(null);
|
||||
|
||||
public Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
=> Task.FromException<FileCasEntry>(new InvalidOperationException("File CAS is disabled via configuration."));
|
||||
|
||||
public Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(false);
|
||||
|
||||
public Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
}
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.FileCas;
|
||||
|
||||
internal sealed class NullFileContentAddressableStore : IFileContentAddressableStore
|
||||
{
|
||||
public ValueTask<FileCasEntry?> TryGetAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<FileCasEntry?>(null);
|
||||
|
||||
public Task<FileCasEntry> PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default)
|
||||
=> Task.FromException<FileCasEntry>(new InvalidOperationException("File CAS is disabled via configuration."));
|
||||
|
||||
public Task<bool> RemoveAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(false);
|
||||
|
||||
public Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
}
|
||||
|
||||
@@ -1,480 +1,480 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.LayerCache;
|
||||
|
||||
public sealed class LayerCacheStore : ILayerCacheStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<LayerCacheStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public LayerCacheStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<LayerCacheStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogTrace("Layer cache miss: directory {Directory} not found for {LayerDigest}", directory, layerDigest);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache metadata missing at {Path} for {LayerDigest}; removing directory", metadataPath, layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
_logger.LogDebug("Layer cache entry {LayerDigest} expired at {Expiration}", metadata.LayerDigest, metadata.CachedAt + _options.LayerTtl);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerHit(layerDigest);
|
||||
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = request.LayerDigest;
|
||||
var directory = GetLayerDirectory(digest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogDebug("Replacing existing layer cache entry for {LayerDigest}", digest);
|
||||
await RemoveInternalAsync(directory, digest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var artifactMetadata = new Dictionary<string, LayerCacheArtifactMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
long totalSize = 0;
|
||||
|
||||
foreach (var artifact in request.Artifacts)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var fileName = SanitizeArtifactName(artifact.Name);
|
||||
var relativePath = Path.Combine("artifacts", fileName);
|
||||
var artifactDirectory = Path.Combine(directory, "artifacts");
|
||||
Directory.CreateDirectory(artifactDirectory);
|
||||
var filePath = Path.Combine(directory, relativePath);
|
||||
|
||||
await using (var target = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await artifact.Content.CopyToAsync(target, cancellationToken).ConfigureAwait(false);
|
||||
await target.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
totalSize += target.Length;
|
||||
}
|
||||
|
||||
if (!artifact.LeaveOpen)
|
||||
{
|
||||
artifact.Content.Dispose();
|
||||
}
|
||||
|
||||
var sizeBytes = new FileInfo(filePath).Length;
|
||||
|
||||
artifactMetadata[artifact.Name] = new LayerCacheArtifactMetadata
|
||||
{
|
||||
Name = artifact.Name,
|
||||
ContentType = artifact.ContentType,
|
||||
RelativePath = relativePath,
|
||||
SizeBytes = sizeBytes,
|
||||
Immutable = artifact.Immutable
|
||||
};
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var metadata = new LayerCacheMetadata
|
||||
{
|
||||
LayerDigest = digest,
|
||||
Architecture = request.Architecture,
|
||||
MediaType = request.MediaType,
|
||||
CachedAt = now,
|
||||
LastAccessed = now,
|
||||
Metadata = new Dictionary<string, string>(request.Metadata, StringComparer.Ordinal),
|
||||
Artifacts = artifactMetadata,
|
||||
SizeBytes = totalSize
|
||||
};
|
||||
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerBytes(totalSize);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Cached layer {LayerDigest} with {ArtifactCount} artifacts ({SizeBytes} bytes)", digest, artifactMetadata.Count, totalSize);
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveInternalAsync(directory, metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(metadata.LayerDigest);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} expired layer cache entries", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!metadata.Artifacts.TryGetValue(artifactName, out var artifact))
|
||||
{
|
||||
_logger.LogDebug("Layer cache artifact {Artifact} missing for {LayerDigest}", artifactName, layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var filePath = Path.Combine(directory, artifact.RelativePath);
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache file {FilePath} not found for artifact {Artifact}", filePath, artifactName);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = _timeProvider.GetUtcNow();
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(LayerCacheMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var targetBytes = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= targetBytes)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveInternalAsync(entry.Directory, entry.Metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordLayerEviction(entry.Metadata.LayerDigest);
|
||||
_logger.LogInformation("Evicted layer {LayerDigest} during compaction (remaining bytes: {Bytes})", entry.Metadata.LayerDigest, totalBytes);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.LayersDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.LayersDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.LayersDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<LayerCacheMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<LayerCacheMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load layer cache metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, LayerCacheMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private Task RemoveInternalAsync(string directory, string layerDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
_logger.LogDebug("Removed layer cache entry {LayerDigest}", layerDigest);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete layer cache directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private bool IsExpired(LayerCacheMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (metadata.CachedAt == default)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CachedAt >= _options.LayerTtl;
|
||||
}
|
||||
|
||||
private LayerCacheEntry Map(LayerCacheMetadata metadata)
|
||||
{
|
||||
var artifacts = metadata.Artifacts?.ToDictionary(
|
||||
pair => pair.Key,
|
||||
pair => new LayerCacheArtifactReference(
|
||||
pair.Value.Name,
|
||||
pair.Value.RelativePath,
|
||||
pair.Value.ContentType,
|
||||
pair.Value.SizeBytes,
|
||||
pair.Value.Immutable),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, LayerCacheArtifactReference>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
return new LayerCacheEntry(
|
||||
metadata.LayerDigest,
|
||||
metadata.Architecture,
|
||||
metadata.MediaType,
|
||||
metadata.CachedAt,
|
||||
metadata.LastAccessed,
|
||||
metadata.SizeBytes,
|
||||
artifacts,
|
||||
metadata.Metadata is null
|
||||
? new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, string>(metadata.Metadata, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
private string GetLayerDirectory(string layerDigest)
|
||||
{
|
||||
var safeDigest = SanitizeDigest(layerDigest);
|
||||
return Path.Combine(_options.LayersDirectoryPath, safeDigest);
|
||||
}
|
||||
|
||||
private static string SanitizeArtifactName(string name)
|
||||
{
|
||||
var fileName = Path.GetFileName(name);
|
||||
return string.IsNullOrWhiteSpace(fileName) ? "artifact" : fileName;
|
||||
}
|
||||
|
||||
private static string SanitizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = digest.Contains(':', StringComparison.Ordinal)
|
||||
? digest[(digest.IndexOf(':') + 1)..]
|
||||
: digest;
|
||||
|
||||
var buffer = new char[hash.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in hash)
|
||||
{
|
||||
buffer[count++] = char.IsLetterOrDigit(ch) ? char.ToLowerInvariant(ch) : '_';
|
||||
}
|
||||
|
||||
return new string(buffer, 0, count);
|
||||
}
|
||||
|
||||
private sealed class LayerCacheMetadata
|
||||
{
|
||||
public string LayerDigest { get; set; } = string.Empty;
|
||||
|
||||
public string Architecture { get; set; } = string.Empty;
|
||||
|
||||
public string MediaType { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CachedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
|
||||
public Dictionary<string, LayerCacheArtifactMetadata> Artifacts { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
}
|
||||
|
||||
private sealed class LayerCacheArtifactMetadata
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
public bool Immutable { get; set; }
|
||||
}
|
||||
}
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.LayerCache;
|
||||
|
||||
public sealed class LayerCacheStore : ILayerCacheStore
|
||||
{
|
||||
private const string MetadataFileName = "meta.json";
|
||||
|
||||
private readonly ScannerCacheOptions _options;
|
||||
private readonly ILogger<LayerCacheStore> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
private readonly SemaphoreSlim _initializationLock = new(1, 1);
|
||||
private volatile bool _initialised;
|
||||
|
||||
public LayerCacheStore(
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<LayerCacheStore> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<LayerCacheEntry?> TryGetAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogTrace("Layer cache miss: directory {Directory} not found for {LayerDigest}", directory, layerDigest);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache metadata missing at {Path} for {LayerDigest}; removing directory", metadataPath, layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
_logger.LogDebug("Layer cache entry {LayerDigest} expired at {Expiration}", metadata.LayerDigest, metadata.CachedAt + _options.LayerTtl);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = now;
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerHit(layerDigest);
|
||||
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task<LayerCacheEntry> PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var digest = request.LayerDigest;
|
||||
var directory = GetLayerDirectory(digest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
|
||||
if (Directory.Exists(directory))
|
||||
{
|
||||
_logger.LogDebug("Replacing existing layer cache entry for {LayerDigest}", digest);
|
||||
await RemoveInternalAsync(directory, digest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(directory);
|
||||
|
||||
var artifactMetadata = new Dictionary<string, LayerCacheArtifactMetadata>(StringComparer.OrdinalIgnoreCase);
|
||||
long totalSize = 0;
|
||||
|
||||
foreach (var artifact in request.Artifacts)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var fileName = SanitizeArtifactName(artifact.Name);
|
||||
var relativePath = Path.Combine("artifacts", fileName);
|
||||
var artifactDirectory = Path.Combine(directory, "artifacts");
|
||||
Directory.CreateDirectory(artifactDirectory);
|
||||
var filePath = Path.Combine(directory, relativePath);
|
||||
|
||||
await using (var target = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous))
|
||||
{
|
||||
await artifact.Content.CopyToAsync(target, cancellationToken).ConfigureAwait(false);
|
||||
await target.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
totalSize += target.Length;
|
||||
}
|
||||
|
||||
if (!artifact.LeaveOpen)
|
||||
{
|
||||
artifact.Content.Dispose();
|
||||
}
|
||||
|
||||
var sizeBytes = new FileInfo(filePath).Length;
|
||||
|
||||
artifactMetadata[artifact.Name] = new LayerCacheArtifactMetadata
|
||||
{
|
||||
Name = artifact.Name,
|
||||
ContentType = artifact.ContentType,
|
||||
RelativePath = relativePath,
|
||||
SizeBytes = sizeBytes,
|
||||
Immutable = artifact.Immutable
|
||||
};
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var metadata = new LayerCacheMetadata
|
||||
{
|
||||
LayerDigest = digest,
|
||||
Architecture = request.Architecture,
|
||||
MediaType = request.MediaType,
|
||||
CachedAt = now,
|
||||
LastAccessed = now,
|
||||
Metadata = new Dictionary<string, string>(request.Metadata, StringComparer.Ordinal),
|
||||
Artifacts = artifactMetadata,
|
||||
SizeBytes = totalSize
|
||||
};
|
||||
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerBytes(totalSize);
|
||||
|
||||
await CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("Cached layer {LayerDigest} with {ArtifactCount} artifacts ({SizeBytes} bytes)", digest, artifactMetadata.Count, totalSize);
|
||||
return Map(metadata);
|
||||
}
|
||||
|
||||
public async Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<int> EvictExpiredAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var evicted = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsExpired(metadata, now))
|
||||
{
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
await RemoveInternalAsync(directory, metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerEviction(metadata.LayerDigest);
|
||||
evicted++;
|
||||
}
|
||||
}
|
||||
|
||||
if (evicted > 0)
|
||||
{
|
||||
_logger.LogInformation("Evicted {Count} expired layer cache entries", evicted);
|
||||
}
|
||||
|
||||
return evicted;
|
||||
}
|
||||
|
||||
public async Task<Stream?> OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(artifactName);
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var directory = GetLayerDirectory(layerDigest);
|
||||
var metadataPath = Path.Combine(directory, MetadataFileName);
|
||||
if (!File.Exists(metadataPath))
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!metadata.Artifacts.TryGetValue(artifactName, out var artifact))
|
||||
{
|
||||
_logger.LogDebug("Layer cache artifact {Artifact} missing for {LayerDigest}", artifactName, layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var filePath = Path.Combine(directory, artifact.RelativePath);
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Layer cache file {FilePath} not found for artifact {Artifact}", filePath, artifactName);
|
||||
await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false);
|
||||
ScannerCacheMetrics.RecordLayerMiss(layerDigest);
|
||||
return null;
|
||||
}
|
||||
|
||||
metadata.LastAccessed = _timeProvider.GetUtcNow();
|
||||
await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false);
|
||||
return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
|
||||
public async Task<int> CompactAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_options.MaxBytes <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var entries = new List<(LayerCacheMetadata Metadata, string Directory)>();
|
||||
long totalBytes = 0;
|
||||
|
||||
foreach (var metadataPath in EnumerateMetadataFiles())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false);
|
||||
if (metadata is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var directory = Path.GetDirectoryName(metadataPath)!;
|
||||
entries.Add((metadata, directory));
|
||||
totalBytes += metadata.SizeBytes;
|
||||
}
|
||||
|
||||
if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed));
|
||||
var targetBytes = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (totalBytes <= targetBytes)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await RemoveInternalAsync(entry.Directory, entry.Metadata.LayerDigest, cancellationToken).ConfigureAwait(false);
|
||||
totalBytes -= entry.Metadata.SizeBytes;
|
||||
removed++;
|
||||
ScannerCacheMetrics.RecordLayerEviction(entry.Metadata.LayerDigest);
|
||||
_logger.LogInformation("Evicted layer {LayerDigest} during compaction (remaining bytes: {Bytes})", entry.Metadata.LayerDigest, totalBytes);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private async Task EnsureInitialisedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_initialised)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(_options.LayersDirectoryPath);
|
||||
_initialised = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initializationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<string> EnumerateMetadataFiles()
|
||||
{
|
||||
if (!Directory.Exists(_options.LayersDirectoryPath))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_options.LayersDirectoryPath, MetadataFileName, SearchOption.AllDirectories))
|
||||
{
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<LayerCacheMetadata?> ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await JsonSerializer.DeserializeAsync<LayerCacheMetadata>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or JsonException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load layer cache metadata from {Path}", metadataPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(string metadataPath, LayerCacheMetadata metadata, CancellationToken cancellationToken)
|
||||
{
|
||||
var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp");
|
||||
await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
File.Move(tempFile, metadataPath, overwrite: true);
|
||||
}
|
||||
|
||||
private Task RemoveInternalAsync(string directory, string layerDigest, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, recursive: true);
|
||||
_logger.LogDebug("Removed layer cache entry {LayerDigest}", layerDigest);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete layer cache directory {Directory}", directory);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private bool IsExpired(LayerCacheMetadata metadata, DateTimeOffset now)
|
||||
{
|
||||
if (_options.LayerTtl <= TimeSpan.Zero)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (metadata.CachedAt == default)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return now - metadata.CachedAt >= _options.LayerTtl;
|
||||
}
|
||||
|
||||
private LayerCacheEntry Map(LayerCacheMetadata metadata)
|
||||
{
|
||||
var artifacts = metadata.Artifacts?.ToDictionary(
|
||||
pair => pair.Key,
|
||||
pair => new LayerCacheArtifactReference(
|
||||
pair.Value.Name,
|
||||
pair.Value.RelativePath,
|
||||
pair.Value.ContentType,
|
||||
pair.Value.SizeBytes,
|
||||
pair.Value.Immutable),
|
||||
StringComparer.OrdinalIgnoreCase)
|
||||
?? new Dictionary<string, LayerCacheArtifactReference>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
return new LayerCacheEntry(
|
||||
metadata.LayerDigest,
|
||||
metadata.Architecture,
|
||||
metadata.MediaType,
|
||||
metadata.CachedAt,
|
||||
metadata.LastAccessed,
|
||||
metadata.SizeBytes,
|
||||
artifacts,
|
||||
metadata.Metadata is null
|
||||
? new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, string>(metadata.Metadata, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
private string GetLayerDirectory(string layerDigest)
|
||||
{
|
||||
var safeDigest = SanitizeDigest(layerDigest);
|
||||
return Path.Combine(_options.LayersDirectoryPath, safeDigest);
|
||||
}
|
||||
|
||||
private static string SanitizeArtifactName(string name)
|
||||
{
|
||||
var fileName = Path.GetFileName(name);
|
||||
return string.IsNullOrWhiteSpace(fileName) ? "artifact" : fileName;
|
||||
}
|
||||
|
||||
private static string SanitizeDigest(string digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var hash = digest.Contains(':', StringComparison.Ordinal)
|
||||
? digest[(digest.IndexOf(':') + 1)..]
|
||||
: digest;
|
||||
|
||||
var buffer = new char[hash.Length];
|
||||
var count = 0;
|
||||
foreach (var ch in hash)
|
||||
{
|
||||
buffer[count++] = char.IsLetterOrDigit(ch) ? char.ToLowerInvariant(ch) : '_';
|
||||
}
|
||||
|
||||
return new string(buffer, 0, count);
|
||||
}
|
||||
|
||||
private sealed class LayerCacheMetadata
|
||||
{
|
||||
public string LayerDigest { get; set; } = string.Empty;
|
||||
|
||||
public string Architecture { get; set; } = string.Empty;
|
||||
|
||||
public string MediaType { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CachedAt { get; set; }
|
||||
|
||||
public DateTimeOffset LastAccessed { get; set; }
|
||||
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
|
||||
public Dictionary<string, LayerCacheArtifactMetadata> Artifacts { get; set; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
}
|
||||
|
||||
private sealed class LayerCacheArtifactMetadata
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
public long SizeBytes { get; set; }
|
||||
|
||||
public bool Immutable { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,85 +1,85 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
public sealed class ScannerCacheMaintenanceService : BackgroundService
|
||||
{
|
||||
private readonly ILayerCacheStore _layerCache;
|
||||
private readonly IFileContentAddressableStore _fileCas;
|
||||
private readonly IOptions<ScannerCacheOptions> _options;
|
||||
private readonly ILogger<ScannerCacheMaintenanceService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ScannerCacheMaintenanceService(
|
||||
ILayerCacheStore layerCache,
|
||||
IFileContentAddressableStore fileCas,
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<ScannerCacheMaintenanceService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_layerCache = layerCache ?? throw new ArgumentNullException(nameof(layerCache));
|
||||
_fileCas = fileCas ?? throw new ArgumentNullException(nameof(fileCas));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var settings = _options.Value;
|
||||
if (!settings.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache disabled; maintenance loop will not start.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings.EnableAutoEviction)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache automatic eviction disabled by configuration.");
|
||||
return;
|
||||
}
|
||||
|
||||
var interval = settings.MaintenanceInterval > TimeSpan.Zero
|
||||
? settings.MaintenanceInterval
|
||||
: TimeSpan.FromMinutes(15);
|
||||
|
||||
_logger.LogInformation("Scanner cache maintenance loop started with interval {Interval}", interval);
|
||||
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
using var timer = new PeriodicTimer(interval, _timeProvider);
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunMaintenanceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var layerExpired = await _layerCache.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var layerCompacted = await _layerCache.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casExpired = await _fileCas.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casCompacted = await _fileCas.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Scanner cache maintenance tick complete (layers expired={LayersExpired}, layers compacted={LayersCompacted}, cas expired={CasExpired}, cas compacted={CasCompacted})",
|
||||
layerExpired,
|
||||
layerCompacted,
|
||||
casExpired,
|
||||
casCompacted);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutting down; ignore.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Scanner cache maintenance tick failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
|
||||
namespace StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
public sealed class ScannerCacheMaintenanceService : BackgroundService
|
||||
{
|
||||
private readonly ILayerCacheStore _layerCache;
|
||||
private readonly IFileContentAddressableStore _fileCas;
|
||||
private readonly IOptions<ScannerCacheOptions> _options;
|
||||
private readonly ILogger<ScannerCacheMaintenanceService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ScannerCacheMaintenanceService(
|
||||
ILayerCacheStore layerCache,
|
||||
IFileContentAddressableStore fileCas,
|
||||
IOptions<ScannerCacheOptions> options,
|
||||
ILogger<ScannerCacheMaintenanceService> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_layerCache = layerCache ?? throw new ArgumentNullException(nameof(layerCache));
|
||||
_fileCas = fileCas ?? throw new ArgumentNullException(nameof(fileCas));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var settings = _options.Value;
|
||||
if (!settings.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache disabled; maintenance loop will not start.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings.EnableAutoEviction)
|
||||
{
|
||||
_logger.LogInformation("Scanner cache automatic eviction disabled by configuration.");
|
||||
return;
|
||||
}
|
||||
|
||||
var interval = settings.MaintenanceInterval > TimeSpan.Zero
|
||||
? settings.MaintenanceInterval
|
||||
: TimeSpan.FromMinutes(15);
|
||||
|
||||
_logger.LogInformation("Scanner cache maintenance loop started with interval {Interval}", interval);
|
||||
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
|
||||
using var timer = new PeriodicTimer(interval, _timeProvider);
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunMaintenanceAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var layerExpired = await _layerCache.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var layerCompacted = await _layerCache.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casExpired = await _fileCas.EvictExpiredAsync(cancellationToken).ConfigureAwait(false);
|
||||
var casCompacted = await _fileCas.CompactAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Scanner cache maintenance tick complete (layers expired={LayersExpired}, layers compacted={LayersCompacted}, cas expired={CasExpired}, cas compacted={CasCompacted})",
|
||||
layerExpired,
|
||||
layerCompacted,
|
||||
casExpired,
|
||||
casCompacted);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutting down; ignore.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Scanner cache maintenance tick failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,43 +1,43 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheMetrics
|
||||
{
|
||||
public const string MeterName = "StellaOps.Scanner.Cache";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName, "1.0.0");
|
||||
|
||||
private static readonly Counter<long> LayerHits = Meter.CreateCounter<long>("scanner.layer_cache_hits_total");
|
||||
private static readonly Counter<long> LayerMisses = Meter.CreateCounter<long>("scanner.layer_cache_misses_total");
|
||||
private static readonly Counter<long> LayerEvictions = Meter.CreateCounter<long>("scanner.layer_cache_evictions_total");
|
||||
private static readonly Histogram<long> LayerBytes = Meter.CreateHistogram<long>("scanner.layer_cache_bytes");
|
||||
private static readonly Counter<long> FileCasHits = Meter.CreateCounter<long>("scanner.file_cas_hits_total");
|
||||
private static readonly Counter<long> FileCasMisses = Meter.CreateCounter<long>("scanner.file_cas_misses_total");
|
||||
private static readonly Counter<long> FileCasEvictions = Meter.CreateCounter<long>("scanner.file_cas_evictions_total");
|
||||
private static readonly Histogram<long> FileCasBytes = Meter.CreateHistogram<long>("scanner.file_cas_bytes");
|
||||
|
||||
public static void RecordLayerHit(string layerDigest)
|
||||
=> LayerHits.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerMiss(string layerDigest)
|
||||
=> LayerMisses.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerEviction(string layerDigest)
|
||||
=> LayerEvictions.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerBytes(long bytes)
|
||||
=> LayerBytes.Record(bytes);
|
||||
|
||||
public static void RecordFileCasHit(string sha256)
|
||||
=> FileCasHits.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasMiss(string sha256)
|
||||
=> FileCasMisses.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasEviction(string sha256)
|
||||
=> FileCasEvictions.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasBytes(long bytes)
|
||||
=> FileCasBytes.Record(bytes);
|
||||
}
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheMetrics
|
||||
{
|
||||
public const string MeterName = "StellaOps.Scanner.Cache";
|
||||
|
||||
private static readonly Meter Meter = new(MeterName, "1.0.0");
|
||||
|
||||
private static readonly Counter<long> LayerHits = Meter.CreateCounter<long>("scanner.layer_cache_hits_total");
|
||||
private static readonly Counter<long> LayerMisses = Meter.CreateCounter<long>("scanner.layer_cache_misses_total");
|
||||
private static readonly Counter<long> LayerEvictions = Meter.CreateCounter<long>("scanner.layer_cache_evictions_total");
|
||||
private static readonly Histogram<long> LayerBytes = Meter.CreateHistogram<long>("scanner.layer_cache_bytes");
|
||||
private static readonly Counter<long> FileCasHits = Meter.CreateCounter<long>("scanner.file_cas_hits_total");
|
||||
private static readonly Counter<long> FileCasMisses = Meter.CreateCounter<long>("scanner.file_cas_misses_total");
|
||||
private static readonly Counter<long> FileCasEvictions = Meter.CreateCounter<long>("scanner.file_cas_evictions_total");
|
||||
private static readonly Histogram<long> FileCasBytes = Meter.CreateHistogram<long>("scanner.file_cas_bytes");
|
||||
|
||||
public static void RecordLayerHit(string layerDigest)
|
||||
=> LayerHits.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerMiss(string layerDigest)
|
||||
=> LayerMisses.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerEviction(string layerDigest)
|
||||
=> LayerEvictions.Add(1, new KeyValuePair<string, object?>("layer", layerDigest));
|
||||
|
||||
public static void RecordLayerBytes(long bytes)
|
||||
=> LayerBytes.Record(bytes);
|
||||
|
||||
public static void RecordFileCasHit(string sha256)
|
||||
=> FileCasHits.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasMiss(string sha256)
|
||||
=> FileCasMisses.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasEviction(string sha256)
|
||||
=> FileCasEvictions.Add(1, new KeyValuePair<string, object?>("sha256", sha256));
|
||||
|
||||
public static void RecordFileCasBytes(long bytes)
|
||||
=> FileCasBytes.Record(bytes);
|
||||
}
|
||||
|
||||
@@ -1,40 +1,40 @@
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public sealed class ScannerCacheOptions
|
||||
{
|
||||
private const long DefaultMaxBytes = 5L * 1024 * 1024 * 1024; // 5 GiB
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string RootPath { get; set; } = Path.Combine("cache", "scanner");
|
||||
|
||||
public string LayersDirectoryName { get; set; } = "layers";
|
||||
|
||||
public string FileCasDirectoryName { get; set; } = "cas";
|
||||
|
||||
public TimeSpan LayerTtl { get; set; } = TimeSpan.FromDays(45);
|
||||
|
||||
public TimeSpan FileTtl { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public long MaxBytes { get; set; } = DefaultMaxBytes;
|
||||
|
||||
public long WarmBytesThreshold { get; set; } = DefaultMaxBytes / 5; // 20 %
|
||||
|
||||
public long ColdBytesThreshold { get; set; } = (DefaultMaxBytes * 4) / 5; // 80 %
|
||||
|
||||
public bool EnableAutoEviction { get; set; } = true;
|
||||
|
||||
public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
public bool EnableFileCas { get; set; } = true;
|
||||
|
||||
public string? ImportDirectory { get; set; }
|
||||
|
||||
public string? ExportDirectory { get; set; }
|
||||
|
||||
public string LayersDirectoryPath => Path.Combine(RootPath, LayersDirectoryName);
|
||||
|
||||
public string FileCasDirectoryPath => Path.Combine(RootPath, FileCasDirectoryName);
|
||||
}
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public sealed class ScannerCacheOptions
|
||||
{
|
||||
private const long DefaultMaxBytes = 5L * 1024 * 1024 * 1024; // 5 GiB
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
public string RootPath { get; set; } = Path.Combine("cache", "scanner");
|
||||
|
||||
public string LayersDirectoryName { get; set; } = "layers";
|
||||
|
||||
public string FileCasDirectoryName { get; set; } = "cas";
|
||||
|
||||
public TimeSpan LayerTtl { get; set; } = TimeSpan.FromDays(45);
|
||||
|
||||
public TimeSpan FileTtl { get; set; } = TimeSpan.FromDays(30);
|
||||
|
||||
public long MaxBytes { get; set; } = DefaultMaxBytes;
|
||||
|
||||
public long WarmBytesThreshold { get; set; } = DefaultMaxBytes / 5; // 20 %
|
||||
|
||||
public long ColdBytesThreshold { get; set; } = (DefaultMaxBytes * 4) / 5; // 80 %
|
||||
|
||||
public bool EnableAutoEviction { get; set; } = true;
|
||||
|
||||
public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
public bool EnableFileCas { get; set; } = true;
|
||||
|
||||
public string? ImportDirectory { get; set; }
|
||||
|
||||
public string? ExportDirectory { get; set; }
|
||||
|
||||
public string LayersDirectoryPath => Path.Combine(RootPath, LayersDirectoryName);
|
||||
|
||||
public string FileCasDirectoryPath => Path.Combine(RootPath, FileCasDirectoryName);
|
||||
}
|
||||
|
||||
@@ -1,51 +1,51 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
using StellaOps.Scanner.Cache.FileCas;
|
||||
using StellaOps.Scanner.Cache.LayerCache;
|
||||
using StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddScannerCache(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "scanner:cache")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<ScannerCacheOptions>()
|
||||
.Bind(configuration.GetSection(sectionName))
|
||||
.Validate(options => !string.IsNullOrWhiteSpace(options.RootPath), "scanner:cache:rootPath must be configured");
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
|
||||
services.TryAddSingleton<ILayerCacheStore, LayerCacheStore>();
|
||||
services.TryAddSingleton<IFileContentAddressableStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerCacheOptions>>();
|
||||
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
|
||||
if (!options.Value.EnableFileCas)
|
||||
{
|
||||
return new NullFileContentAddressableStore();
|
||||
}
|
||||
|
||||
return new FileContentAddressableStore(
|
||||
options,
|
||||
loggerFactory.CreateLogger<FileContentAddressableStore>(),
|
||||
timeProvider);
|
||||
});
|
||||
|
||||
services.AddHostedService<ScannerCacheMaintenanceService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Cache.Abstractions;
|
||||
using StellaOps.Scanner.Cache.FileCas;
|
||||
using StellaOps.Scanner.Cache.LayerCache;
|
||||
using StellaOps.Scanner.Cache.Maintenance;
|
||||
|
||||
namespace StellaOps.Scanner.Cache;
|
||||
|
||||
public static class ScannerCacheServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddScannerCache(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "scanner:cache")
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddOptions<ScannerCacheOptions>()
|
||||
.Bind(configuration.GetSection(sectionName))
|
||||
.Validate(options => !string.IsNullOrWhiteSpace(options.RootPath), "scanner:cache:rootPath must be configured");
|
||||
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
|
||||
services.TryAddSingleton<ILayerCacheStore, LayerCacheStore>();
|
||||
services.TryAddSingleton<IFileContentAddressableStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerCacheOptions>>();
|
||||
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
|
||||
if (!options.Value.EnableFileCas)
|
||||
{
|
||||
return new NullFileContentAddressableStore();
|
||||
}
|
||||
|
||||
return new FileContentAddressableStore(
|
||||
options,
|
||||
loggerFactory.CreateLogger<FileContentAddressableStore>(),
|
||||
timeProvider);
|
||||
});
|
||||
|
||||
services.AddHostedService<ScannerCacheMaintenanceService>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user