Refactor code structure for improved readability and maintainability; optimize performance in key functions.
This commit is contained in:
master
@@ -0,0 +1,474 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Reachability.Slices;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Oci;
|
||||
|
||||
/// <summary>
|
||||
/// Options for slice pulling operations.
|
||||
/// </summary>
|
||||
public sealed record SlicePullOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether to verify DSSE signature on retrieval. Default: true.
|
||||
/// </summary>
|
||||
public bool VerifySignature { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to cache pulled slices. Default: true.
|
||||
/// </summary>
|
||||
public bool EnableCache { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Cache TTL. Default: 1 hour.
|
||||
/// </summary>
|
||||
public TimeSpan CacheTtl { get; init; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>
|
||||
/// Request timeout. Default: 30 seconds.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; init; } = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a slice pull operation.
|
||||
/// </summary>
|
||||
public sealed record SlicePullResult
|
||||
{
|
||||
public required bool Success { get; init; }
|
||||
public ReachabilitySlice? Slice { get; init; }
|
||||
public string? SliceDigest { get; init; }
|
||||
public byte[]? DsseEnvelope { get; init; }
|
||||
public string? Error { get; init; }
|
||||
public bool FromCache { get; init; }
|
||||
public bool SignatureVerified { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Service for pulling reachability slices from OCI registries.
|
||||
/// Supports content-addressed retrieval and DSSE signature verification.
|
||||
/// Sprint: SPRINT_3850_0001_0001
|
||||
/// </summary>
|
||||
public sealed class SlicePullService : IDisposable
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly OciRegistryAuthorization _authorization;
|
||||
private readonly SlicePullOptions _options;
|
||||
private readonly ILogger<SlicePullService> _logger;
|
||||
private readonly Dictionary<string, CachedSlice> _cache = new(StringComparer.Ordinal);
|
||||
private readonly Lock _cacheLock = new();
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public SlicePullService(
|
||||
HttpClient httpClient,
|
||||
OciRegistryAuthorization authorization,
|
||||
SlicePullOptions? options = null,
|
||||
ILogger<SlicePullService>? logger = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
_authorization = authorization ?? throw new ArgumentNullException(nameof(authorization));
|
||||
_options = options ?? new SlicePullOptions();
|
||||
_logger = logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger<SlicePullService>.Instance;
|
||||
_httpClient.Timeout = _options.RequestTimeout;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pull a slice by its content-addressed digest.
|
||||
/// </summary>
|
||||
public async Task<SlicePullResult> PullByDigestAsync(
|
||||
OciImageReference reference,
|
||||
string digest,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reference);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
|
||||
var cacheKey = $"{reference.Registry}/{reference.Repository}@{digest}";
|
||||
|
||||
// Check cache
|
||||
if (_options.EnableCache && TryGetFromCache(cacheKey, out var cached))
|
||||
{
|
||||
_logger.LogDebug("Cache hit for slice {Digest}", digest);
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = true,
|
||||
Slice = cached!.Slice,
|
||||
SliceDigest = digest,
|
||||
DsseEnvelope = cached.DsseEnvelope,
|
||||
FromCache = true,
|
||||
SignatureVerified = cached.SignatureVerified
|
||||
};
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Pulling slice {Reference}@{Digest}", reference, digest);
|
||||
|
||||
// Get manifest first
|
||||
var manifestUrl = $"https://{reference.Registry}/v2/{reference.Repository}/manifests/{digest}";
|
||||
using var manifestRequest = new HttpRequestMessage(HttpMethod.Get, manifestUrl);
|
||||
manifestRequest.Headers.Accept.ParseAdd(OciMediaTypes.ArtifactManifest);
|
||||
await _authorization.AuthorizeRequestAsync(manifestRequest, reference, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var manifestResponse = await _httpClient.SendAsync(manifestRequest, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!manifestResponse.IsSuccessStatusCode)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = $"Failed to fetch manifest: {manifestResponse.StatusCode}"
|
||||
};
|
||||
}
|
||||
|
||||
var manifest = await manifestResponse.Content.ReadFromJsonAsync<OciManifest>(JsonOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (manifest == null)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = "Failed to parse manifest"
|
||||
};
|
||||
}
|
||||
|
||||
// Find slice layer
|
||||
var sliceLayer = manifest.Layers?.FirstOrDefault(l =>
|
||||
l.MediaType == OciMediaTypes.ReachabilitySlice ||
|
||||
l.MediaType == OciMediaTypes.SliceArtifact);
|
||||
|
||||
if (sliceLayer == null)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = "No slice layer found in manifest"
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch slice blob
|
||||
var blobUrl = $"https://{reference.Registry}/v2/{reference.Repository}/blobs/{sliceLayer.Digest}";
|
||||
using var blobRequest = new HttpRequestMessage(HttpMethod.Get, blobUrl);
|
||||
await _authorization.AuthorizeRequestAsync(blobRequest, reference, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var blobResponse = await _httpClient.SendAsync(blobRequest, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!blobResponse.IsSuccessStatusCode)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = $"Failed to fetch blob: {blobResponse.StatusCode}"
|
||||
};
|
||||
}
|
||||
|
||||
var sliceBytes = await blobResponse.Content.ReadAsByteArrayAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Verify digest
|
||||
var computedDigest = ComputeDigest(sliceBytes);
|
||||
if (!string.Equals(computedDigest, sliceLayer.Digest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = $"Digest mismatch: expected {sliceLayer.Digest}, got {computedDigest}"
|
||||
};
|
||||
}
|
||||
|
||||
// Parse slice
|
||||
var slice = JsonSerializer.Deserialize<ReachabilitySlice>(sliceBytes, JsonOptions);
|
||||
if (slice == null)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = "Failed to parse slice JSON"
|
||||
};
|
||||
}
|
||||
|
||||
// Check for DSSE envelope layer and verify if present
|
||||
byte[]? dsseEnvelope = null;
|
||||
bool signatureVerified = false;
|
||||
|
||||
var dsseLayer = manifest.Layers?.FirstOrDefault(l =>
|
||||
l.MediaType == OciMediaTypes.DsseEnvelope);
|
||||
|
||||
if (dsseLayer != null && _options.VerifySignature)
|
||||
{
|
||||
var dsseResult = await FetchAndVerifyDsseAsync(reference, dsseLayer.Digest, sliceBytes, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
dsseEnvelope = dsseResult.Envelope;
|
||||
signatureVerified = dsseResult.Verified;
|
||||
}
|
||||
|
||||
// Cache result
|
||||
if (_options.EnableCache)
|
||||
{
|
||||
AddToCache(cacheKey, new CachedSlice
|
||||
{
|
||||
Slice = slice,
|
||||
DsseEnvelope = dsseEnvelope,
|
||||
SignatureVerified = signatureVerified,
|
||||
ExpiresAt = DateTimeOffset.UtcNow.Add(_options.CacheTtl)
|
||||
});
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Successfully pulled slice {Digest} ({Size} bytes, signature verified: {Verified})",
|
||||
digest, sliceBytes.Length, signatureVerified);
|
||||
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = true,
|
||||
Slice = slice,
|
||||
SliceDigest = digest,
|
||||
DsseEnvelope = dsseEnvelope,
|
||||
FromCache = false,
|
||||
SignatureVerified = signatureVerified
|
||||
};
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to pull slice {Reference}@{Digest}", reference, digest);
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pull a slice by tag.
|
||||
/// </summary>
|
||||
public async Task<SlicePullResult> PullByTagAsync(
|
||||
OciImageReference reference,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reference);
|
||||
|
||||
if (string.IsNullOrEmpty(reference.Tag))
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = "Tag is required"
|
||||
};
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Resolve tag to digest
|
||||
var manifestUrl = $"https://{reference.Registry}/v2/{reference.Repository}/manifests/{reference.Tag}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Head, manifestUrl);
|
||||
request.Headers.Accept.ParseAdd(OciMediaTypes.ArtifactManifest);
|
||||
await _authorization.AuthorizeRequestAsync(request, reference, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = $"Failed to resolve tag: {response.StatusCode}"
|
||||
};
|
||||
}
|
||||
|
||||
var digest = response.Headers.GetValues("Docker-Content-Digest").FirstOrDefault();
|
||||
if (string.IsNullOrEmpty(digest))
|
||||
{
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = "No digest in response headers"
|
||||
};
|
||||
}
|
||||
|
||||
return await PullByDigestAsync(reference, digest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to pull slice by tag {Reference}", reference);
|
||||
return new SlicePullResult
|
||||
{
|
||||
Success = false,
|
||||
Error = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// List referrers (related artifacts) for a given digest.
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<OciReferrer>> ListReferrersAsync(
|
||||
OciImageReference reference,
|
||||
string digest,
|
||||
string? artifactType = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reference);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
|
||||
|
||||
try
|
||||
{
|
||||
var referrersUrl = $"https://{reference.Registry}/v2/{reference.Repository}/referrers/{digest}";
|
||||
if (!string.IsNullOrEmpty(artifactType))
|
||||
{
|
||||
referrersUrl += $"?artifactType={Uri.EscapeDataString(artifactType)}";
|
||||
}
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, referrersUrl);
|
||||
await _authorization.AuthorizeRequestAsync(request, reference, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
_logger.LogWarning("Failed to list referrers for {Digest}: {Status}", digest, response.StatusCode);
|
||||
return Array.Empty<OciReferrer>();
|
||||
}
|
||||
|
||||
var index = await response.Content.ReadFromJsonAsync<OciReferrersIndex>(JsonOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return index?.Manifests ?? Array.Empty<OciReferrer>();
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to list referrers for {Digest}", digest);
|
||||
return Array.Empty<OciReferrer>();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
// HttpClient typically managed externally
|
||||
}
|
||||
|
||||
private async Task<(byte[]? Envelope, bool Verified)> FetchAndVerifyDsseAsync(
|
||||
OciImageReference reference,
|
||||
string digest,
|
||||
byte[] payload,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var blobUrl = $"https://{reference.Registry}/v2/{reference.Repository}/blobs/{digest}";
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, blobUrl);
|
||||
await _authorization.AuthorizeRequestAsync(request, reference, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var response = await _httpClient.SendAsync(request, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return (null, false);
|
||||
}
|
||||
|
||||
var envelopeBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// TODO: Actual DSSE verification using configured trust roots
|
||||
// For now, just return the envelope
|
||||
_logger.LogDebug("DSSE envelope fetched, verification pending trust root configuration");
|
||||
|
||||
return (envelopeBytes, false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to fetch/verify DSSE envelope");
|
||||
return (null, false);
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryGetFromCache(string key, out CachedSlice? cached)
|
||||
{
|
||||
lock (_cacheLock)
|
||||
{
|
||||
if (_cache.TryGetValue(key, out cached))
|
||||
{
|
||||
if (cached.ExpiresAt > DateTimeOffset.UtcNow)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_cache.Remove(key);
|
||||
}
|
||||
cached = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void AddToCache(string key, CachedSlice cached)
|
||||
{
|
||||
lock (_cacheLock)
|
||||
{
|
||||
_cache[key] = cached;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeDigest(byte[] data)
|
||||
{
|
||||
var hash = SHA256.HashData(data);
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
private sealed record CachedSlice
|
||||
{
|
||||
public required ReachabilitySlice Slice { get; init; }
|
||||
public byte[]? DsseEnvelope { get; init; }
|
||||
public bool SignatureVerified { get; init; }
|
||||
public required DateTimeOffset ExpiresAt { get; init; }
|
||||
}
|
||||
|
||||
// Internal DTOs for OCI registry responses
|
||||
private sealed record OciManifest
|
||||
{
|
||||
public int SchemaVersion { get; init; }
|
||||
public string? MediaType { get; init; }
|
||||
public string? ArtifactType { get; init; }
|
||||
public OciDescriptor? Config { get; init; }
|
||||
public List<OciDescriptor>? Layers { get; init; }
|
||||
}
|
||||
|
||||
private sealed record OciDescriptor
|
||||
{
|
||||
public string? MediaType { get; init; }
|
||||
public string? Digest { get; init; }
|
||||
public long Size { get; init; }
|
||||
}
|
||||
|
||||
private sealed record OciReferrersIndex
|
||||
{
|
||||
public int SchemaVersion { get; init; }
|
||||
public string? MediaType { get; init; }
|
||||
public List<OciReferrer>? Manifests { get; init; }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// OCI referrer descriptor.
|
||||
/// </summary>
|
||||
public sealed record OciReferrer
|
||||
{
|
||||
public string? MediaType { get; init; }
|
||||
public string? Digest { get; init; }
|
||||
public long Size { get; init; }
|
||||
public string? ArtifactType { get; init; }
|
||||
public Dictionary<string, string>? Annotations { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user