stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

stabilizaiton work - projects rework for maintenanceability and ui livening

Browse Source
This commit is contained in:
master
2026-02-03 23:40:04 +02:00
parent 074ce117ba
commit 557feefdc3
3305 changed files with 186813 additions and 107843 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/__Libraries/StellaOps.Provcache/LazyFetch/FetchedChunk.cs Normal file
View File

@@ -0,0 +1,23 @@
namespace StellaOps.Provcache;
/// <summary>
/// Simplified chunk representation for lazy fetch interface.
/// Contains only the index and data for transport.
/// </summary>
public sealed record FetchedChunk
{
/// <summary>
/// Zero-based chunk index.
/// </summary>
public required int Index { get; init; }
/// <summary>
/// The chunk data.
/// </summary>
public required byte[] Data { get; init; }
/// <summary>
/// SHA256 hash of the data for verification.
/// </summary>
public required string Hash { get; init; }
}

17
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.Availability.cs Normal file
View File

@@ -0,0 +1,17 @@
using Microsoft.Extensions.Logging;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <inheritdoc />
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
{
var isAvailable = Directory.Exists(_basePath);
_logger.LogDebug("File fetcher availability check: {IsAvailable}", isAvailable);
return Task.FromResult(isAvailable);
}
}

82
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.Export.cs Normal file
View File

@@ -0,0 +1,82 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <summary>
/// Exports chunks to files for sneakernet transfer.
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="manifest">The chunk manifest.</param>
/// <param name="chunks">The chunks to export.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public async Task ExportToFilesAsync(
string proofRoot,
ChunkManifest manifest,
IEnumerable<FetchedChunk> chunks,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(chunks);
var safeProofRoot = SanitizeForPath(proofRoot);
var proofDir = Path.Combine(_basePath, safeProofRoot);
Directory.CreateDirectory(proofDir);
_logger.LogInformation("Exporting to {Directory}", proofDir);
// Write manifest.
var manifestPath = GetManifestPath(proofRoot);
await using (var manifestStream = File.Create(manifestPath))
{
await JsonSerializer.SerializeAsync(manifestStream, manifest, _jsonOptions, cancellationToken);
}
_logger.LogDebug("Wrote manifest to {Path}", manifestPath);
// Write chunks.
var count = 0;
foreach (var chunk in chunks)
{
cancellationToken.ThrowIfCancellationRequested();
var chunkPath = GetChunkPath(proofRoot, chunk.Index);
await using var chunkStream = File.Create(chunkPath);
await JsonSerializer.SerializeAsync(chunkStream, chunk, _jsonOptions, cancellationToken);
count++;
}
_logger.LogInformation("Exported {Count} chunks to {Directory}", count, proofDir);
}
/// <summary>
/// Exports EvidenceChunks to files (converts to FetchedChunk format).
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="manifest">The chunk manifest.</param>
/// <param name="chunks">The evidence chunks to export.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public Task ExportEvidenceChunksToFilesAsync(
string proofRoot,
ChunkManifest manifest,
IEnumerable<EvidenceChunk> chunks,
CancellationToken cancellationToken = default)
{
var fetchedChunks = chunks.Select(c => new FetchedChunk
{
Index = c.ChunkIndex,
Data = c.Blob,
Hash = c.ChunkHash
});
return ExportToFilesAsync(proofRoot, manifest, fetchedChunks, cancellationToken);
}
}

66
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.Fetch.cs Normal file
View File

@@ -0,0 +1,66 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchChunksAsync(
string proofRoot,
IEnumerable<int> chunkIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(chunkIndices);
var indices = chunkIndices.ToList();
_logger.LogInformation(
"Fetching {Count} chunks from file system for proof root {ProofRoot}",
indices.Count,
proofRoot);
foreach (var index in indices)
{
cancellationToken.ThrowIfCancellationRequested();
var chunk = await FetchChunkAsync(proofRoot, index, cancellationToken);
if (chunk is not null)
{
yield return chunk;
}
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchRemainingChunksAsync(
string proofRoot,
ChunkManifest manifest,
IReadOnlySet<int> existingIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(existingIndices);
var missingIndices = Enumerable.Range(0, manifest.TotalChunks)
.Where(i => !existingIndices.Contains(i))
.ToList();
_logger.LogInformation(
"Fetching {MissingCount} remaining chunks from files (have {ExistingCount}/{TotalCount})",
missingIndices.Count,
existingIndices.Count,
manifest.TotalChunks);
await foreach (var chunk in FetchChunksAsync(proofRoot, missingIndices, cancellationToken))
{
yield return chunk;
}
}
}

46
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.FetchChunk.cs Normal file
View File

@@ -0,0 +1,46 @@
using Microsoft.Extensions.Logging;
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <inheritdoc />
public async Task<FetchedChunk?> FetchChunkAsync(
string proofRoot,
int chunkIndex,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentOutOfRangeException.ThrowIfNegative(chunkIndex);
var chunkPath = GetChunkPath(proofRoot, chunkIndex);
_logger.LogDebug("Looking for chunk at {Path}", chunkPath);
if (!File.Exists(chunkPath))
{
_logger.LogDebug("Chunk file not found: {Path}", chunkPath);
return null;
}
try
{
await using var stream = File.OpenRead(chunkPath);
var chunk = await JsonSerializer.DeserializeAsync<FetchedChunk>(stream, _jsonOptions, cancellationToken);
_logger.LogDebug(
"Successfully loaded chunk {Index}, {Bytes} bytes",
chunkIndex,
chunk?.Data.Length ?? 0);
return chunk;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error reading chunk file {Path}", chunkPath);
throw;
}
}
}

42
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.Manifest.cs Normal file
View File

@@ -0,0 +1,42 @@
using Microsoft.Extensions.Logging;
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <inheritdoc />
public async Task<ChunkManifest?> FetchManifestAsync(
string proofRoot,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
var manifestPath = GetManifestPath(proofRoot);
_logger.LogDebug("Looking for manifest at {Path}", manifestPath);
if (!File.Exists(manifestPath))
{
_logger.LogDebug("Manifest file not found: {Path}", manifestPath);
return null;
}
try
{
await using var stream = File.OpenRead(manifestPath);
return await JsonSerializer.DeserializeAsync<ChunkManifest>(
stream,
_jsonOptions,
cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error reading manifest file {Path}", manifestPath);
throw;
}
}
}

41
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.Paths.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Provcache;
public sealed partial class FileChunkFetcher
{
/// <summary>
/// Gets the file path for a chunk.
/// </summary>
private string GetChunkPath(string proofRoot, int chunkIndex)
{
// Sanitize proof root for use in file paths.
var safeProofRoot = SanitizeForPath(proofRoot);
return Path.Combine(_basePath, safeProofRoot, $"chunk_{chunkIndex:D4}.json");
}
/// <summary>
/// Gets the file path for a manifest.
/// </summary>
private string GetManifestPath(string proofRoot)
{
var safeProofRoot = SanitizeForPath(proofRoot);
return Path.Combine(_basePath, safeProofRoot, "manifest.json");
}
/// <summary>
/// Sanitizes a proof root for use in file paths.
/// </summary>
private static string SanitizeForPath(string input)
{
// Use hash prefix to ensure consistent directory naming.
var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(input)))
.ToLowerInvariant();
// Return first 16 chars of hash for reasonable directory names.
return hash[..16];
}
}

226
src/__Libraries/StellaOps.Provcache/LazyFetch/FileChunkFetcher.cs
View File

@@ -1,7 +1,6 @@
using Microsoft.Extensions.Logging;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System;
using System.IO;
using System.Text.Json;
namespace StellaOps.Provcache;
@@ -10,7 +9,7 @@ namespace StellaOps.Provcache;
/// File-based lazy evidence chunk fetcher for sneakernet mode.
/// Fetches chunks from a local directory (e.g., USB drive, NFS mount).
/// </summary>
public sealed class FileChunkFetcher : ILazyEvidenceFetcher
public sealed partial class FileChunkFetcher : ILazyEvidenceFetcher
{
private readonly string _basePath;
private readonly ILogger<FileChunkFetcher> _logger;
@@ -27,7 +26,7 @@ public sealed class FileChunkFetcher : ILazyEvidenceFetcher
public FileChunkFetcher(string basePath, ILogger<FileChunkFetcher> logger)
{
ArgumentException.ThrowIfNullOrWhiteSpace(basePath);
_basePath = Path.GetFullPath(basePath);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_jsonOptions = new JsonSerializerOptions
@@ -38,221 +37,4 @@ public sealed class FileChunkFetcher : ILazyEvidenceFetcher
_logger.LogDebug("FileChunkFetcher initialized with base path: {BasePath}", _basePath);
}
/// <inheritdoc />
public async Task<FetchedChunk?> FetchChunkAsync(
string proofRoot,
int chunkIndex,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentOutOfRangeException.ThrowIfNegative(chunkIndex);
var chunkPath = GetChunkPath(proofRoot, chunkIndex);
_logger.LogDebug("Looking for chunk at {Path}", chunkPath);
if (!File.Exists(chunkPath))
{
_logger.LogDebug("Chunk file not found: {Path}", chunkPath);
return null;
}
try
{
await using var stream = File.OpenRead(chunkPath);
var chunk = await JsonSerializer.DeserializeAsync<FetchedChunk>(stream, _jsonOptions, cancellationToken);
_logger.LogDebug("Successfully loaded chunk {Index}, {Bytes} bytes", chunkIndex, chunk?.Data.Length ?? 0);
return chunk;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error reading chunk file {Path}", chunkPath);
throw;
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchChunksAsync(
string proofRoot,
IEnumerable<int> chunkIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(chunkIndices);
var indices = chunkIndices.ToList();
_logger.LogInformation("Fetching {Count} chunks from file system for proof root {ProofRoot}", indices.Count, proofRoot);
foreach (var index in indices)
{
cancellationToken.ThrowIfCancellationRequested();
var chunk = await FetchChunkAsync(proofRoot, index, cancellationToken);
if (chunk is not null)
{
yield return chunk;
}
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchRemainingChunksAsync(
string proofRoot,
ChunkManifest manifest,
IReadOnlySet<int> existingIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(existingIndices);
var missingIndices = Enumerable.Range(0, manifest.TotalChunks)
.Where(i => !existingIndices.Contains(i))
.ToList();
_logger.LogInformation(
"Fetching {MissingCount} remaining chunks from files (have {ExistingCount}/{TotalCount})",
missingIndices.Count, existingIndices.Count, manifest.TotalChunks);
await foreach (var chunk in FetchChunksAsync(proofRoot, missingIndices, cancellationToken))
{
yield return chunk;
}
}
/// <inheritdoc />
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
{
var isAvailable = Directory.Exists(_basePath);
_logger.LogDebug("File fetcher availability check: {IsAvailable}", isAvailable);
return Task.FromResult(isAvailable);
}
/// <inheritdoc />
public async Task<ChunkManifest?> FetchManifestAsync(
string proofRoot,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
var manifestPath = GetManifestPath(proofRoot);
_logger.LogDebug("Looking for manifest at {Path}", manifestPath);
if (!File.Exists(manifestPath))
{
_logger.LogDebug("Manifest file not found: {Path}", manifestPath);
return null;
}
try
{
await using var stream = File.OpenRead(manifestPath);
return await JsonSerializer.DeserializeAsync<ChunkManifest>(stream, _jsonOptions, cancellationToken);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Error reading manifest file {Path}", manifestPath);
throw;
}
}
/// <summary>
/// Gets the file path for a chunk.
/// </summary>
private string GetChunkPath(string proofRoot, int chunkIndex)
{
// Sanitize proof root for use in file paths
var safeProofRoot = SanitizeForPath(proofRoot);
return Path.Combine(_basePath, safeProofRoot, $"chunk_{chunkIndex:D4}.json");
}
/// <summary>
/// Gets the file path for a manifest.
/// </summary>
private string GetManifestPath(string proofRoot)
{
var safeProofRoot = SanitizeForPath(proofRoot);
return Path.Combine(_basePath, safeProofRoot, "manifest.json");
}
/// <summary>
/// Sanitizes a proof root for use in file paths.
/// </summary>
private static string SanitizeForPath(string input)
{
// Use hash prefix to ensure consistent directory naming
var hash = Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(input))).ToLowerInvariant();
// Return first 16 chars of hash for reasonable directory names
return hash[..16];
}
/// <summary>
/// Exports chunks to files for sneakernet transfer.
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="manifest">The chunk manifest.</param>
/// <param name="chunks">The chunks to export.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public async Task ExportToFilesAsync(
string proofRoot,
ChunkManifest manifest,
IEnumerable<FetchedChunk> chunks,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(chunks);
var safeProofRoot = SanitizeForPath(proofRoot);
var proofDir = Path.Combine(_basePath, safeProofRoot);
Directory.CreateDirectory(proofDir);
_logger.LogInformation("Exporting to {Directory}", proofDir);
// Write manifest
var manifestPath = GetManifestPath(proofRoot);
await using (var manifestStream = File.Create(manifestPath))
{
await JsonSerializer.SerializeAsync(manifestStream, manifest, _jsonOptions, cancellationToken);
}
_logger.LogDebug("Wrote manifest to {Path}", manifestPath);
// Write chunks
var count = 0;
foreach (var chunk in chunks)
{
cancellationToken.ThrowIfCancellationRequested();
var chunkPath = GetChunkPath(proofRoot, chunk.Index);
await using var chunkStream = File.Create(chunkPath);
await JsonSerializer.SerializeAsync(chunkStream, chunk, _jsonOptions, cancellationToken);
count++;
}
_logger.LogInformation("Exported {Count} chunks to {Directory}", count, proofDir);
}
/// <summary>
/// Exports EvidenceChunks to files (converts to FetchedChunk format).
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="manifest">The chunk manifest.</param>
/// <param name="chunks">The evidence chunks to export.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public Task ExportEvidenceChunksToFilesAsync(
string proofRoot,
ChunkManifest manifest,
IEnumerable<EvidenceChunk> chunks,
CancellationToken cancellationToken = default)
{
var fetchedChunks = chunks.Select(c => new FetchedChunk
{
Index = c.ChunkIndex,
Data = c.Blob,
Hash = c.ChunkHash
});
return ExportToFilesAsync(proofRoot, manifest, fetchedChunks, cancellationToken);
}
}

65
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Allowlist.cs Normal file
View File

@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
private static IReadOnlySet<string> NormalizeSchemes(IList<string> schemes)
{
var normalized = schemes
.Where(s => !string.IsNullOrWhiteSpace(s))
.Select(s => s.Trim())
.ToArray();
if (normalized.Length == 0)
{
normalized = DefaultSchemes;
}
return new HashSet<string>(normalized, StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyList<string> NormalizeHosts(
IList<string> hosts,
string baseHost,
out bool allowAllHosts)
{
var normalized = hosts
.Where(h => !string.IsNullOrWhiteSpace(h))
.Select(h => h.Trim())
.ToList();
allowAllHosts = normalized.Any(h => string.Equals(h, "*", StringComparison.Ordinal));
if (!allowAllHosts && normalized.Count == 0)
{
normalized.Add(baseHost);
}
return normalized;
}
private static bool IsHostAllowed(string host, IReadOnlyList<string> allowedHosts)
{
foreach (var allowed in allowedHosts)
{
if (string.Equals(allowed, host, StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (allowed.StartsWith("*.", StringComparison.Ordinal))
{
var suffix = allowed[1..];
if (host.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
}
return false;
}
}

24
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Availability.cs Normal file
View File

@@ -0,0 +1,24 @@
using Microsoft.Extensions.Logging;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
/// <inheritdoc />
public async Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
{
try
{
var response = await _httpClient.GetAsync("api/v1/health", cancellationToken);
return response.IsSuccessStatusCode;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Health check failed");
return false;
}
}
}

41
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Configuration.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
private static HttpClient CreateClient(IHttpClientFactory httpClientFactory, Uri baseUrl)
{
ArgumentNullException.ThrowIfNull(httpClientFactory);
ArgumentNullException.ThrowIfNull(baseUrl);
var client = httpClientFactory.CreateClient(HttpClientName);
client.BaseAddress = baseUrl;
return client;
}
private void ApplyClientConfiguration()
{
var timeout = _options.Timeout;
if (timeout <= TimeSpan.Zero || timeout == Timeout.InfiniteTimeSpan)
{
throw new InvalidOperationException("Lazy fetch HTTP timeout must be a positive, non-infinite duration.");
}
if (_httpClient.Timeout == Timeout.InfiniteTimeSpan || _httpClient.Timeout > timeout)
{
_httpClient.Timeout = timeout;
}
if (!_httpClient.DefaultRequestHeaders.Accept.Any(header =>
string.Equals(header.MediaType, "application/json", StringComparison.OrdinalIgnoreCase)))
{
_httpClient.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
}
}
}

66
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Fetch.cs Normal file
View File

@@ -0,0 +1,66 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchChunksAsync(
string proofRoot,
IEnumerable<int> chunkIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(chunkIndices);
var indices = chunkIndices.ToList();
_logger.LogInformation(
"Fetching {Count} chunks for proof root {ProofRoot}",
indices.Count,
proofRoot);
foreach (var index in indices)
{
cancellationToken.ThrowIfCancellationRequested();
var chunk = await FetchChunkAsync(proofRoot, index, cancellationToken);
if (chunk is not null)
{
yield return chunk;
}
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchRemainingChunksAsync(
string proofRoot,
ChunkManifest manifest,
IReadOnlySet<int> existingIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(existingIndices);
var missingIndices = Enumerable.Range(0, manifest.TotalChunks)
.Where(i => !existingIndices.Contains(i))
.ToList();
_logger.LogInformation(
"Fetching {MissingCount} remaining chunks (have {ExistingCount}/{TotalCount})",
missingIndices.Count,
existingIndices.Count,
manifest.TotalChunks);
await foreach (var chunk in FetchChunksAsync(proofRoot, missingIndices, cancellationToken))
{
yield return chunk;
}
}
}

51
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.FetchChunk.cs Normal file
View File

@@ -0,0 +1,51 @@
using Microsoft.Extensions.Logging;
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
/// <inheritdoc />
public async Task<FetchedChunk?> FetchChunkAsync(
string proofRoot,
int chunkIndex,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentOutOfRangeException.ThrowIfNegative(chunkIndex);
var url = $"api/v1/evidence/{Uri.EscapeDataString(proofRoot)}/chunks/{chunkIndex}";
_logger.LogDebug("Fetching chunk {Index} from {Url}", chunkIndex, url);
try
{
var response = await _httpClient.GetAsync(url, cancellationToken);
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Chunk {Index} not found at remote", chunkIndex);
return null;
}
response.EnsureSuccessStatusCode();
var chunk = await response.Content.ReadFromJsonAsync<FetchedChunk>(
_jsonOptions,
cancellationToken);
_logger.LogDebug(
"Successfully fetched chunk {Index}, {Bytes} bytes",
chunkIndex,
chunk?.Data.Length ?? 0);
return chunk;
}
catch (HttpRequestException ex)
{
_logger.LogWarning(ex, "HTTP error fetching chunk {Index}", chunkIndex);
throw;
}
}
}

41
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Manifest.cs Normal file
View File

@@ -0,0 +1,41 @@
using Microsoft.Extensions.Logging;
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
/// <inheritdoc />
public async Task<ChunkManifest?> FetchManifestAsync(
string proofRoot,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
var url = $"api/v1/evidence/{Uri.EscapeDataString(proofRoot)}/manifest";
_logger.LogDebug("Fetching manifest from {Url}", url);
try
{
var response = await _httpClient.GetAsync(url, cancellationToken);
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Manifest not found for proof root {ProofRoot}", proofRoot);
return null;
}
response.EnsureSuccessStatusCode();
return await response.Content.ReadFromJsonAsync<ChunkManifest>(_jsonOptions, cancellationToken);
}
catch (HttpRequestException ex)
{
_logger.LogWarning(ex, "HTTP error fetching manifest for {ProofRoot}", proofRoot);
throw;
}
}
}

34
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.Validation.cs Normal file
View File

@@ -0,0 +1,34 @@
using System;
namespace StellaOps.Provcache;
public sealed partial class HttpChunkFetcher
{
private void ValidateBaseAddress(Uri baseAddress)
{
if (!baseAddress.IsAbsoluteUri)
{
throw new InvalidOperationException("Lazy fetch base URL must be absolute.");
}
if (!string.IsNullOrWhiteSpace(baseAddress.UserInfo))
{
throw new InvalidOperationException("Lazy fetch base URL must not include user info.");
}
if (string.IsNullOrWhiteSpace(baseAddress.Host))
{
throw new InvalidOperationException("Lazy fetch base URL must include a host.");
}
if (!_allowedSchemes.Contains(baseAddress.Scheme))
{
throw new InvalidOperationException($"Lazy fetch base URL scheme '{baseAddress.Scheme}' is not allowed.");
}
if (!_allowAllHosts && !IsHostAllowed(baseAddress.Host, _allowedHosts))
{
throw new InvalidOperationException($"Lazy fetch base URL host '{baseAddress.Host}' is not allowlisted.");
}
}
}

254
src/__Libraries/StellaOps.Provcache/LazyFetch/HttpChunkFetcher.cs
View File

@@ -1,8 +1,7 @@
using Microsoft.Extensions.Logging;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Runtime.CompilerServices;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
namespace StellaOps.Provcache;
@@ -11,7 +10,7 @@ namespace StellaOps.Provcache;
/// HTTP-based lazy evidence chunk fetcher for connected mode.
/// Fetches chunks from a remote Stella API endpoint.
/// </summary>
public sealed class HttpChunkFetcher : ILazyEvidenceFetcher, IDisposable
public sealed partial class HttpChunkFetcher : ILazyEvidenceFetcher, IDisposable
{
/// <summary>
/// Named client for use with IHttpClientFactory.
@@ -81,251 +80,6 @@ public sealed class HttpChunkFetcher : ILazyEvidenceFetcher, IDisposable
ApplyClientConfiguration();
}
private static HttpClient CreateClient(IHttpClientFactory httpClientFactory, Uri baseUrl)
{
ArgumentNullException.ThrowIfNull(httpClientFactory);
ArgumentNullException.ThrowIfNull(baseUrl);
var client = httpClientFactory.CreateClient(HttpClientName);
client.BaseAddress = baseUrl;
return client;
}
private void ApplyClientConfiguration()
{
var timeout = _options.Timeout;
if (timeout <= TimeSpan.Zero || timeout == Timeout.InfiniteTimeSpan)
{
throw new InvalidOperationException("Lazy fetch HTTP timeout must be a positive, non-infinite duration.");
}
if (_httpClient.Timeout == Timeout.InfiniteTimeSpan || _httpClient.Timeout > timeout)
{
_httpClient.Timeout = timeout;
}
if (!_httpClient.DefaultRequestHeaders.Accept.Any(header =>
string.Equals(header.MediaType, "application/json", StringComparison.OrdinalIgnoreCase)))
{
_httpClient.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
}
}
private void ValidateBaseAddress(Uri baseAddress)
{
if (!baseAddress.IsAbsoluteUri)
{
throw new InvalidOperationException("Lazy fetch base URL must be absolute.");
}
if (!string.IsNullOrWhiteSpace(baseAddress.UserInfo))
{
throw new InvalidOperationException("Lazy fetch base URL must not include user info.");
}
if (string.IsNullOrWhiteSpace(baseAddress.Host))
{
throw new InvalidOperationException("Lazy fetch base URL must include a host.");
}
if (!_allowedSchemes.Contains(baseAddress.Scheme))
{
throw new InvalidOperationException($"Lazy fetch base URL scheme '{baseAddress.Scheme}' is not allowed.");
}
if (!_allowAllHosts && !IsHostAllowed(baseAddress.Host, _allowedHosts))
{
throw new InvalidOperationException($"Lazy fetch base URL host '{baseAddress.Host}' is not allowlisted.");
}
}
private static IReadOnlySet<string> NormalizeSchemes(IList<string> schemes)
{
var normalized = schemes
.Where(s => !string.IsNullOrWhiteSpace(s))
.Select(s => s.Trim())
.ToArray();
if (normalized.Length == 0)
{
normalized = DefaultSchemes;
}
return new HashSet<string>(normalized, StringComparer.OrdinalIgnoreCase);
}
private static IReadOnlyList<string> NormalizeHosts(
IList<string> hosts,
string baseHost,
out bool allowAllHosts)
{
var normalized = hosts
.Where(h => !string.IsNullOrWhiteSpace(h))
.Select(h => h.Trim())
.ToList();
allowAllHosts = normalized.Any(h => string.Equals(h, "*", StringComparison.Ordinal));
if (!allowAllHosts && normalized.Count == 0)
{
normalized.Add(baseHost);
}
return normalized;
}
private static bool IsHostAllowed(string host, IReadOnlyList<string> allowedHosts)
{
foreach (var allowed in allowedHosts)
{
if (string.Equals(allowed, host, StringComparison.OrdinalIgnoreCase))
{
return true;
}
if (allowed.StartsWith("*.", StringComparison.Ordinal))
{
var suffix = allowed[1..];
if (host.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
}
return false;
}
/// <inheritdoc />
public async Task<FetchedChunk?> FetchChunkAsync(
string proofRoot,
int chunkIndex,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentOutOfRangeException.ThrowIfNegative(chunkIndex);
var url = $"api/v1/evidence/{Uri.EscapeDataString(proofRoot)}/chunks/{chunkIndex}";
_logger.LogDebug("Fetching chunk {Index} from {Url}", chunkIndex, url);
try
{
var response = await _httpClient.GetAsync(url, cancellationToken);
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Chunk {Index} not found at remote", chunkIndex);
return null;
}
response.EnsureSuccessStatusCode();
var chunk = await response.Content.ReadFromJsonAsync<FetchedChunk>(_jsonOptions, cancellationToken);
_logger.LogDebug("Successfully fetched chunk {Index}, {Bytes} bytes", chunkIndex, chunk?.Data.Length ?? 0);
return chunk;
}
catch (HttpRequestException ex)
{
_logger.LogWarning(ex, "HTTP error fetching chunk {Index}", chunkIndex);
throw;
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchChunksAsync(
string proofRoot,
IEnumerable<int> chunkIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(chunkIndices);
var indices = chunkIndices.ToList();
_logger.LogInformation("Fetching {Count} chunks for proof root {ProofRoot}", indices.Count, proofRoot);
foreach (var index in indices)
{
cancellationToken.ThrowIfCancellationRequested();
var chunk = await FetchChunkAsync(proofRoot, index, cancellationToken);
if (chunk is not null)
{
yield return chunk;
}
}
}
/// <inheritdoc />
public async IAsyncEnumerable<FetchedChunk> FetchRemainingChunksAsync(
string proofRoot,
ChunkManifest manifest,
IReadOnlySet<int> existingIndices,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(manifest);
ArgumentNullException.ThrowIfNull(existingIndices);
var missingIndices = Enumerable.Range(0, manifest.TotalChunks)
.Where(i => !existingIndices.Contains(i))
.ToList();
_logger.LogInformation(
"Fetching {MissingCount} remaining chunks (have {ExistingCount}/{TotalCount})",
missingIndices.Count, existingIndices.Count, manifest.TotalChunks);
await foreach (var chunk in FetchChunksAsync(proofRoot, missingIndices, cancellationToken))
{
yield return chunk;
}
}
/// <inheritdoc />
public async Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
{
try
{
var response = await _httpClient.GetAsync("api/v1/health", cancellationToken);
return response.IsSuccessStatusCode;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Health check failed");
return false;
}
}
/// <inheritdoc />
public async Task<ChunkManifest?> FetchManifestAsync(
string proofRoot,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
var url = $"api/v1/evidence/{Uri.EscapeDataString(proofRoot)}/manifest";
_logger.LogDebug("Fetching manifest from {Url}", url);
try
{
var response = await _httpClient.GetAsync(url, cancellationToken);
if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogDebug("Manifest not found for proof root {ProofRoot}", proofRoot);
return null;
}
response.EnsureSuccessStatusCode();
return await response.Content.ReadFromJsonAsync<ChunkManifest>(_jsonOptions, cancellationToken);
}
catch (HttpRequestException ex)
{
_logger.LogWarning(ex, "HTTP error fetching manifest for {ProofRoot}", proofRoot);
throw;
}
}
/// <inheritdoc />
public void Dispose()
{

63
src/__Libraries/StellaOps.Provcache/LazyFetch/ILazyEvidenceFetcher.cs
View File

@@ -66,66 +66,3 @@ public interface ILazyEvidenceFetcher
string proofRoot,
CancellationToken cancellationToken = default);
}
/// <summary>
/// Simplified chunk representation for lazy fetch interface.
/// Contains only the index and data for transport.
/// </summary>
public sealed record FetchedChunk
{
/// <summary>
/// Zero-based chunk index.
/// </summary>
public required int Index { get; init; }
/// <summary>
/// The chunk data.
/// </summary>
public required byte[] Data { get; init; }
/// <summary>
/// SHA256 hash of the data for verification.
/// </summary>
public required string Hash { get; init; }
}
/// <summary>
/// Result of a lazy fetch operation.
/// </summary>
public sealed record LazyFetchResult
{
/// <summary>
/// Whether the fetch was successful.
/// </summary>
public required bool Success { get; init; }
/// <summary>
/// Number of chunks fetched.
/// </summary>
public required int ChunksFetched { get; init; }
/// <summary>
/// Total bytes fetched.
/// </summary>
public required long BytesFetched { get; init; }
/// <summary>
/// Number of chunks that failed verification.
/// </summary>
public required int ChunksFailedVerification { get; init; }
/// <summary>
/// Indices of failed chunks.
/// </summary>
public IReadOnlyList<int> FailedIndices { get; init; } = [];
/// <summary>
/// Any errors encountered.
/// </summary>
public IReadOnlyList<string> Errors { get; init; } = [];
/// <summary>
/// Time taken for the fetch operation.
/// </summary>
public TimeSpan Duration { get; init; }
}

27
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOptions.cs Normal file
View File

@@ -0,0 +1,27 @@
namespace StellaOps.Provcache;
/// <summary>
/// Options for lazy fetch operations.
/// </summary>
public sealed class LazyFetchOptions
{
/// <summary>
/// Whether to verify chunks on fetch.
/// </summary>
public bool VerifyOnFetch { get; init; } = true;
/// <summary>
/// Whether to fail the entire operation on verification error.
/// </summary>
public bool FailOnVerificationError { get; init; } = false;
/// <summary>
/// Batch size for storing chunks.
/// </summary>
public int BatchSize { get; init; } = 100;
/// <summary>
/// Maximum number of chunks to fetch (0 = unlimited).
/// </summary>
public int MaxChunksToFetch { get; init; } = 0;
}

97
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Fetch.cs Normal file
View File

@@ -0,0 +1,97 @@
using Microsoft.Extensions.Logging;
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
/// <summary>
/// Fetches remaining chunks for a proof root and stores them locally.
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="fetcher">The fetcher to use.</param>
/// <param name="options">Fetch options.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The fetch result.</returns>
public async Task<LazyFetchResult> FetchAndStoreAsync(
string proofRoot,
ILazyEvidenceFetcher fetcher,
LazyFetchOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(fetcher);
options ??= new LazyFetchOptions();
var stopwatch = Stopwatch.StartNew();
var state = new LazyFetchRunState();
_logger.LogInformation(
"Starting lazy fetch for {ProofRoot} using {FetcherType} fetcher",
proofRoot,
fetcher.FetcherType);
try
{
if (!await fetcher.IsAvailableAsync(cancellationToken))
{
_logger.LogWarning("Fetcher {FetcherType} is not available", fetcher.FetcherType);
state.Errors.Add($"Fetcher {fetcher.FetcherType} is not available");
return BuildResult(state, success: false, stopwatch.Elapsed);
}
var manifest = await ResolveManifestAsync(proofRoot, fetcher, cancellationToken);
if (manifest is null)
{
state.Errors.Add($"No manifest found for proof root {proofRoot}");
return BuildResult(state, success: false, stopwatch.Elapsed);
}
var existingIndices = await GetExistingIndicesAsync(proofRoot, cancellationToken);
var missingCount = manifest.TotalChunks - existingIndices.Count;
_logger.LogInformation(
"Have {Existing}/{Total} chunks, need to fetch {Missing}",
existingIndices.Count,
manifest.TotalChunks,
missingCount);
if (missingCount == 0)
{
_logger.LogInformation("All chunks already present, nothing to fetch");
return BuildResult(state, success: true, stopwatch.Elapsed);
}
await FetchAndStoreChunksAsync(
proofRoot,
fetcher,
manifest,
existingIndices,
options,
state,
cancellationToken);
stopwatch.Stop();
var success = state.ChunksFailedVerification == 0 || !options.FailOnVerificationError;
_logger.LogInformation(
"Lazy fetch complete: {Fetched} chunks, {Bytes} bytes, {Failed} verification failures in {Duration}",
state.ChunksFetched,
state.BytesFetched,
state.ChunksFailedVerification,
stopwatch.Elapsed);
return BuildResult(state, success, stopwatch.Elapsed);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during lazy fetch for {ProofRoot}", proofRoot);
state.Errors.Add(ex.Message);
return BuildResult(state, success: false, stopwatch.Elapsed);
}
}
}

18
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Indices.cs Normal file
View File

@@ -0,0 +1,18 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
private async Task<HashSet<int>> GetExistingIndicesAsync(
string proofRoot,
CancellationToken cancellationToken)
{
return (await _repository.GetChunksAsync(proofRoot, cancellationToken))
.Select(c => c.ChunkIndex)
.ToHashSet();
}
}

28
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Manifest.cs Normal file
View File

@@ -0,0 +1,28 @@
using Microsoft.Extensions.Logging;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
private async Task<ChunkManifest?> ResolveManifestAsync(
string proofRoot,
ILazyEvidenceFetcher fetcher,
CancellationToken cancellationToken)
{
var manifest = await _repository.GetManifestAsync(proofRoot, cancellationToken);
if (manifest is not null)
{
return manifest;
}
manifest = await fetcher.FetchManifestAsync(proofRoot, cancellationToken);
if (manifest is null)
{
_logger.LogWarning("No manifest found for {ProofRoot}", proofRoot);
}
return manifest;
}
}

23
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Results.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
private static LazyFetchResult BuildResult(
LazyFetchRunState state,
bool success,
TimeSpan duration)
{
return new LazyFetchResult
{
Success = success,
ChunksFetched = state.ChunksFetched,
BytesFetched = state.BytesFetched,
ChunksFailedVerification = state.ChunksFailedVerification,
FailedIndices = state.FailedIndices,
Errors = state.Errors,
Duration = duration
};
}
}

15
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.State.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Collections.Generic;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
private sealed class LazyFetchRunState
{
public List<string> Errors { get; } = [];
public List<int> FailedIndices { get; } = [];
public int ChunksFetched { get; set; }
public long BytesFetched { get; set; }
public int ChunksFailedVerification { get; set; }
}
}

94
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Storage.cs Normal file
View File

@@ -0,0 +1,94 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
private async Task FetchAndStoreChunksAsync(
string proofRoot,
ILazyEvidenceFetcher fetcher,
ChunkManifest manifest,
IReadOnlySet<int> existingIndices,
LazyFetchOptions options,
LazyFetchRunState state,
CancellationToken cancellationToken)
{
var chunksToStore = new List<EvidenceChunk>();
var now = _timeProvider.GetUtcNow();
await foreach (var fetchedChunk in fetcher.FetchRemainingChunksAsync(
proofRoot,
manifest,
existingIndices,
cancellationToken))
{
if (options.VerifyOnFetch && !VerifyChunk(fetchedChunk, manifest))
{
state.ChunksFailedVerification++;
state.FailedIndices.Add(fetchedChunk.Index);
state.Errors.Add($"Chunk {fetchedChunk.Index} failed verification");
if (options.FailOnVerificationError)
{
_logger.LogError("Chunk {Index} failed verification, aborting", fetchedChunk.Index);
break;
}
_logger.LogWarning("Chunk {Index} failed verification, skipping", fetchedChunk.Index);
continue;
}
chunksToStore.Add(CreateEvidenceChunk(proofRoot, fetchedChunk, now));
state.BytesFetched += fetchedChunk.Data.Length;
state.ChunksFetched++;
if (chunksToStore.Count >= options.BatchSize)
{
await StoreChunkBatchAsync(proofRoot, chunksToStore, cancellationToken);
chunksToStore.Clear();
}
if (options.MaxChunksToFetch > 0 && state.ChunksFetched >= options.MaxChunksToFetch)
{
_logger.LogInformation("Reached max chunks limit ({Max})", options.MaxChunksToFetch);
break;
}
}
if (chunksToStore.Count > 0)
{
await StoreChunkBatchAsync(proofRoot, chunksToStore, cancellationToken);
}
}
private async Task StoreChunkBatchAsync(
string proofRoot,
List<EvidenceChunk> chunksToStore,
CancellationToken cancellationToken)
{
await _repository.StoreChunksAsync(proofRoot, chunksToStore, cancellationToken);
_logger.LogDebug("Stored batch of {Count} chunks", chunksToStore.Count);
}
private EvidenceChunk CreateEvidenceChunk(
string proofRoot,
FetchedChunk fetchedChunk,
DateTimeOffset createdAt)
{
return new EvidenceChunk
{
ChunkId = _guidProvider.NewGuid(),
ProofRoot = proofRoot,
ChunkIndex = fetchedChunk.Index,
ChunkHash = fetchedChunk.Hash,
Blob = fetchedChunk.Data,
BlobSize = fetchedChunk.Data.Length,
ContentType = "application/octet-stream",
CreatedAt = createdAt
};
}
}

51
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.Verify.cs Normal file
View File

@@ -0,0 +1,51 @@
using Microsoft.Extensions.Logging;
using System;
using System.Security.Cryptography;
namespace StellaOps.Provcache;
public sealed partial class LazyFetchOrchestrator
{
/// <summary>
/// Verifies a chunk against the manifest.
/// </summary>
private bool VerifyChunk(FetchedChunk chunk, ChunkManifest manifest)
{
// Check index bounds.
if (chunk.Index < 0 || chunk.Index >= manifest.TotalChunks)
{
_logger.LogWarning(
"Chunk index {Index} out of bounds (max {Max})",
chunk.Index,
manifest.TotalChunks - 1);
return false;
}
// Verify hash against manifest metadata.
if (manifest.Chunks is not null && chunk.Index < manifest.Chunks.Count)
{
var expectedHash = manifest.Chunks[chunk.Index].Hash;
var actualHash = Convert.ToHexString(SHA256.HashData(chunk.Data)).ToLowerInvariant();
if (!string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Chunk {Index} hash mismatch: expected {Expected}, got {Actual}",
chunk.Index, expectedHash, actualHash);
return false;
}
}
// Also verify the chunk's own hash claim.
var claimedHash = Convert.ToHexString(SHA256.HashData(chunk.Data)).ToLowerInvariant();
if (!string.Equals(claimedHash, chunk.Hash, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Chunk {Index} self-hash mismatch: claimed {Claimed}, actual {Actual}",
chunk.Index, chunk.Hash, claimedHash);
return false;
}
return true;
}
}

270
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchOrchestrator.cs
View File

@@ -1,8 +1,6 @@
using Microsoft.Extensions.Logging;
using StellaOps.Determinism;
using System.Diagnostics;
using System.Security.Cryptography;
using System;
namespace StellaOps.Provcache;
@@ -10,7 +8,7 @@ namespace StellaOps.Provcache;
/// Orchestrates lazy evidence fetching with verification.
/// Coordinates between fetchers and the local evidence store.
/// </summary>
public sealed class LazyFetchOrchestrator
public sealed partial class LazyFetchOrchestrator
{
private readonly IEvidenceChunkRepository _repository;
private readonly ILogger<LazyFetchOrchestrator> _logger;
@@ -35,268 +33,4 @@ public sealed class LazyFetchOrchestrator
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? SystemGuidProvider.Instance;
}
/// <summary>
/// Fetches remaining chunks for a proof root and stores them locally.
/// </summary>
/// <param name="proofRoot">The proof root.</param>
/// <param name="fetcher">The fetcher to use.</param>
/// <param name="options">Fetch options.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The fetch result.</returns>
public async Task<LazyFetchResult> FetchAndStoreAsync(
string proofRoot,
ILazyEvidenceFetcher fetcher,
LazyFetchOptions? options = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proofRoot);
ArgumentNullException.ThrowIfNull(fetcher);
options ??= new LazyFetchOptions();
var stopwatch = Stopwatch.StartNew();
var errors = new List<string>();
var failedIndices = new List<int>();
var chunksFetched = 0;
long bytesFetched = 0;
var chunksFailedVerification = 0;
_logger.LogInformation(
"Starting lazy fetch for {ProofRoot} using {FetcherType} fetcher",
proofRoot, fetcher.FetcherType);
try
{
// Check fetcher availability
if (!await fetcher.IsAvailableAsync(cancellationToken))
{
_logger.LogWarning("Fetcher {FetcherType} is not available", fetcher.FetcherType);
return new LazyFetchResult
{
Success = false,
ChunksFetched = 0,
BytesFetched = 0,
ChunksFailedVerification = 0,
Errors = [$"Fetcher {fetcher.FetcherType} is not available"],
Duration = stopwatch.Elapsed
};
}
// Get local manifest
var localManifest = await _repository.GetManifestAsync(proofRoot, cancellationToken);
if (localManifest is null)
{
// Try to fetch manifest from remote
localManifest = await fetcher.FetchManifestAsync(proofRoot, cancellationToken);
if (localManifest is null)
{
_logger.LogWarning("No manifest found for {ProofRoot}", proofRoot);
return new LazyFetchResult
{
Success = false,
ChunksFetched = 0,
BytesFetched = 0,
ChunksFailedVerification = 0,
Errors = [$"No manifest found for proof root {proofRoot}"],
Duration = stopwatch.Elapsed
};
}
}
// Get existing chunks
var existingChunks = (await _repository.GetChunksAsync(proofRoot, cancellationToken))
.Select(c => c.ChunkIndex)
.ToHashSet();
var totalChunks = localManifest.TotalChunks;
var missingCount = totalChunks - existingChunks.Count;
_logger.LogInformation(
"Have {Existing}/{Total} chunks, need to fetch {Missing}",
existingChunks.Count, totalChunks, missingCount);
if (missingCount == 0)
{
_logger.LogInformation("All chunks already present, nothing to fetch");
return new LazyFetchResult
{
Success = true,
ChunksFetched = 0,
BytesFetched = 0,
ChunksFailedVerification = 0,
Duration = stopwatch.Elapsed
};
}
// Fetch remaining chunks
var chunksToStore = new List<EvidenceChunk>();
var now = _timeProvider.GetUtcNow();
await foreach (var fetchedChunk in fetcher.FetchRemainingChunksAsync(
proofRoot, localManifest, existingChunks, cancellationToken))
{
// Verify chunk if enabled
if (options.VerifyOnFetch)
{
var isValid = VerifyChunk(fetchedChunk, localManifest);
if (!isValid)
{
chunksFailedVerification++;
failedIndices.Add(fetchedChunk.Index);
errors.Add($"Chunk {fetchedChunk.Index} failed verification");
if (options.FailOnVerificationError)
{
_logger.LogError("Chunk {Index} failed verification, aborting", fetchedChunk.Index);
break;
}
_logger.LogWarning("Chunk {Index} failed verification, skipping", fetchedChunk.Index);
continue;
}
}
// Convert FetchedChunk to EvidenceChunk for storage
var evidenceChunk = new EvidenceChunk
{
ChunkId = _guidProvider.NewGuid(),
ProofRoot = proofRoot,
ChunkIndex = fetchedChunk.Index,
ChunkHash = fetchedChunk.Hash,
Blob = fetchedChunk.Data,
BlobSize = fetchedChunk.Data.Length,
ContentType = "application/octet-stream",
CreatedAt = now
};
chunksToStore.Add(evidenceChunk);
bytesFetched += fetchedChunk.Data.Length;
chunksFetched++;
// Batch store to reduce database round-trips
if (chunksToStore.Count >= options.BatchSize)
{
await _repository.StoreChunksAsync(proofRoot, chunksToStore, cancellationToken);
_logger.LogDebug("Stored batch of {Count} chunks", chunksToStore.Count);
chunksToStore.Clear();
}
// Check max chunks limit
if (options.MaxChunksToFetch > 0 && chunksFetched >= options.MaxChunksToFetch)
{
_logger.LogInformation("Reached max chunks limit ({Max})", options.MaxChunksToFetch);
break;
}
}
// Store any remaining chunks
if (chunksToStore.Count > 0)
{
await _repository.StoreChunksAsync(proofRoot, chunksToStore, cancellationToken);
_logger.LogDebug("Stored final batch of {Count} chunks", chunksToStore.Count);
}
stopwatch.Stop();
var success = chunksFailedVerification == 0 || !options.FailOnVerificationError;
_logger.LogInformation(
"Lazy fetch complete: {Fetched} chunks, {Bytes} bytes, {Failed} verification failures in {Duration}",
chunksFetched, bytesFetched, chunksFailedVerification, stopwatch.Elapsed);
return new LazyFetchResult
{
Success = success,
ChunksFetched = chunksFetched,
BytesFetched = bytesFetched,
ChunksFailedVerification = chunksFailedVerification,
FailedIndices = failedIndices,
Errors = errors,
Duration = stopwatch.Elapsed
};
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during lazy fetch for {ProofRoot}", proofRoot);
errors.Add(ex.Message);
return new LazyFetchResult
{
Success = false,
ChunksFetched = chunksFetched,
BytesFetched = bytesFetched,
ChunksFailedVerification = chunksFailedVerification,
FailedIndices = failedIndices,
Errors = errors,
Duration = stopwatch.Elapsed
};
}
}
/// <summary>
/// Verifies a chunk against the manifest.
/// </summary>
private bool VerifyChunk(FetchedChunk chunk, ChunkManifest manifest)
{
// Check index bounds
if (chunk.Index < 0 || chunk.Index >= manifest.TotalChunks)
{
_logger.LogWarning("Chunk index {Index} out of bounds (max {Max})", chunk.Index, manifest.TotalChunks - 1);
return false;
}
// Verify hash against manifest metadata
if (manifest.Chunks is not null && chunk.Index < manifest.Chunks.Count)
{
var expectedHash = manifest.Chunks[chunk.Index].Hash;
var actualHash = Convert.ToHexString(SHA256.HashData(chunk.Data)).ToLowerInvariant();
if (!string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Chunk {Index} hash mismatch: expected {Expected}, got {Actual}",
chunk.Index, expectedHash, actualHash);
return false;
}
}
// Also verify the chunk's own hash claim
var claimedHash = Convert.ToHexString(SHA256.HashData(chunk.Data)).ToLowerInvariant();
if (!string.Equals(claimedHash, chunk.Hash, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning(
"Chunk {Index} self-hash mismatch: claimed {Claimed}, actual {Actual}",
chunk.Index, chunk.Hash, claimedHash);
return false;
}
return true;
}
}
/// <summary>
/// Options for lazy fetch operations.
/// </summary>
public sealed class LazyFetchOptions
{
/// <summary>
/// Whether to verify chunks on fetch.
/// </summary>
public bool VerifyOnFetch { get; init; } = true;
/// <summary>
/// Whether to fail the entire operation on verification error.
/// </summary>
public bool FailOnVerificationError { get; init; } = false;
/// <summary>
/// Batch size for storing chunks.
/// </summary>
public int BatchSize { get; init; } = 100;
/// <summary>
/// Maximum number of chunks to fetch (0 = unlimited).
/// </summary>
public int MaxChunksToFetch { get; init; } = 0;
}

45
src/__Libraries/StellaOps.Provcache/LazyFetch/LazyFetchResult.cs Normal file
View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Provcache;
/// <summary>
/// Result of a lazy fetch operation.
/// </summary>
public sealed record LazyFetchResult
{
/// <summary>
/// Whether the fetch was successful.
/// </summary>
public required bool Success { get; init; }
/// <summary>
/// Number of chunks fetched.
/// </summary>
public required int ChunksFetched { get; init; }
/// <summary>
/// Total bytes fetched.
/// </summary>
public required long BytesFetched { get; init; }
/// <summary>
/// Number of chunks that failed verification.
/// </summary>
public required int ChunksFailedVerification { get; init; }
/// <summary>
/// Indices of failed chunks.
/// </summary>
public IReadOnlyList<int> FailedIndices { get; init; } = [];
/// <summary>
/// Any errors encountered.
/// </summary>
public IReadOnlyList<string> Errors { get; init; } = [];
/// <summary>
/// Time taken for the fetch operation.
/// </summary>
public TimeSpan Duration { get; init; }
}