sprints enhancements
This commit is contained in:
StellaOps Bot
318
src/__Libraries/StellaOps.Provcache/Chunking/EvidenceChunker.cs
Normal file
318
src/__Libraries/StellaOps.Provcache/Chunking/EvidenceChunker.cs
Normal file
@@ -0,0 +1,318 @@
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace StellaOps.Provcache;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for splitting large evidence into fixed-size chunks
|
||||
/// and reassembling them with Merkle verification.
|
||||
/// </summary>
|
||||
public interface IEvidenceChunker
|
||||
{
|
||||
/// <summary>
|
||||
/// Splits evidence into chunks.
|
||||
/// </summary>
|
||||
/// <param name="evidence">The evidence bytes to split.</param>
|
||||
/// <param name="contentType">MIME type of the evidence.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The chunking result with chunks and proof root.</returns>
|
||||
Task<ChunkingResult> ChunkAsync(
|
||||
ReadOnlyMemory<byte> evidence,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Splits evidence from a stream.
|
||||
/// </summary>
|
||||
/// <param name="evidenceStream">Stream containing evidence.</param>
|
||||
/// <param name="contentType">MIME type of the evidence.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Async enumerable of chunks as they are created.</returns>
|
||||
IAsyncEnumerable<EvidenceChunk> ChunkStreamAsync(
|
||||
Stream evidenceStream,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reassembles chunks into the original evidence.
|
||||
/// </summary>
|
||||
/// <param name="chunks">The chunks to reassemble (must be in order).</param>
|
||||
/// <param name="expectedProofRoot">Expected Merkle root for verification.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The reassembled evidence bytes.</returns>
|
||||
Task<byte[]> ReassembleAsync(
|
||||
IEnumerable<EvidenceChunk> chunks,
|
||||
string expectedProofRoot,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a single chunk against its hash.
|
||||
/// </summary>
|
||||
/// <param name="chunk">The chunk to verify.</param>
|
||||
/// <returns>True if the chunk is valid.</returns>
|
||||
bool VerifyChunk(EvidenceChunk chunk);
|
||||
|
||||
/// <summary>
|
||||
/// Computes the Merkle root from chunk hashes.
|
||||
/// </summary>
|
||||
/// <param name="chunkHashes">Ordered list of chunk hashes.</param>
|
||||
/// <returns>The Merkle root.</returns>
|
||||
string ComputeMerkleRoot(IEnumerable<string> chunkHashes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of chunking evidence.
|
||||
/// </summary>
|
||||
public sealed record ChunkingResult
|
||||
{
|
||||
/// <summary>
|
||||
/// The computed Merkle root of all chunks.
|
||||
/// </summary>
|
||||
public required string ProofRoot { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The generated chunks.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<EvidenceChunk> Chunks { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total size of the original evidence.
|
||||
/// </summary>
|
||||
public required long TotalSize { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IEvidenceChunker"/>.
|
||||
/// </summary>
|
||||
public sealed class EvidenceChunker : IEvidenceChunker
|
||||
{
|
||||
private readonly ProvcacheOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public EvidenceChunker(ProvcacheOptions options, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<ChunkingResult> ChunkAsync(
|
||||
ReadOnlyMemory<byte> evidence,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(contentType);
|
||||
|
||||
var chunks = new List<EvidenceChunk>();
|
||||
var chunkHashes = new List<string>();
|
||||
var chunkSize = _options.ChunkSize;
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
var span = evidence.Span;
|
||||
var totalSize = span.Length;
|
||||
var chunkIndex = 0;
|
||||
|
||||
for (var offset = 0; offset < totalSize; offset += chunkSize)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var remainingBytes = totalSize - offset;
|
||||
var currentChunkSize = Math.Min(chunkSize, remainingBytes);
|
||||
var chunkData = span.Slice(offset, currentChunkSize).ToArray();
|
||||
var chunkHash = ComputeHash(chunkData);
|
||||
|
||||
chunks.Add(new EvidenceChunk
|
||||
{
|
||||
ChunkId = Guid.NewGuid(),
|
||||
ProofRoot = string.Empty, // Will be set after computing Merkle root
|
||||
ChunkIndex = chunkIndex,
|
||||
ChunkHash = chunkHash,
|
||||
Blob = chunkData,
|
||||
BlobSize = currentChunkSize,
|
||||
ContentType = contentType,
|
||||
CreatedAt = now
|
||||
});
|
||||
|
||||
chunkHashes.Add(chunkHash);
|
||||
chunkIndex++;
|
||||
}
|
||||
|
||||
var proofRoot = ComputeMerkleRoot(chunkHashes);
|
||||
|
||||
// Update proof root in all chunks
|
||||
var finalChunks = chunks.Select(c => c with { ProofRoot = proofRoot }).ToList();
|
||||
|
||||
return Task.FromResult(new ChunkingResult
|
||||
{
|
||||
ProofRoot = proofRoot,
|
||||
Chunks = finalChunks,
|
||||
TotalSize = totalSize
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<EvidenceChunk> ChunkStreamAsync(
|
||||
Stream evidenceStream,
|
||||
string contentType,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evidenceStream);
|
||||
ArgumentNullException.ThrowIfNull(contentType);
|
||||
|
||||
var chunkSize = _options.ChunkSize;
|
||||
var buffer = new byte[chunkSize];
|
||||
var chunkIndex = 0;
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
int bytesRead;
|
||||
while ((bytesRead = await evidenceStream.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
var chunkData = bytesRead == chunkSize ? buffer : buffer[..bytesRead];
|
||||
var chunkHash = ComputeHash(chunkData);
|
||||
|
||||
yield return new EvidenceChunk
|
||||
{
|
||||
ChunkId = Guid.NewGuid(),
|
||||
ProofRoot = string.Empty, // Caller must compute after all chunks
|
||||
ChunkIndex = chunkIndex,
|
||||
ChunkHash = chunkHash,
|
||||
Blob = chunkData.ToArray(),
|
||||
BlobSize = bytesRead,
|
||||
ContentType = contentType,
|
||||
CreatedAt = now
|
||||
};
|
||||
|
||||
chunkIndex++;
|
||||
buffer = new byte[chunkSize]; // New buffer for next chunk
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<byte[]> ReassembleAsync(
|
||||
IEnumerable<EvidenceChunk> chunks,
|
||||
string expectedProofRoot,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(chunks);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(expectedProofRoot);
|
||||
|
||||
var orderedChunks = chunks.OrderBy(c => c.ChunkIndex).ToList();
|
||||
|
||||
if (orderedChunks.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("No chunks provided.", nameof(chunks));
|
||||
}
|
||||
|
||||
// Verify Merkle root
|
||||
var chunkHashes = orderedChunks.Select(c => c.ChunkHash).ToList();
|
||||
var computedRoot = ComputeMerkleRoot(chunkHashes);
|
||||
|
||||
if (!string.Equals(computedRoot, expectedProofRoot, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Merkle root mismatch. Expected: {expectedProofRoot}, Computed: {computedRoot}");
|
||||
}
|
||||
|
||||
// Verify each chunk
|
||||
foreach (var chunk in orderedChunks)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!VerifyChunk(chunk))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Chunk {chunk.ChunkIndex} verification failed. Expected hash: {chunk.ChunkHash}");
|
||||
}
|
||||
}
|
||||
|
||||
// Reassemble
|
||||
var totalSize = orderedChunks.Sum(c => c.BlobSize);
|
||||
var result = new byte[totalSize];
|
||||
var offset = 0;
|
||||
|
||||
foreach (var chunk in orderedChunks)
|
||||
{
|
||||
chunk.Blob.CopyTo(result, offset);
|
||||
offset += chunk.BlobSize;
|
||||
}
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool VerifyChunk(EvidenceChunk chunk)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(chunk);
|
||||
|
||||
var computedHash = ComputeHash(chunk.Blob);
|
||||
return string.Equals(computedHash, chunk.ChunkHash, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string ComputeMerkleRoot(IEnumerable<string> chunkHashes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(chunkHashes);
|
||||
|
||||
var hashes = chunkHashes.ToList();
|
||||
|
||||
if (hashes.Count == 0)
|
||||
{
|
||||
// Empty Merkle tree
|
||||
return ComputeHash([]);
|
||||
}
|
||||
|
||||
if (hashes.Count == 1)
|
||||
{
|
||||
return hashes[0];
|
||||
}
|
||||
|
||||
// Build Merkle tree bottom-up
|
||||
var currentLevel = hashes.Select(h => HexToBytes(h)).ToList();
|
||||
|
||||
while (currentLevel.Count > 1)
|
||||
{
|
||||
var nextLevel = new List<byte[]>();
|
||||
|
||||
for (var i = 0; i < currentLevel.Count; i += 2)
|
||||
{
|
||||
byte[] combined;
|
||||
|
||||
if (i + 1 < currentLevel.Count)
|
||||
{
|
||||
// Pair exists - concatenate and hash
|
||||
combined = new byte[currentLevel[i].Length + currentLevel[i + 1].Length];
|
||||
currentLevel[i].CopyTo(combined, 0);
|
||||
currentLevel[i + 1].CopyTo(combined, currentLevel[i].Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Odd node - duplicate itself
|
||||
combined = new byte[currentLevel[i].Length * 2];
|
||||
currentLevel[i].CopyTo(combined, 0);
|
||||
currentLevel[i].CopyTo(combined, currentLevel[i].Length);
|
||||
}
|
||||
|
||||
nextLevel.Add(SHA256.HashData(combined));
|
||||
}
|
||||
|
||||
currentLevel = nextLevel;
|
||||
}
|
||||
|
||||
return $"sha256:{Convert.ToHexStringLower(currentLevel[0])}";
|
||||
}
|
||||
|
||||
private static string ComputeHash(ReadOnlySpan<byte> data)
|
||||
{
|
||||
var hash = SHA256.HashData(data);
|
||||
return $"sha256:{Convert.ToHexStringLower(hash)}";
|
||||
}
|
||||
|
||||
private static byte[] HexToBytes(string hash)
|
||||
{
|
||||
// Strip sha256: prefix if present
|
||||
var hex = hash.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
|
||||
? hash[7..]
|
||||
: hash;
|
||||
|
||||
return Convert.FromHexString(hex);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user