// // Copyright (c) StellaOps. Licensed under AGPL-3.0-or-later. // using Microsoft.Extensions.Logging; using StellaOps.HybridLogicalClock; using StellaOps.Scheduler.Persistence; using StellaOps.Scheduler.Persistence.Postgres.Repositories; namespace StellaOps.Scheduler.Queue.Hlc; /// /// Service for verifying the integrity of the scheduler chain. /// public interface ISchedulerChainVerifier { /// /// Verifies the integrity of the scheduler chain within an HLC range. /// /// Tenant identifier. /// Start of the HLC range (inclusive, null for unbounded). /// End of the HLC range (inclusive, null for unbounded). /// Optional partition key to verify (null for all partitions). /// Cancellation token. /// Verification result. Task VerifyAsync( string tenantId, HlcTimestamp? startHlc = null, HlcTimestamp? endHlc = null, string? partitionKey = null, CancellationToken cancellationToken = default); /// /// Verifies a single chain link. /// /// Tenant identifier. /// The job identifier to verify. /// Cancellation token. /// Verification result for the single entry. Task VerifyEntryAsync( string tenantId, Guid jobId, CancellationToken cancellationToken = default); } /// /// Result of chain verification. /// /// Whether the chain is valid. /// Number of entries checked. /// List of verification issues found. public readonly record struct ChainVerificationResult( bool IsValid, int EntriesChecked, IReadOnlyList Issues); /// /// A specific issue found during chain verification. /// /// The job ID where the issue was found. /// The HLC timestamp of the problematic entry. /// Type of issue found. /// Human-readable description of the issue. public readonly record struct ChainVerificationIssue( Guid JobId, string THlc, string IssueType, string Description); /// /// Implementation of scheduler chain verification. /// public sealed class SchedulerChainVerifier : ISchedulerChainVerifier { private readonly ISchedulerLogRepository _logRepository; private readonly ILogger _logger; /// /// Creates a new chain verifier. /// public SchedulerChainVerifier( ISchedulerLogRepository logRepository, ILogger logger) { _logRepository = logRepository ?? throw new ArgumentNullException(nameof(logRepository)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// public async Task VerifyAsync( string tenantId, HlcTimestamp? startHlc = null, HlcTimestamp? endHlc = null, string? partitionKey = null, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(tenantId); var startT = startHlc?.ToSortableString(); var endT = endHlc?.ToSortableString(); var entries = await _logRepository.GetByHlcRangeAsync( tenantId, startT, endT, limit: 0, // No limit partitionKey, cancellationToken).ConfigureAwait(false); if (entries.Count == 0) { _logger.LogDebug( "No entries to verify in range [{Start}, {End}] for tenant {TenantId}", startT ?? "(unbounded)", endT ?? "(unbounded)", tenantId); return new ChainVerificationResult(IsValid: true, EntriesChecked: 0, Issues: []); } var issues = new List(); byte[]? expectedPrevLink = null; // If starting mid-chain, we need to get the previous entry's link if (startHlc is not null) { var previousEntries = await _logRepository.GetByHlcRangeAsync( tenantId, startTHlc: null, startT, limit: 1, partitionKey, cancellationToken).ConfigureAwait(false); if (previousEntries.Count > 0 && previousEntries[0].THlc != startT) { expectedPrevLink = previousEntries[0].Link; } } foreach (var entry in entries) { // Verify prev_link matches expected if (!ByteArrayEquals(entry.PrevLink, expectedPrevLink)) { issues.Add(new ChainVerificationIssue( entry.JobId, entry.THlc, "PrevLinkMismatch", $"Expected {ToHex(expectedPrevLink)}, got {ToHex(entry.PrevLink)}")); } // Recompute link and verify var computed = SchedulerChainLinking.ComputeLink( entry.PrevLink, entry.JobId, HlcTimestamp.Parse(entry.THlc), entry.PayloadHash); if (!ByteArrayEquals(entry.Link, computed)) { issues.Add(new ChainVerificationIssue( entry.JobId, entry.THlc, "LinkMismatch", $"Stored link doesn't match computed. Stored={ToHex(entry.Link)}, Computed={ToHex(computed)}")); } expectedPrevLink = entry.Link; } var isValid = issues.Count == 0; _logger.LogInformation( "Chain verification complete. TenantId={TenantId}, Range=[{Start}, {End}], EntriesChecked={Count}, IsValid={IsValid}, IssueCount={IssueCount}", tenantId, startT ?? "(unbounded)", endT ?? "(unbounded)", entries.Count, isValid, issues.Count); return new ChainVerificationResult(isValid, entries.Count, issues); } /// public async Task VerifyEntryAsync( string tenantId, Guid jobId, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(tenantId); var entry = await _logRepository.GetByJobIdAsync(jobId, cancellationToken).ConfigureAwait(false); if (entry is null) { return new ChainVerificationResult( IsValid: false, EntriesChecked: 0, Issues: [new ChainVerificationIssue(jobId, string.Empty, "NotFound", "Entry not found")]); } // Verify tenant isolation if (!string.Equals(entry.TenantId, tenantId, StringComparison.Ordinal)) { return new ChainVerificationResult( IsValid: false, EntriesChecked: 0, Issues: [new ChainVerificationIssue(jobId, entry.THlc, "TenantMismatch", "Entry belongs to different tenant")]); } var issues = new List(); // Recompute link and verify var computed = SchedulerChainLinking.ComputeLink( entry.PrevLink, entry.JobId, HlcTimestamp.Parse(entry.THlc), entry.PayloadHash); if (!ByteArrayEquals(entry.Link, computed)) { issues.Add(new ChainVerificationIssue( entry.JobId, entry.THlc, "LinkMismatch", $"Stored link doesn't match computed")); } // If there's a prev_link, verify it exists and matches if (entry.PrevLink is { Length: > 0 }) { // Find the previous entry var allEntries = await _logRepository.GetByHlcRangeAsync( tenantId, startTHlc: null, entry.THlc, limit: 0, partitionKey: entry.PartitionKey, cancellationToken).ConfigureAwait(false); var prevEntry = allEntries .Where(e => e.THlc != entry.THlc) .OrderByDescending(e => e.THlc) .FirstOrDefault(); if (prevEntry is null) { issues.Add(new ChainVerificationIssue( entry.JobId, entry.THlc, "PrevEntryNotFound", "Entry has prev_link but no previous entry found")); } else if (!ByteArrayEquals(prevEntry.Link, entry.PrevLink)) { issues.Add(new ChainVerificationIssue( entry.JobId, entry.THlc, "PrevLinkMismatch", $"prev_link doesn't match previous entry's link")); } } return new ChainVerificationResult(issues.Count == 0, 1, issues); } private static bool ByteArrayEquals(byte[]? a, byte[]? b) { if (a is null && b is null) { return true; } if (a is null || b is null) { return false; } if (a.Length == 0 && b.Length == 0) { return true; } return a.AsSpan().SequenceEqual(b); } private static string ToHex(byte[]? bytes) { return bytes is null ? "(null)" : Convert.ToHexString(bytes); } }