//
// Copyright (c) StellaOps. Licensed under AGPL-3.0-or-later.
//
using Microsoft.Extensions.Logging;
using StellaOps.HybridLogicalClock;
using StellaOps.Scheduler.Persistence;
using StellaOps.Scheduler.Persistence.Postgres.Repositories;
namespace StellaOps.Scheduler.Queue.Hlc;
///
/// Service for verifying the integrity of the scheduler chain.
///
public interface ISchedulerChainVerifier
{
///
/// Verifies the integrity of the scheduler chain within an HLC range.
///
/// Tenant identifier.
/// Start of the HLC range (inclusive, null for unbounded).
/// End of the HLC range (inclusive, null for unbounded).
/// Optional partition key to verify (null for all partitions).
/// Cancellation token.
/// Verification result.
Task VerifyAsync(
string tenantId,
HlcTimestamp? startHlc = null,
HlcTimestamp? endHlc = null,
string? partitionKey = null,
CancellationToken cancellationToken = default);
///
/// Verifies a single chain link.
///
/// Tenant identifier.
/// The job identifier to verify.
/// Cancellation token.
/// Verification result for the single entry.
Task VerifyEntryAsync(
string tenantId,
Guid jobId,
CancellationToken cancellationToken = default);
}
///
/// Result of chain verification.
///
/// Whether the chain is valid.
/// Number of entries checked.
/// List of verification issues found.
public readonly record struct ChainVerificationResult(
bool IsValid,
int EntriesChecked,
IReadOnlyList Issues);
///
/// A specific issue found during chain verification.
///
/// The job ID where the issue was found.
/// The HLC timestamp of the problematic entry.
/// Type of issue found.
/// Human-readable description of the issue.
public readonly record struct ChainVerificationIssue(
Guid JobId,
string THlc,
string IssueType,
string Description);
///
/// Implementation of scheduler chain verification.
///
public sealed class SchedulerChainVerifier : ISchedulerChainVerifier
{
private readonly ISchedulerLogRepository _logRepository;
private readonly ILogger _logger;
///
/// Creates a new chain verifier.
///
public SchedulerChainVerifier(
ISchedulerLogRepository logRepository,
ILogger logger)
{
_logRepository = logRepository ?? throw new ArgumentNullException(nameof(logRepository));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
///
public async Task VerifyAsync(
string tenantId,
HlcTimestamp? startHlc = null,
HlcTimestamp? endHlc = null,
string? partitionKey = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
var startT = startHlc?.ToSortableString();
var endT = endHlc?.ToSortableString();
var entries = await _logRepository.GetByHlcRangeAsync(
tenantId,
startT,
endT,
limit: 0, // No limit
partitionKey,
cancellationToken).ConfigureAwait(false);
if (entries.Count == 0)
{
_logger.LogDebug(
"No entries to verify in range [{Start}, {End}] for tenant {TenantId}",
startT ?? "(unbounded)",
endT ?? "(unbounded)",
tenantId);
return new ChainVerificationResult(IsValid: true, EntriesChecked: 0, Issues: []);
}
var issues = new List();
byte[]? expectedPrevLink = null;
// If starting mid-chain, we need to get the previous entry's link
if (startHlc is not null)
{
var previousEntries = await _logRepository.GetByHlcRangeAsync(
tenantId,
startTHlc: null,
startT,
limit: 1,
partitionKey,
cancellationToken).ConfigureAwait(false);
if (previousEntries.Count > 0 && previousEntries[0].THlc != startT)
{
expectedPrevLink = previousEntries[0].Link;
}
}
foreach (var entry in entries)
{
// Verify prev_link matches expected
if (!ByteArrayEquals(entry.PrevLink, expectedPrevLink))
{
issues.Add(new ChainVerificationIssue(
entry.JobId,
entry.THlc,
"PrevLinkMismatch",
$"Expected {ToHex(expectedPrevLink)}, got {ToHex(entry.PrevLink)}"));
}
// Recompute link and verify
var computed = SchedulerChainLinking.ComputeLink(
entry.PrevLink,
entry.JobId,
HlcTimestamp.Parse(entry.THlc),
entry.PayloadHash);
if (!ByteArrayEquals(entry.Link, computed))
{
issues.Add(new ChainVerificationIssue(
entry.JobId,
entry.THlc,
"LinkMismatch",
$"Stored link doesn't match computed. Stored={ToHex(entry.Link)}, Computed={ToHex(computed)}"));
}
expectedPrevLink = entry.Link;
}
var isValid = issues.Count == 0;
_logger.LogInformation(
"Chain verification complete. TenantId={TenantId}, Range=[{Start}, {End}], EntriesChecked={Count}, IsValid={IsValid}, IssueCount={IssueCount}",
tenantId,
startT ?? "(unbounded)",
endT ?? "(unbounded)",
entries.Count,
isValid,
issues.Count);
return new ChainVerificationResult(isValid, entries.Count, issues);
}
///
public async Task VerifyEntryAsync(
string tenantId,
Guid jobId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
var entry = await _logRepository.GetByJobIdAsync(jobId, cancellationToken).ConfigureAwait(false);
if (entry is null)
{
return new ChainVerificationResult(
IsValid: false,
EntriesChecked: 0,
Issues: [new ChainVerificationIssue(jobId, string.Empty, "NotFound", "Entry not found")]);
}
// Verify tenant isolation
if (!string.Equals(entry.TenantId, tenantId, StringComparison.Ordinal))
{
return new ChainVerificationResult(
IsValid: false,
EntriesChecked: 0,
Issues: [new ChainVerificationIssue(jobId, entry.THlc, "TenantMismatch", "Entry belongs to different tenant")]);
}
var issues = new List();
// Recompute link and verify
var computed = SchedulerChainLinking.ComputeLink(
entry.PrevLink,
entry.JobId,
HlcTimestamp.Parse(entry.THlc),
entry.PayloadHash);
if (!ByteArrayEquals(entry.Link, computed))
{
issues.Add(new ChainVerificationIssue(
entry.JobId,
entry.THlc,
"LinkMismatch",
$"Stored link doesn't match computed"));
}
// If there's a prev_link, verify it exists and matches
if (entry.PrevLink is { Length: > 0 })
{
// Find the previous entry
var allEntries = await _logRepository.GetByHlcRangeAsync(
tenantId,
startTHlc: null,
entry.THlc,
limit: 0,
partitionKey: entry.PartitionKey,
cancellationToken).ConfigureAwait(false);
var prevEntry = allEntries
.Where(e => e.THlc != entry.THlc)
.OrderByDescending(e => e.THlc)
.FirstOrDefault();
if (prevEntry is null)
{
issues.Add(new ChainVerificationIssue(
entry.JobId,
entry.THlc,
"PrevEntryNotFound",
"Entry has prev_link but no previous entry found"));
}
else if (!ByteArrayEquals(prevEntry.Link, entry.PrevLink))
{
issues.Add(new ChainVerificationIssue(
entry.JobId,
entry.THlc,
"PrevLinkMismatch",
$"prev_link doesn't match previous entry's link"));
}
}
return new ChainVerificationResult(issues.Count == 0, 1, issues);
}
private static bool ByteArrayEquals(byte[]? a, byte[]? b)
{
if (a is null && b is null)
{
return true;
}
if (a is null || b is null)
{
return false;
}
if (a.Length == 0 && b.Length == 0)
{
return true;
}
return a.AsSpan().SequenceEqual(b);
}
private static string ToHex(byte[]? bytes)
{
return bytes is null ? "(null)" : Convert.ToHexString(bytes);
}
}