using StellaOps.Cryptography.Digests; using StellaOps.Scheduler.Models; using System.Collections.Immutable; using System.ComponentModel.DataAnnotations; namespace StellaOps.Scheduler.WebService.GraphJobs; internal sealed class GraphJobService : IGraphJobService { private readonly IGraphJobStore _store; private readonly TimeProvider _timeProvider; private readonly IGraphJobCompletionPublisher _completionPublisher; private readonly ICartographerWebhookClient _cartographerWebhook; public GraphJobService( IGraphJobStore store, TimeProvider timeProvider, IGraphJobCompletionPublisher completionPublisher, ICartographerWebhookClient cartographerWebhook) { _store = store ?? throw new ArgumentNullException(nameof(store)); _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); _completionPublisher = completionPublisher ?? throw new ArgumentNullException(nameof(completionPublisher)); _cartographerWebhook = cartographerWebhook ?? throw new ArgumentNullException(nameof(cartographerWebhook)); } public async Task CreateBuildJobAsync(string tenantId, GraphBuildJobRequest request, CancellationToken cancellationToken) { Validate(request); var trigger = request.Trigger ?? GraphBuildJobTrigger.SbomVersion; var metadata = request.Metadata ?? new Dictionary(StringComparer.Ordinal); var now = _timeProvider.GetUtcNow(); var id = GenerateIdentifier("gbj"); var job = new GraphBuildJob( id, tenantId, request.SbomId.Trim(), request.SbomVersionId.Trim(), NormalizeDigest(request.SbomDigest), GraphJobStatus.Pending, trigger, now, request.GraphSnapshotId, attempts: 0, cartographerJobId: null, correlationId: request.CorrelationId?.Trim(), startedAt: null, completedAt: null, error: null, metadata: metadata.Select(pair => new KeyValuePair(pair.Key, pair.Value))); return await _store.AddAsync(job, cancellationToken); } public async Task CreateOverlayJobAsync(string tenantId, GraphOverlayJobRequest request, CancellationToken cancellationToken) { Validate(request); var subjects = (request.Subjects ?? Array.Empty()) .Where(subject => !string.IsNullOrWhiteSpace(subject)) .Select(subject => subject.Trim()) .ToArray(); var metadata = request.Metadata ?? new Dictionary(StringComparer.Ordinal); var trigger = request.Trigger ?? GraphOverlayJobTrigger.Policy; var now = _timeProvider.GetUtcNow(); var id = GenerateIdentifier("goj"); var job = new GraphOverlayJob( id: id, tenantId: tenantId, graphSnapshotId: request.GraphSnapshotId.Trim(), overlayKind: request.OverlayKind, overlayKey: request.OverlayKey.Trim(), status: GraphJobStatus.Pending, trigger: trigger, createdAt: now, subjects: subjects, attempts: 0, buildJobId: request.BuildJobId?.Trim(), correlationId: request.CorrelationId?.Trim(), metadata: metadata.Select(pair => new KeyValuePair(pair.Key, pair.Value))); return await _store.AddAsync(job, cancellationToken); } public async Task GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken) { return await _store.GetJobsAsync(tenantId, query, cancellationToken); } public async Task CompleteJobAsync(string tenantId, GraphJobCompletionRequest request, CancellationToken cancellationToken) { if (request.Status is not (GraphJobStatus.Completed or GraphJobStatus.Failed or GraphJobStatus.Cancelled)) { throw new ValidationException("Completion requires status completed, failed, or cancelled."); } var occurredAt = request.OccurredAt == default ? _timeProvider.GetUtcNow() : request.OccurredAt.ToUniversalTime(); var graphSnapshotId = Normalize(request.GraphSnapshotId); var correlationId = Normalize(request.CorrelationId); var resultUri = Normalize(request.ResultUri); var error = request.Status == GraphJobStatus.Failed ? Normalize(request.Error) : null; switch (request.JobType) { case GraphJobQueryType.Build: { var existing = await _store.GetBuildJobAsync(tenantId, request.JobId, cancellationToken).ConfigureAwait(false); if (existing is null) { throw new KeyNotFoundException($"Graph build job '{request.JobId}' not found."); } return await CompleteBuildJobInternal( tenantId, existing, request.Status, occurredAt, graphSnapshotId, correlationId, resultUri, error, cancellationToken).ConfigureAwait(false); } case GraphJobQueryType.Overlay: { var existing = await _store.GetOverlayJobAsync(tenantId, request.JobId, cancellationToken).ConfigureAwait(false); if (existing is null) { throw new KeyNotFoundException($"Graph overlay job '{request.JobId}' not found."); } return await CompleteOverlayJobInternal( tenantId, existing, request.Status, occurredAt, graphSnapshotId, correlationId, resultUri, error, cancellationToken).ConfigureAwait(false); } default: throw new ValidationException("Unsupported job type."); } } private async Task CompleteBuildJobInternal( string tenantId, GraphBuildJob current, GraphJobStatus requestedStatus, DateTimeOffset occurredAt, string? graphSnapshotId, string? correlationId, string? resultUri, string? error, CancellationToken cancellationToken) { var latest = current; for (var attempt = 0; attempt < 3; attempt++) { var transition = PrepareBuildTransition(latest, requestedStatus, occurredAt, graphSnapshotId, correlationId, resultUri, error); if (!transition.HasChanges) { return GraphJobResponse.From(latest); } var updateResult = await _store.UpdateAsync(transition.Job, transition.ExpectedStatus, cancellationToken).ConfigureAwait(false); if (updateResult.Updated) { var stored = updateResult.Job; var response = GraphJobResponse.From(stored); if (transition.ShouldPublish) { await PublishCompletionAsync( tenantId, GraphJobQueryType.Build, stored.Status, occurredAt, response, ExtractResultUri(response), stored.CorrelationId, stored.Error, cancellationToken).ConfigureAwait(false); } return response; } latest = updateResult.Job; } return GraphJobResponse.From(latest); } private async Task CompleteOverlayJobInternal( string tenantId, GraphOverlayJob current, GraphJobStatus requestedStatus, DateTimeOffset occurredAt, string? graphSnapshotId, string? correlationId, string? resultUri, string? error, CancellationToken cancellationToken) { var latest = current; for (var attempt = 0; attempt < 3; attempt++) { var transition = PrepareOverlayTransition(latest, requestedStatus, occurredAt, graphSnapshotId, correlationId, resultUri, error); if (!transition.HasChanges) { return GraphJobResponse.From(latest); } var updateResult = await _store.UpdateAsync(transition.Job, transition.ExpectedStatus, cancellationToken).ConfigureAwait(false); if (updateResult.Updated) { var stored = updateResult.Job; var response = GraphJobResponse.From(stored); if (transition.ShouldPublish) { await PublishCompletionAsync( tenantId, GraphJobQueryType.Overlay, stored.Status, occurredAt, response, ExtractResultUri(response), stored.CorrelationId, stored.Error, cancellationToken).ConfigureAwait(false); } return response; } latest = updateResult.Job; } return GraphJobResponse.From(latest); } private static CompletionTransition PrepareBuildTransition( GraphBuildJob current, GraphJobStatus requestedStatus, DateTimeOffset occurredAt, string? graphSnapshotId, string? correlationId, string? resultUri, string? error) { var transitional = current; if (transitional.Status is GraphJobStatus.Pending or GraphJobStatus.Queued) { transitional = GraphJobStateMachine.EnsureTransition(transitional, GraphJobStatus.Running, occurredAt, attempts: transitional.Attempts); } var desiredAttempts = transitional.Status == requestedStatus ? transitional.Attempts : transitional.Attempts + 1; var updated = GraphJobStateMachine.EnsureTransition(transitional, requestedStatus, occurredAt, attempts: desiredAttempts, errorMessage: error); var metadata = updated.Metadata; if (resultUri is { Length: > 0 }) { if (!metadata.TryGetValue("resultUri", out var existingValue) || !string.Equals(existingValue, resultUri, StringComparison.Ordinal)) { metadata = MergeMetadata(metadata, resultUri); } } var normalized = new GraphBuildJob( id: updated.Id, tenantId: updated.TenantId, sbomId: updated.SbomId, sbomVersionId: updated.SbomVersionId, sbomDigest: updated.SbomDigest, graphSnapshotId: graphSnapshotId ?? updated.GraphSnapshotId, status: updated.Status, trigger: updated.Trigger, attempts: updated.Attempts, cartographerJobId: updated.CartographerJobId, correlationId: correlationId ?? updated.CorrelationId, createdAt: updated.CreatedAt, startedAt: updated.StartedAt, completedAt: updated.CompletedAt, error: updated.Error, metadata: metadata, schemaVersion: updated.SchemaVersion); var hasChanges = !normalized.Equals(current); var shouldPublish = hasChanges && current.Status != normalized.Status; return new CompletionTransition(normalized, current.Status, hasChanges, shouldPublish); } private static CompletionTransition PrepareOverlayTransition( GraphOverlayJob current, GraphJobStatus requestedStatus, DateTimeOffset occurredAt, string? graphSnapshotId, string? correlationId, string? resultUri, string? error) { var transitional = current; if (transitional.Status is GraphJobStatus.Pending or GraphJobStatus.Queued) { transitional = GraphJobStateMachine.EnsureTransition(transitional, GraphJobStatus.Running, occurredAt, attempts: transitional.Attempts); } var desiredAttempts = transitional.Status == requestedStatus ? transitional.Attempts : transitional.Attempts + 1; var updated = GraphJobStateMachine.EnsureTransition(transitional, requestedStatus, occurredAt, attempts: desiredAttempts, errorMessage: error); var metadata = updated.Metadata; if (resultUri is { Length: > 0 }) { if (!metadata.TryGetValue("resultUri", out var existingValue) || !string.Equals(existingValue, resultUri, StringComparison.Ordinal)) { metadata = MergeMetadata(metadata, resultUri); } } var normalized = new GraphOverlayJob( id: updated.Id, tenantId: updated.TenantId, graphSnapshotId: graphSnapshotId ?? updated.GraphSnapshotId, buildJobId: updated.BuildJobId, overlayKind: updated.OverlayKind, overlayKey: updated.OverlayKey, subjects: updated.Subjects, status: updated.Status, trigger: updated.Trigger, attempts: updated.Attempts, correlationId: correlationId ?? updated.CorrelationId, createdAt: updated.CreatedAt, startedAt: updated.StartedAt, completedAt: updated.CompletedAt, error: updated.Error, metadata: metadata, schemaVersion: updated.SchemaVersion); var hasChanges = !normalized.Equals(current); var shouldPublish = hasChanges && current.Status != normalized.Status; return new CompletionTransition(normalized, current.Status, hasChanges, shouldPublish); } private static string? Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); private static string? ExtractResultUri(GraphJobResponse response) => response.Payload switch { GraphBuildJob build when build.Metadata.TryGetValue("resultUri", out var value) => value, GraphOverlayJob overlay when overlay.Metadata.TryGetValue("resultUri", out var value) => value, _ => null }; private sealed record CompletionTransition(TJob Job, GraphJobStatus ExpectedStatus, bool HasChanges, bool ShouldPublish) where TJob : class; public async Task GetOverlayLagMetricsAsync(string tenantId, CancellationToken cancellationToken) { var now = _timeProvider.GetUtcNow(); var overlayJobs = await _store.GetOverlayJobsAsync(tenantId, cancellationToken); var pending = overlayJobs.Count(job => job.Status == GraphJobStatus.Pending); var running = overlayJobs.Count(job => job.Status == GraphJobStatus.Running || job.Status == GraphJobStatus.Queued); var completed = overlayJobs.Count(job => job.Status == GraphJobStatus.Completed); var failed = overlayJobs.Count(job => job.Status == GraphJobStatus.Failed); var cancelled = overlayJobs.Count(job => job.Status == GraphJobStatus.Cancelled); var completedJobs = overlayJobs .Where(job => job.Status == GraphJobStatus.Completed && job.CompletedAt is not null) .OrderByDescending(job => job.CompletedAt) .ToArray(); double? minLag = null; double? maxLag = null; double? avgLag = null; List recent = new(); if (completedJobs.Length > 0) { var lags = completedJobs .Select(job => (now - job.CompletedAt!.Value).TotalSeconds) .ToArray(); minLag = lags.Min(); maxLag = lags.Max(); avgLag = lags.Average(); recent = completedJobs .Take(5) .Select(job => new OverlayLagEntry( JobId: job.Id, CompletedAt: job.CompletedAt!.Value, LagSeconds: (now - job.CompletedAt!.Value).TotalSeconds, CorrelationId: job.CorrelationId, ResultUri: job.Metadata.TryGetValue("resultUri", out var value) ? value : null)) .ToList(); } return new OverlayLagMetricsResponse( TenantId: tenantId, Pending: pending, Running: running, Completed: completed, Failed: failed, Cancelled: cancelled, MinLagSeconds: minLag, MaxLagSeconds: maxLag, AverageLagSeconds: avgLag, RecentCompleted: recent); } private static void Validate(GraphBuildJobRequest request) { if (string.IsNullOrWhiteSpace(request.SbomId)) { throw new ValidationException("sbomId is required."); } if (string.IsNullOrWhiteSpace(request.SbomVersionId)) { throw new ValidationException("sbomVersionId is required."); } if (string.IsNullOrWhiteSpace(request.SbomDigest)) { throw new ValidationException("sbomDigest is required."); } } private static void Validate(GraphOverlayJobRequest request) { if (string.IsNullOrWhiteSpace(request.GraphSnapshotId)) { throw new ValidationException("graphSnapshotId is required."); } if (string.IsNullOrWhiteSpace(request.OverlayKey)) { throw new ValidationException("overlayKey is required."); } } private static string GenerateIdentifier(string prefix) => $"{prefix}_{Guid.CreateVersion7().ToString("n")}"; private static string NormalizeDigest(string value) { try { return Sha256Digest.Normalize(value, requirePrefix: true, parameterName: "sbomDigest"); } catch (Exception ex) when (ex is ArgumentException or FormatException) { throw new ValidationException(ex.Message); } } private static ImmutableSortedDictionary MergeMetadata(ImmutableSortedDictionary existing, string? resultUri) { if (string.IsNullOrWhiteSpace(resultUri)) { return existing; } var builder = existing.ToBuilder(); builder["resultUri"] = resultUri.Trim(); return builder.ToImmutableSortedDictionary(StringComparer.Ordinal); } private async Task PublishCompletionAsync( string tenantId, GraphJobQueryType jobType, GraphJobStatus status, DateTimeOffset occurredAt, GraphJobResponse response, string? resultUri, string? correlationId, string? error, CancellationToken cancellationToken) { var notification = new GraphJobCompletionNotification( tenantId, jobType, status, occurredAt, response, resultUri, correlationId, error); await _completionPublisher.PublishAsync(notification, cancellationToken).ConfigureAwait(false); await _cartographerWebhook.NotifyAsync(notification, cancellationToken).ConfigureAwait(false); } }