509 lines
19 KiB
C#
509 lines
19 KiB
C#
|
|
using StellaOps.Cryptography.Digests;
|
|
using StellaOps.Scheduler.Models;
|
|
using System.Collections.Immutable;
|
|
using System.ComponentModel.DataAnnotations;
|
|
|
|
namespace StellaOps.Scheduler.WebService.GraphJobs;
|
|
|
|
internal sealed class GraphJobService : IGraphJobService
|
|
{
|
|
private readonly IGraphJobStore _store;
|
|
private readonly TimeProvider _timeProvider;
|
|
private readonly IGraphJobCompletionPublisher _completionPublisher;
|
|
private readonly ICartographerWebhookClient _cartographerWebhook;
|
|
|
|
public GraphJobService(
|
|
IGraphJobStore store,
|
|
TimeProvider timeProvider,
|
|
IGraphJobCompletionPublisher completionPublisher,
|
|
ICartographerWebhookClient cartographerWebhook)
|
|
{
|
|
_store = store ?? throw new ArgumentNullException(nameof(store));
|
|
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
|
_completionPublisher = completionPublisher ?? throw new ArgumentNullException(nameof(completionPublisher));
|
|
_cartographerWebhook = cartographerWebhook ?? throw new ArgumentNullException(nameof(cartographerWebhook));
|
|
}
|
|
|
|
public async Task<GraphBuildJob> CreateBuildJobAsync(string tenantId, GraphBuildJobRequest request, CancellationToken cancellationToken)
|
|
{
|
|
Validate(request);
|
|
|
|
var trigger = request.Trigger ?? GraphBuildJobTrigger.SbomVersion;
|
|
var metadata = request.Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal);
|
|
|
|
var now = _timeProvider.GetUtcNow();
|
|
var id = GenerateIdentifier("gbj");
|
|
var job = new GraphBuildJob(
|
|
id,
|
|
tenantId,
|
|
request.SbomId.Trim(),
|
|
request.SbomVersionId.Trim(),
|
|
NormalizeDigest(request.SbomDigest),
|
|
GraphJobStatus.Pending,
|
|
trigger,
|
|
now,
|
|
request.GraphSnapshotId,
|
|
attempts: 0,
|
|
cartographerJobId: null,
|
|
correlationId: request.CorrelationId?.Trim(),
|
|
startedAt: null,
|
|
completedAt: null,
|
|
error: null,
|
|
metadata: metadata.Select(pair => new KeyValuePair<string, string>(pair.Key, pair.Value)));
|
|
|
|
return await _store.AddAsync(job, cancellationToken);
|
|
}
|
|
|
|
public async Task<GraphOverlayJob> CreateOverlayJobAsync(string tenantId, GraphOverlayJobRequest request, CancellationToken cancellationToken)
|
|
{
|
|
Validate(request);
|
|
|
|
var subjects = (request.Subjects ?? Array.Empty<string>())
|
|
.Where(subject => !string.IsNullOrWhiteSpace(subject))
|
|
.Select(subject => subject.Trim())
|
|
.ToArray();
|
|
var metadata = request.Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal);
|
|
var trigger = request.Trigger ?? GraphOverlayJobTrigger.Policy;
|
|
|
|
var now = _timeProvider.GetUtcNow();
|
|
var id = GenerateIdentifier("goj");
|
|
|
|
var job = new GraphOverlayJob(
|
|
id: id,
|
|
tenantId: tenantId,
|
|
graphSnapshotId: request.GraphSnapshotId.Trim(),
|
|
overlayKind: request.OverlayKind,
|
|
overlayKey: request.OverlayKey.Trim(),
|
|
status: GraphJobStatus.Pending,
|
|
trigger: trigger,
|
|
createdAt: now,
|
|
subjects: subjects,
|
|
attempts: 0,
|
|
buildJobId: request.BuildJobId?.Trim(),
|
|
correlationId: request.CorrelationId?.Trim(),
|
|
metadata: metadata.Select(pair => new KeyValuePair<string, string>(pair.Key, pair.Value)));
|
|
|
|
return await _store.AddAsync(job, cancellationToken);
|
|
}
|
|
|
|
public async Task<GraphJobCollection> GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken)
|
|
{
|
|
return await _store.GetJobsAsync(tenantId, query, cancellationToken);
|
|
}
|
|
|
|
public async Task<GraphJobResponse> CompleteJobAsync(string tenantId, GraphJobCompletionRequest request, CancellationToken cancellationToken)
|
|
{
|
|
if (request.Status is not (GraphJobStatus.Completed or GraphJobStatus.Failed or GraphJobStatus.Cancelled))
|
|
{
|
|
throw new ValidationException("Completion requires status completed, failed, or cancelled.");
|
|
}
|
|
|
|
var occurredAt = request.OccurredAt == default ? _timeProvider.GetUtcNow() : request.OccurredAt.ToUniversalTime();
|
|
var graphSnapshotId = Normalize(request.GraphSnapshotId);
|
|
var correlationId = Normalize(request.CorrelationId);
|
|
var resultUri = Normalize(request.ResultUri);
|
|
var error = request.Status == GraphJobStatus.Failed ? Normalize(request.Error) : null;
|
|
|
|
switch (request.JobType)
|
|
{
|
|
case GraphJobQueryType.Build:
|
|
{
|
|
var existing = await _store.GetBuildJobAsync(tenantId, request.JobId, cancellationToken).ConfigureAwait(false);
|
|
if (existing is null)
|
|
{
|
|
throw new KeyNotFoundException($"Graph build job '{request.JobId}' not found.");
|
|
}
|
|
|
|
return await CompleteBuildJobInternal(
|
|
tenantId,
|
|
existing,
|
|
request.Status,
|
|
occurredAt,
|
|
graphSnapshotId,
|
|
correlationId,
|
|
resultUri,
|
|
error,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
case GraphJobQueryType.Overlay:
|
|
{
|
|
var existing = await _store.GetOverlayJobAsync(tenantId, request.JobId, cancellationToken).ConfigureAwait(false);
|
|
if (existing is null)
|
|
{
|
|
throw new KeyNotFoundException($"Graph overlay job '{request.JobId}' not found.");
|
|
}
|
|
|
|
return await CompleteOverlayJobInternal(
|
|
tenantId,
|
|
existing,
|
|
request.Status,
|
|
occurredAt,
|
|
graphSnapshotId,
|
|
correlationId,
|
|
resultUri,
|
|
error,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
default:
|
|
throw new ValidationException("Unsupported job type.");
|
|
}
|
|
}
|
|
|
|
private async Task<GraphJobResponse> CompleteBuildJobInternal(
|
|
string tenantId,
|
|
GraphBuildJob current,
|
|
GraphJobStatus requestedStatus,
|
|
DateTimeOffset occurredAt,
|
|
string? graphSnapshotId,
|
|
string? correlationId,
|
|
string? resultUri,
|
|
string? error,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var latest = current;
|
|
|
|
for (var attempt = 0; attempt < 3; attempt++)
|
|
{
|
|
var transition = PrepareBuildTransition(latest, requestedStatus, occurredAt, graphSnapshotId, correlationId, resultUri, error);
|
|
if (!transition.HasChanges)
|
|
{
|
|
return GraphJobResponse.From(latest);
|
|
}
|
|
|
|
var updateResult = await _store.UpdateAsync(transition.Job, transition.ExpectedStatus, cancellationToken).ConfigureAwait(false);
|
|
if (updateResult.Updated)
|
|
{
|
|
var stored = updateResult.Job;
|
|
var response = GraphJobResponse.From(stored);
|
|
|
|
if (transition.ShouldPublish)
|
|
{
|
|
await PublishCompletionAsync(
|
|
tenantId,
|
|
GraphJobQueryType.Build,
|
|
stored.Status,
|
|
occurredAt,
|
|
response,
|
|
ExtractResultUri(response),
|
|
stored.CorrelationId,
|
|
stored.Error,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
return response;
|
|
}
|
|
|
|
latest = updateResult.Job;
|
|
}
|
|
|
|
return GraphJobResponse.From(latest);
|
|
}
|
|
|
|
private async Task<GraphJobResponse> CompleteOverlayJobInternal(
|
|
string tenantId,
|
|
GraphOverlayJob current,
|
|
GraphJobStatus requestedStatus,
|
|
DateTimeOffset occurredAt,
|
|
string? graphSnapshotId,
|
|
string? correlationId,
|
|
string? resultUri,
|
|
string? error,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var latest = current;
|
|
|
|
for (var attempt = 0; attempt < 3; attempt++)
|
|
{
|
|
var transition = PrepareOverlayTransition(latest, requestedStatus, occurredAt, graphSnapshotId, correlationId, resultUri, error);
|
|
if (!transition.HasChanges)
|
|
{
|
|
return GraphJobResponse.From(latest);
|
|
}
|
|
|
|
var updateResult = await _store.UpdateAsync(transition.Job, transition.ExpectedStatus, cancellationToken).ConfigureAwait(false);
|
|
if (updateResult.Updated)
|
|
{
|
|
var stored = updateResult.Job;
|
|
var response = GraphJobResponse.From(stored);
|
|
|
|
if (transition.ShouldPublish)
|
|
{
|
|
await PublishCompletionAsync(
|
|
tenantId,
|
|
GraphJobQueryType.Overlay,
|
|
stored.Status,
|
|
occurredAt,
|
|
response,
|
|
ExtractResultUri(response),
|
|
stored.CorrelationId,
|
|
stored.Error,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
return response;
|
|
}
|
|
|
|
latest = updateResult.Job;
|
|
}
|
|
|
|
return GraphJobResponse.From(latest);
|
|
}
|
|
|
|
private static CompletionTransition<GraphBuildJob> PrepareBuildTransition(
|
|
GraphBuildJob current,
|
|
GraphJobStatus requestedStatus,
|
|
DateTimeOffset occurredAt,
|
|
string? graphSnapshotId,
|
|
string? correlationId,
|
|
string? resultUri,
|
|
string? error)
|
|
{
|
|
var transitional = current;
|
|
if (transitional.Status is GraphJobStatus.Pending or GraphJobStatus.Queued)
|
|
{
|
|
transitional = GraphJobStateMachine.EnsureTransition(transitional, GraphJobStatus.Running, occurredAt, attempts: transitional.Attempts);
|
|
}
|
|
|
|
var desiredAttempts = transitional.Status == requestedStatus ? transitional.Attempts : transitional.Attempts + 1;
|
|
var updated = GraphJobStateMachine.EnsureTransition(transitional, requestedStatus, occurredAt, attempts: desiredAttempts, errorMessage: error);
|
|
|
|
var metadata = updated.Metadata;
|
|
if (resultUri is { Length: > 0 })
|
|
{
|
|
if (!metadata.TryGetValue("resultUri", out var existingValue) || !string.Equals(existingValue, resultUri, StringComparison.Ordinal))
|
|
{
|
|
metadata = MergeMetadata(metadata, resultUri);
|
|
}
|
|
}
|
|
|
|
var normalized = new GraphBuildJob(
|
|
id: updated.Id,
|
|
tenantId: updated.TenantId,
|
|
sbomId: updated.SbomId,
|
|
sbomVersionId: updated.SbomVersionId,
|
|
sbomDigest: updated.SbomDigest,
|
|
graphSnapshotId: graphSnapshotId ?? updated.GraphSnapshotId,
|
|
status: updated.Status,
|
|
trigger: updated.Trigger,
|
|
attempts: updated.Attempts,
|
|
cartographerJobId: updated.CartographerJobId,
|
|
correlationId: correlationId ?? updated.CorrelationId,
|
|
createdAt: updated.CreatedAt,
|
|
startedAt: updated.StartedAt,
|
|
completedAt: updated.CompletedAt,
|
|
error: updated.Error,
|
|
metadata: metadata,
|
|
schemaVersion: updated.SchemaVersion);
|
|
|
|
var hasChanges = !normalized.Equals(current);
|
|
var shouldPublish = hasChanges && current.Status != normalized.Status;
|
|
return new CompletionTransition<GraphBuildJob>(normalized, current.Status, hasChanges, shouldPublish);
|
|
}
|
|
|
|
private static CompletionTransition<GraphOverlayJob> PrepareOverlayTransition(
|
|
GraphOverlayJob current,
|
|
GraphJobStatus requestedStatus,
|
|
DateTimeOffset occurredAt,
|
|
string? graphSnapshotId,
|
|
string? correlationId,
|
|
string? resultUri,
|
|
string? error)
|
|
{
|
|
var transitional = current;
|
|
if (transitional.Status is GraphJobStatus.Pending or GraphJobStatus.Queued)
|
|
{
|
|
transitional = GraphJobStateMachine.EnsureTransition(transitional, GraphJobStatus.Running, occurredAt, attempts: transitional.Attempts);
|
|
}
|
|
|
|
var desiredAttempts = transitional.Status == requestedStatus ? transitional.Attempts : transitional.Attempts + 1;
|
|
var updated = GraphJobStateMachine.EnsureTransition(transitional, requestedStatus, occurredAt, attempts: desiredAttempts, errorMessage: error);
|
|
|
|
var metadata = updated.Metadata;
|
|
if (resultUri is { Length: > 0 })
|
|
{
|
|
if (!metadata.TryGetValue("resultUri", out var existingValue) || !string.Equals(existingValue, resultUri, StringComparison.Ordinal))
|
|
{
|
|
metadata = MergeMetadata(metadata, resultUri);
|
|
}
|
|
}
|
|
|
|
var normalized = new GraphOverlayJob(
|
|
id: updated.Id,
|
|
tenantId: updated.TenantId,
|
|
graphSnapshotId: graphSnapshotId ?? updated.GraphSnapshotId,
|
|
buildJobId: updated.BuildJobId,
|
|
overlayKind: updated.OverlayKind,
|
|
overlayKey: updated.OverlayKey,
|
|
subjects: updated.Subjects,
|
|
status: updated.Status,
|
|
trigger: updated.Trigger,
|
|
attempts: updated.Attempts,
|
|
correlationId: correlationId ?? updated.CorrelationId,
|
|
createdAt: updated.CreatedAt,
|
|
startedAt: updated.StartedAt,
|
|
completedAt: updated.CompletedAt,
|
|
error: updated.Error,
|
|
metadata: metadata,
|
|
schemaVersion: updated.SchemaVersion);
|
|
|
|
var hasChanges = !normalized.Equals(current);
|
|
var shouldPublish = hasChanges && current.Status != normalized.Status;
|
|
return new CompletionTransition<GraphOverlayJob>(normalized, current.Status, hasChanges, shouldPublish);
|
|
}
|
|
|
|
private static string? Normalize(string? value)
|
|
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
|
|
|
private static string? ExtractResultUri(GraphJobResponse response)
|
|
=> response.Payload switch
|
|
{
|
|
GraphBuildJob build when build.Metadata.TryGetValue("resultUri", out var value) => value,
|
|
GraphOverlayJob overlay when overlay.Metadata.TryGetValue("resultUri", out var value) => value,
|
|
_ => null
|
|
};
|
|
|
|
private sealed record CompletionTransition<TJob>(TJob Job, GraphJobStatus ExpectedStatus, bool HasChanges, bool ShouldPublish)
|
|
where TJob : class;
|
|
|
|
public async Task<OverlayLagMetricsResponse> GetOverlayLagMetricsAsync(string tenantId, CancellationToken cancellationToken)
|
|
{
|
|
var now = _timeProvider.GetUtcNow();
|
|
var overlayJobs = await _store.GetOverlayJobsAsync(tenantId, cancellationToken);
|
|
|
|
var pending = overlayJobs.Count(job => job.Status == GraphJobStatus.Pending);
|
|
var running = overlayJobs.Count(job => job.Status == GraphJobStatus.Running || job.Status == GraphJobStatus.Queued);
|
|
var completed = overlayJobs.Count(job => job.Status == GraphJobStatus.Completed);
|
|
var failed = overlayJobs.Count(job => job.Status == GraphJobStatus.Failed);
|
|
var cancelled = overlayJobs.Count(job => job.Status == GraphJobStatus.Cancelled);
|
|
|
|
var completedJobs = overlayJobs
|
|
.Where(job => job.Status == GraphJobStatus.Completed && job.CompletedAt is not null)
|
|
.OrderByDescending(job => job.CompletedAt)
|
|
.ToArray();
|
|
|
|
double? minLag = null;
|
|
double? maxLag = null;
|
|
double? avgLag = null;
|
|
List<OverlayLagEntry> recent = new();
|
|
|
|
if (completedJobs.Length > 0)
|
|
{
|
|
var lags = completedJobs
|
|
.Select(job => (now - job.CompletedAt!.Value).TotalSeconds)
|
|
.ToArray();
|
|
|
|
minLag = lags.Min();
|
|
maxLag = lags.Max();
|
|
avgLag = lags.Average();
|
|
|
|
recent = completedJobs
|
|
.Take(5)
|
|
.Select(job => new OverlayLagEntry(
|
|
JobId: job.Id,
|
|
CompletedAt: job.CompletedAt!.Value,
|
|
LagSeconds: (now - job.CompletedAt!.Value).TotalSeconds,
|
|
CorrelationId: job.CorrelationId,
|
|
ResultUri: job.Metadata.TryGetValue("resultUri", out var value) ? value : null))
|
|
.ToList();
|
|
}
|
|
|
|
return new OverlayLagMetricsResponse(
|
|
TenantId: tenantId,
|
|
Pending: pending,
|
|
Running: running,
|
|
Completed: completed,
|
|
Failed: failed,
|
|
Cancelled: cancelled,
|
|
MinLagSeconds: minLag,
|
|
MaxLagSeconds: maxLag,
|
|
AverageLagSeconds: avgLag,
|
|
RecentCompleted: recent);
|
|
}
|
|
|
|
private static void Validate(GraphBuildJobRequest request)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(request.SbomId))
|
|
{
|
|
throw new ValidationException("sbomId is required.");
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(request.SbomVersionId))
|
|
{
|
|
throw new ValidationException("sbomVersionId is required.");
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(request.SbomDigest))
|
|
{
|
|
throw new ValidationException("sbomDigest is required.");
|
|
}
|
|
}
|
|
|
|
private static void Validate(GraphOverlayJobRequest request)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(request.GraphSnapshotId))
|
|
{
|
|
throw new ValidationException("graphSnapshotId is required.");
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(request.OverlayKey))
|
|
{
|
|
throw new ValidationException("overlayKey is required.");
|
|
}
|
|
}
|
|
|
|
private static string GenerateIdentifier(string prefix)
|
|
=> $"{prefix}_{Guid.CreateVersion7().ToString("n")}";
|
|
|
|
private static string NormalizeDigest(string value)
|
|
{
|
|
try
|
|
{
|
|
return Sha256Digest.Normalize(value, requirePrefix: true, parameterName: "sbomDigest");
|
|
}
|
|
catch (Exception ex) when (ex is ArgumentException or FormatException)
|
|
{
|
|
throw new ValidationException(ex.Message);
|
|
}
|
|
}
|
|
|
|
private static ImmutableSortedDictionary<string, string> MergeMetadata(ImmutableSortedDictionary<string, string> existing, string? resultUri)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(resultUri))
|
|
{
|
|
return existing;
|
|
}
|
|
|
|
var builder = existing.ToBuilder();
|
|
builder["resultUri"] = resultUri.Trim();
|
|
return builder.ToImmutableSortedDictionary(StringComparer.Ordinal);
|
|
}
|
|
|
|
private async Task PublishCompletionAsync(
|
|
string tenantId,
|
|
GraphJobQueryType jobType,
|
|
GraphJobStatus status,
|
|
DateTimeOffset occurredAt,
|
|
GraphJobResponse response,
|
|
string? resultUri,
|
|
string? correlationId,
|
|
string? error,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var notification = new GraphJobCompletionNotification(
|
|
tenantId,
|
|
jobType,
|
|
status,
|
|
occurredAt,
|
|
response,
|
|
resultUri,
|
|
correlationId,
|
|
error);
|
|
|
|
await _completionPublisher.PublishAsync(notification, cancellationToken).ConfigureAwait(false);
|
|
await _cartographerWebhook.NotifyAsync(notification, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|