Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added MongoPackRunApprovalStore for managing approval states with MongoDB. - Introduced MongoPackRunArtifactUploader for uploading and storing artifacts. - Created MongoPackRunLogStore to handle logging of pack run events. - Developed MongoPackRunStateStore for persisting and retrieving pack run states. - Implemented unit tests for MongoDB stores to ensure correct functionality. - Added MongoTaskRunnerTestContext for setting up MongoDB test environment. - Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
This commit is contained in:
master
@@ -0,0 +1,31 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.TaskRunner.Core.Configuration;
|
||||
|
||||
public static class TaskRunnerStorageModes
|
||||
{
|
||||
public const string Filesystem = "filesystem";
|
||||
public const string Mongo = "mongo";
|
||||
}
|
||||
|
||||
public sealed class TaskRunnerStorageOptions
|
||||
{
|
||||
public string Mode { get; set; } = TaskRunnerStorageModes.Filesystem;
|
||||
|
||||
public TaskRunnerMongoOptions Mongo { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class TaskRunnerMongoOptions
|
||||
{
|
||||
public string ConnectionString { get; set; } = "mongodb://127.0.0.1:27017/stellaops-taskrunner";
|
||||
|
||||
public string? Database { get; set; }
|
||||
|
||||
public string RunsCollection { get; set; } = "pack_runs";
|
||||
|
||||
public string LogsCollection { get; set; } = "pack_run_logs";
|
||||
|
||||
public string ArtifactsCollection { get; set; } = "pack_artifacts";
|
||||
|
||||
public string ApprovalsCollection { get; set; } = "pack_run_approvals";
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
/// <summary>
|
||||
/// Persists pack run log entries in a deterministic append-only fashion.
|
||||
/// </summary>
|
||||
public interface IPackRunLogStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Appends a single log entry to the run log.
|
||||
/// </summary>
|
||||
Task AppendAsync(string runId, PackRunLogEntry entry, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the log entries for the specified run in chronological order.
|
||||
/// </summary>
|
||||
IAsyncEnumerable<PackRunLogEntry> ReadAsync(string runId, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether any log entries exist for the specified run.
|
||||
/// </summary>
|
||||
Task<bool> ExistsAsync(string runId, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single structured log entry emitted during a pack run.
|
||||
/// </summary>
|
||||
public sealed record PackRunLogEntry(
|
||||
DateTimeOffset Timestamp,
|
||||
string Level,
|
||||
string EventType,
|
||||
string Message,
|
||||
string? StepId,
|
||||
IReadOnlyDictionary<string, string>? Metadata);
|
||||
@@ -0,0 +1,116 @@
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
/// <summary>
|
||||
/// Builds deterministic <see cref="PackRunState"/> snapshots for freshly scheduled runs.
|
||||
/// </summary>
|
||||
public static class PackRunStateFactory
|
||||
{
|
||||
public static PackRunState CreateInitialState(
|
||||
PackRunExecutionContext context,
|
||||
PackRunExecutionGraph graph,
|
||||
PackRunSimulationEngine simulationEngine,
|
||||
DateTimeOffset timestamp)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(graph);
|
||||
ArgumentNullException.ThrowIfNull(simulationEngine);
|
||||
|
||||
var simulation = simulationEngine.Simulate(context.Plan);
|
||||
var simulationIndex = IndexSimulation(simulation.Steps);
|
||||
|
||||
var stepRecords = new Dictionary<string, PackRunStepStateRecord>(StringComparer.Ordinal);
|
||||
foreach (var step in EnumerateSteps(graph.Steps))
|
||||
{
|
||||
var simulationStatus = simulationIndex.TryGetValue(step.Id, out var node)
|
||||
? node.Status
|
||||
: PackRunSimulationStatus.Pending;
|
||||
|
||||
var status = step.Enabled ? PackRunStepExecutionStatus.Pending : PackRunStepExecutionStatus.Skipped;
|
||||
string? statusReason = null;
|
||||
|
||||
if (!step.Enabled)
|
||||
{
|
||||
statusReason = "disabled";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.RequiresApproval)
|
||||
{
|
||||
statusReason = "requires-approval";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.RequiresPolicy)
|
||||
{
|
||||
statusReason = "requires-policy";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.Skipped)
|
||||
{
|
||||
status = PackRunStepExecutionStatus.Skipped;
|
||||
statusReason = "condition-false";
|
||||
}
|
||||
|
||||
var record = new PackRunStepStateRecord(
|
||||
step.Id,
|
||||
step.Kind,
|
||||
step.Enabled,
|
||||
step.ContinueOnError,
|
||||
step.MaxParallel,
|
||||
step.ApprovalId,
|
||||
step.GateMessage,
|
||||
status,
|
||||
Attempts: 0,
|
||||
LastTransitionAt: null,
|
||||
NextAttemptAt: null,
|
||||
StatusReason: statusReason);
|
||||
|
||||
stepRecords[step.Id] = record;
|
||||
}
|
||||
|
||||
var failurePolicy = graph.FailurePolicy ?? PackRunExecutionGraph.DefaultFailurePolicy;
|
||||
|
||||
return PackRunState.Create(
|
||||
context.RunId,
|
||||
context.Plan.Hash,
|
||||
context.Plan,
|
||||
failurePolicy,
|
||||
context.RequestedAt,
|
||||
stepRecords,
|
||||
timestamp);
|
||||
}
|
||||
|
||||
private static Dictionary<string, PackRunSimulationNode> IndexSimulation(IReadOnlyList<PackRunSimulationNode> nodes)
|
||||
{
|
||||
var result = new Dictionary<string, PackRunSimulationNode>(StringComparer.Ordinal);
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
IndexSimulationNode(node, result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void IndexSimulationNode(PackRunSimulationNode node, Dictionary<string, PackRunSimulationNode> accumulator)
|
||||
{
|
||||
accumulator[node.Id] = node;
|
||||
foreach (var child in node.Children)
|
||||
{
|
||||
IndexSimulationNode(child, accumulator);
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<PackRunExecutionStep> EnumerateSteps(IReadOnlyList<PackRunExecutionStep> steps)
|
||||
{
|
||||
foreach (var step in steps)
|
||||
{
|
||||
yield return step;
|
||||
if (step.Children.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var child in EnumerateSteps(step.Children))
|
||||
{
|
||||
yield return child;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
/// <summary>
|
||||
/// Persists pack run logs as newline-delimited JSON for deterministic replay and offline mirroring.
|
||||
/// </summary>
|
||||
public sealed class FilePackRunLogStore : IPackRunLogStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly string rootPath;
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> fileLocks = new(StringComparer.Ordinal);
|
||||
|
||||
public FilePackRunLogStore(string rootPath)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
|
||||
|
||||
this.rootPath = Path.GetFullPath(rootPath);
|
||||
Directory.CreateDirectory(this.rootPath);
|
||||
}
|
||||
|
||||
public async Task AppendAsync(string runId, PackRunLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
var path = GetPath(runId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
|
||||
var gate = fileLocks.GetOrAdd(path, _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var document = PackRunLogEntryDocument.FromDomain(entry);
|
||||
var json = JsonSerializer.Serialize(document, SerializerOptions);
|
||||
await File.AppendAllTextAsync(path, json + Environment.NewLine, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<PackRunLogEntry> ReadAsync(
|
||||
string runId,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var path = GetPath(runId);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8);
|
||||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var line = await reader.ReadLineAsync().ConfigureAwait(false);
|
||||
if (line is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
PackRunLogEntryDocument? document = null;
|
||||
try
|
||||
{
|
||||
document = JsonSerializer.Deserialize<PackRunLogEntryDocument>(line, SerializerOptions);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Skip malformed entries to avoid stopping the stream; diagnostics are captured via worker logs.
|
||||
}
|
||||
|
||||
if (document is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return document.ToDomain();
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> ExistsAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
var path = GetPath(runId);
|
||||
return Task.FromResult(File.Exists(path));
|
||||
}
|
||||
|
||||
private string GetPath(string runId)
|
||||
{
|
||||
var safe = Sanitize(runId);
|
||||
return Path.Combine(rootPath, $"{safe}.ndjson");
|
||||
}
|
||||
|
||||
private static string Sanitize(string value)
|
||||
{
|
||||
var result = value.Trim();
|
||||
foreach (var invalid in Path.GetInvalidFileNameChars())
|
||||
{
|
||||
result = result.Replace(invalid, '_');
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(result) ? "run" : result;
|
||||
}
|
||||
|
||||
private sealed record PackRunLogEntryDocument(
|
||||
DateTimeOffset Timestamp,
|
||||
string Level,
|
||||
string EventType,
|
||||
string Message,
|
||||
string? StepId,
|
||||
Dictionary<string, string>? Metadata)
|
||||
{
|
||||
public static PackRunLogEntryDocument FromDomain(PackRunLogEntry entry)
|
||||
{
|
||||
var metadata = entry.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(entry.Metadata, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunLogEntryDocument(
|
||||
entry.Timestamp,
|
||||
entry.Level,
|
||||
entry.EventType,
|
||||
entry.Message,
|
||||
entry.StepId,
|
||||
metadata);
|
||||
}
|
||||
|
||||
public PackRunLogEntry ToDomain()
|
||||
{
|
||||
IReadOnlyDictionary<string, string>? metadata = Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(Metadata, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunLogEntry(
|
||||
Timestamp,
|
||||
Level,
|
||||
EventType,
|
||||
Message,
|
||||
StepId,
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunApprovalStore : IPackRunApprovalStore
|
||||
{
|
||||
private readonly IMongoCollection<PackRunApprovalDocument> collection;
|
||||
|
||||
public MongoPackRunApprovalStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunApprovalDocument>(options.ApprovalsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task SaveAsync(string runId, IReadOnlyList<PackRunApprovalState> approvals, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(approvals);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (approvals.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var documents = approvals
|
||||
.Select(approval => PackRunApprovalDocument.FromDomain(runId, approval))
|
||||
.ToList();
|
||||
|
||||
await collection.InsertManyAsync(documents, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunApprovalState>> GetAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.SortBy(document => document.ApprovalId)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(document => document.ToDomain())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public async Task UpdateAsync(string runId, PackRunApprovalState approval, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(approval);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.And(
|
||||
Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId),
|
||||
Builders<PackRunApprovalDocument>.Filter.Eq(document => document.ApprovalId, approval.ApprovalId));
|
||||
|
||||
var existingDocument = await collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (existingDocument is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Approval '{approval.ApprovalId}' not found for run '{runId}'.");
|
||||
}
|
||||
|
||||
var document = PackRunApprovalDocument.FromDomain(runId, approval, existingDocument.Id);
|
||||
await collection
|
||||
.ReplaceOneAsync(filter, document, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.ApprovalId),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status))
|
||||
};
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
}
|
||||
|
||||
private sealed class PackRunApprovalDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string ApprovalId { get; init; } = default!;
|
||||
|
||||
public IReadOnlyList<string> RequiredGrants { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> StepIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> Messages { get; init; } = Array.Empty<string>();
|
||||
|
||||
public string? ReasonTemplate { get; init; }
|
||||
|
||||
public DateTime RequestedAt { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public string? ActorId { get; init; }
|
||||
|
||||
public DateTime? CompletedAt { get; init; }
|
||||
|
||||
public string? Summary { get; init; }
|
||||
|
||||
public static PackRunApprovalDocument FromDomain(string runId, PackRunApprovalState approval, ObjectId? id = null)
|
||||
=> new()
|
||||
{
|
||||
Id = id ?? ObjectId.GenerateNewId(),
|
||||
RunId = runId,
|
||||
ApprovalId = approval.ApprovalId,
|
||||
RequiredGrants = approval.RequiredGrants ?? Array.Empty<string>(),
|
||||
StepIds = approval.StepIds ?? Array.Empty<string>(),
|
||||
Messages = approval.Messages ?? Array.Empty<string>(),
|
||||
ReasonTemplate = approval.ReasonTemplate,
|
||||
RequestedAt = approval.RequestedAt.UtcDateTime,
|
||||
Status = approval.Status.ToString(),
|
||||
ActorId = approval.ActorId,
|
||||
CompletedAt = approval.CompletedAt?.UtcDateTime,
|
||||
Summary = approval.Summary
|
||||
};
|
||||
|
||||
public PackRunApprovalState ToDomain()
|
||||
{
|
||||
var status = Enum.Parse<PackRunApprovalStatus>(Status, ignoreCase: true);
|
||||
|
||||
return new PackRunApprovalState(
|
||||
ApprovalId,
|
||||
RequiredGrants?.ToList() ?? new List<string>(),
|
||||
StepIds?.ToList() ?? new List<string>(),
|
||||
Messages?.ToList() ?? new List<string>(),
|
||||
ReasonTemplate,
|
||||
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
|
||||
status,
|
||||
ActorId,
|
||||
CompletedAt is null ? null : new DateTimeOffset(CompletedAt.Value, TimeSpan.Zero),
|
||||
Summary);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunArtifactUploader : IPackRunArtifactUploader
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<PackRunArtifactDocument> collection;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ILogger<MongoPackRunArtifactUploader> logger;
|
||||
|
||||
public MongoPackRunArtifactUploader(
|
||||
IMongoDatabase database,
|
||||
TaskRunnerMongoOptions options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<MongoPackRunArtifactUploader> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunArtifactDocument>(options.ArtifactsCollection);
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task UploadAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunState state,
|
||||
IReadOnlyList<TaskPackPlanOutput> outputs,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
ArgumentNullException.ThrowIfNull(outputs);
|
||||
|
||||
var filter = Builders<PackRunArtifactDocument>.Filter.Eq(document => document.RunId, context.RunId);
|
||||
await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (outputs.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var timestamp = timeProvider.GetUtcNow();
|
||||
var documents = new List<PackRunArtifactDocument>(outputs.Count);
|
||||
|
||||
foreach (var output in outputs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
documents.Add(ProcessOutput(context, output, timestamp));
|
||||
}
|
||||
|
||||
await collection.InsertManyAsync(documents, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private PackRunArtifactDocument ProcessOutput(
|
||||
PackRunExecutionContext context,
|
||||
TaskPackPlanOutput output,
|
||||
DateTimeOffset capturedAt)
|
||||
{
|
||||
var sourcePath = ResolveString(output.Path);
|
||||
var expressionNode = ResolveExpression(output.Expression);
|
||||
string status = "skipped";
|
||||
string? notes = null;
|
||||
string? storedPath = null;
|
||||
|
||||
if (IsFileOutput(output))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourcePath))
|
||||
{
|
||||
status = "unresolved";
|
||||
notes = "Output path requires runtime value.";
|
||||
}
|
||||
else if (!File.Exists(sourcePath))
|
||||
{
|
||||
status = "missing";
|
||||
notes = $"Source file '{sourcePath}' not found.";
|
||||
logger.LogWarning(
|
||||
"Pack run {RunId} output {Output} referenced missing file {Path}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
sourcePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
status = "referenced";
|
||||
storedPath = sourcePath;
|
||||
}
|
||||
}
|
||||
|
||||
BsonDocument? expressionDocument = null;
|
||||
if (expressionNode is not null)
|
||||
{
|
||||
var json = expressionNode.ToJsonString(SerializerOptions);
|
||||
expressionDocument = BsonDocument.Parse(json);
|
||||
status = status is "referenced" ? status : "materialized";
|
||||
}
|
||||
|
||||
return new PackRunArtifactDocument
|
||||
{
|
||||
Id = ObjectId.GenerateNewId(),
|
||||
RunId = context.RunId,
|
||||
Name = output.Name,
|
||||
Type = output.Type,
|
||||
SourcePath = sourcePath,
|
||||
StoredPath = storedPath,
|
||||
Status = status,
|
||||
Notes = notes,
|
||||
CapturedAt = capturedAt.UtcDateTime,
|
||||
Expression = expressionDocument
|
||||
};
|
||||
}
|
||||
|
||||
private static bool IsFileOutput(TaskPackPlanOutput output)
|
||||
=> string.Equals(output.Type, "file", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string? ResolveString(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue || parameter.Value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parameter.Value is JsonValue jsonValue && jsonValue.TryGetValue<string>(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static JsonNode? ResolveExpression(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return parameter.Value;
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Name),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status))
|
||||
};
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
}
|
||||
|
||||
public sealed class PackRunArtifactDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string Name { get; init; } = default!;
|
||||
|
||||
public string Type { get; init; } = default!;
|
||||
|
||||
public string? SourcePath { get; init; }
|
||||
|
||||
public string? StoredPath { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public string? Notes { get; init; }
|
||||
|
||||
public DateTime CapturedAt { get; init; }
|
||||
|
||||
public BsonDocument? Expression { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunLogStore : IPackRunLogStore
|
||||
{
|
||||
private readonly IMongoCollection<PackRunLogDocument> collection;
|
||||
|
||||
public MongoPackRunLogStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunLogDocument>(options.LogsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task AppendAsync(string runId, PackRunLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
for (var attempt = 0; attempt < 5; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var last = await collection
|
||||
.Find(filter)
|
||||
.SortByDescending(document => document.Sequence)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var nextSequence = last is null ? 1 : last.Sequence + 1;
|
||||
|
||||
var document = PackRunLogDocument.FromDomain(runId, nextSequence, entry);
|
||||
|
||||
try
|
||||
{
|
||||
await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Failed to append log entry for run '{runId}' after multiple attempts.");
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<PackRunLogEntry> ReadAsync(
|
||||
string runId,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
using var cursor = await collection
|
||||
.Find(filter)
|
||||
.SortBy(document => document.Sequence)
|
||||
.ToCursorAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var document in cursor.Current)
|
||||
{
|
||||
yield return document.ToDomain();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> ExistsAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
return await collection
|
||||
.Find(filter)
|
||||
.Limit(1)
|
||||
.AnyAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Sequence),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Timestamp))
|
||||
};
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
}
|
||||
|
||||
public sealed class PackRunLogDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public long Sequence { get; init; }
|
||||
|
||||
public DateTime Timestamp { get; init; }
|
||||
|
||||
public string Level { get; init; } = default!;
|
||||
|
||||
public string EventType { get; init; } = default!;
|
||||
|
||||
public string Message { get; init; } = default!;
|
||||
|
||||
public string? StepId { get; init; }
|
||||
|
||||
public Dictionary<string, string>? Metadata { get; init; }
|
||||
|
||||
public static PackRunLogDocument FromDomain(string runId, long sequence, PackRunLogEntry entry)
|
||||
=> new()
|
||||
{
|
||||
Id = ObjectId.GenerateNewId(),
|
||||
RunId = runId,
|
||||
Sequence = sequence,
|
||||
Timestamp = entry.Timestamp.UtcDateTime,
|
||||
Level = entry.Level,
|
||||
EventType = entry.EventType,
|
||||
Message = entry.Message,
|
||||
StepId = entry.StepId,
|
||||
Metadata = entry.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(entry.Metadata, StringComparer.Ordinal)
|
||||
};
|
||||
|
||||
public PackRunLogEntry ToDomain()
|
||||
{
|
||||
IReadOnlyDictionary<string, string>? metadata = Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(Metadata, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunLogEntry(
|
||||
new DateTimeOffset(Timestamp, TimeSpan.Zero),
|
||||
Level,
|
||||
EventType,
|
||||
Message,
|
||||
StepId,
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<PackRunStateDocument> collection;
|
||||
|
||||
public MongoPackRunStateStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunStateDocument>(options.RunsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task<PackRunState?> GetAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunStateDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
var document = await collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return document?.ToDomain();
|
||||
}
|
||||
|
||||
public async Task SaveAsync(PackRunState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var document = PackRunStateDocument.FromDomain(state);
|
||||
var filter = Builders<PackRunStateDocument>.Filter.Eq(existing => existing.RunId, state.RunId);
|
||||
|
||||
await collection
|
||||
.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunState>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var documents = await collection
|
||||
.Find(FilterDefinition<PackRunStateDocument>.Empty)
|
||||
.SortByDescending(document => document.UpdatedAt)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(document => document.ToDomain())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt)),
|
||||
new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Ascending(document => document.PlanHash))
|
||||
};
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
}
|
||||
|
||||
private sealed class PackRunStateDocument
|
||||
{
|
||||
[BsonId]
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string PlanHash { get; init; } = default!;
|
||||
|
||||
public BsonDocument Plan { get; init; } = default!;
|
||||
|
||||
public BsonDocument FailurePolicy { get; init; } = default!;
|
||||
|
||||
public DateTime RequestedAt { get; init; }
|
||||
|
||||
public DateTime CreatedAt { get; init; }
|
||||
|
||||
public DateTime UpdatedAt { get; init; }
|
||||
|
||||
public List<PackRunStepDocument> Steps { get; init; } = new();
|
||||
|
||||
public static PackRunStateDocument FromDomain(PackRunState state)
|
||||
{
|
||||
var planDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.Plan, SerializerOptions));
|
||||
var failurePolicyDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.FailurePolicy, SerializerOptions));
|
||||
|
||||
var steps = state.Steps.Values
|
||||
.OrderBy(step => step.StepId, StringComparer.Ordinal)
|
||||
.Select(PackRunStepDocument.FromDomain)
|
||||
.ToList();
|
||||
|
||||
return new PackRunStateDocument
|
||||
{
|
||||
RunId = state.RunId,
|
||||
PlanHash = state.PlanHash,
|
||||
Plan = planDocument,
|
||||
FailurePolicy = failurePolicyDocument,
|
||||
RequestedAt = state.RequestedAt.UtcDateTime,
|
||||
CreatedAt = state.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = state.UpdatedAt.UtcDateTime,
|
||||
Steps = steps
|
||||
};
|
||||
}
|
||||
|
||||
public PackRunState ToDomain()
|
||||
{
|
||||
var planJson = Plan.ToJson();
|
||||
var plan = JsonSerializer.Deserialize<TaskPackPlan>(planJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize stored TaskPackPlan.");
|
||||
|
||||
var failurePolicyJson = FailurePolicy.ToJson();
|
||||
var failurePolicy = JsonSerializer.Deserialize<TaskPackPlanFailurePolicy>(failurePolicyJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize stored TaskPackPlanFailurePolicy.");
|
||||
|
||||
var stepRecords = Steps
|
||||
.Select(step => step.ToDomain())
|
||||
.ToDictionary(record => record.StepId, record => record, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunState(
|
||||
RunId,
|
||||
PlanHash,
|
||||
plan,
|
||||
failurePolicy,
|
||||
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(CreatedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(UpdatedAt, TimeSpan.Zero),
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PackRunStepDocument
|
||||
{
|
||||
public string StepId { get; init; } = default!;
|
||||
|
||||
public string Kind { get; init; } = default!;
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
|
||||
public bool ContinueOnError { get; init; }
|
||||
|
||||
public int? MaxParallel { get; init; }
|
||||
|
||||
public string? ApprovalId { get; init; }
|
||||
|
||||
public string? GateMessage { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public int Attempts { get; init; }
|
||||
|
||||
public DateTime? LastTransitionAt { get; init; }
|
||||
|
||||
public DateTime? NextAttemptAt { get; init; }
|
||||
|
||||
public string? StatusReason { get; init; }
|
||||
|
||||
public static PackRunStepDocument FromDomain(PackRunStepStateRecord record)
|
||||
=> new()
|
||||
{
|
||||
StepId = record.StepId,
|
||||
Kind = record.Kind.ToString(),
|
||||
Enabled = record.Enabled,
|
||||
ContinueOnError = record.ContinueOnError,
|
||||
MaxParallel = record.MaxParallel,
|
||||
ApprovalId = record.ApprovalId,
|
||||
GateMessage = record.GateMessage,
|
||||
Status = record.Status.ToString(),
|
||||
Attempts = record.Attempts,
|
||||
LastTransitionAt = record.LastTransitionAt?.UtcDateTime,
|
||||
NextAttemptAt = record.NextAttemptAt?.UtcDateTime,
|
||||
StatusReason = record.StatusReason
|
||||
};
|
||||
|
||||
public PackRunStepStateRecord ToDomain()
|
||||
{
|
||||
var kind = Enum.Parse<PackRunStepKind>(Kind, ignoreCase: true);
|
||||
var status = Enum.Parse<PackRunStepExecutionStatus>(Status, ignoreCase: true);
|
||||
|
||||
return new PackRunStepStateRecord(
|
||||
StepId,
|
||||
kind,
|
||||
Enabled,
|
||||
ContinueOnError,
|
||||
MaxParallel,
|
||||
ApprovalId,
|
||||
GateMessage,
|
||||
status,
|
||||
Attempts,
|
||||
LastTransitionAt is null ? null : new DateTimeOffset(LastTransitionAt.Value, TimeSpan.Zero),
|
||||
NextAttemptAt is null ? null : new DateTimeOffset(NextAttemptAt.Value, TimeSpan.Zero),
|
||||
StatusReason);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class FilePackRunLogStoreTests : IDisposable
|
||||
{
|
||||
private readonly string rootPath;
|
||||
|
||||
public FilePackRunLogStoreTests()
|
||||
{
|
||||
rootPath = Path.Combine(Path.GetTempPath(), "StellaOps_TaskRunnerTests", Guid.NewGuid().ToString("n"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppendAndReadAsync_RoundTripsEntriesInOrder()
|
||||
{
|
||||
var store = new FilePackRunLogStore(rootPath);
|
||||
var runId = "run-append-test";
|
||||
|
||||
var first = new PackRunLogEntry(
|
||||
DateTimeOffset.UtcNow,
|
||||
"info",
|
||||
"run.created",
|
||||
"Run created.",
|
||||
StepId: null,
|
||||
Metadata: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["planHash"] = "hash-1"
|
||||
});
|
||||
|
||||
var second = new PackRunLogEntry(
|
||||
DateTimeOffset.UtcNow.AddSeconds(1),
|
||||
"info",
|
||||
"step.started",
|
||||
"Step started.",
|
||||
StepId: "build",
|
||||
Metadata: null);
|
||||
|
||||
await store.AppendAsync(runId, first, CancellationToken.None);
|
||||
await store.AppendAsync(runId, second, CancellationToken.None);
|
||||
|
||||
var reloaded = new List<PackRunLogEntry>();
|
||||
await foreach (var entry in store.ReadAsync(runId, CancellationToken.None))
|
||||
{
|
||||
reloaded.Add(entry);
|
||||
}
|
||||
|
||||
Assert.Collection(
|
||||
reloaded,
|
||||
entry =>
|
||||
{
|
||||
Assert.Equal("run.created", entry.EventType);
|
||||
Assert.NotNull(entry.Metadata);
|
||||
Assert.Equal("hash-1", entry.Metadata!["planHash"]);
|
||||
},
|
||||
entry =>
|
||||
{
|
||||
Assert.Equal("step.started", entry.EventType);
|
||||
Assert.Equal("build", entry.StepId);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExistsAsync_ReturnsFalseWhenNoLogPresent()
|
||||
{
|
||||
var store = new FilePackRunLogStore(rootPath);
|
||||
|
||||
var exists = await store.ExistsAsync("missing-run", CancellationToken.None);
|
||||
|
||||
Assert.False(exists);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(rootPath))
|
||||
{
|
||||
Directory.Delete(rootPath, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore cleanup failures to keep tests deterministic.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
using StellaOps.TaskRunner.Core.TaskPacks;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using Xunit;
|
||||
using Xunit.Sdk;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class MongoPackRunStoresTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task StateStore_RoundTrips_State()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var stateStore = new MongoPackRunStateStore(context.Database, mongoOptions);
|
||||
|
||||
var plan = CreatePlan();
|
||||
var executionContext = new PackRunExecutionContext("mongo-run-state", plan, DateTimeOffset.UtcNow);
|
||||
var graph = new PackRunExecutionGraphBuilder().Build(plan);
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
|
||||
|
||||
await stateStore.SaveAsync(state, CancellationToken.None);
|
||||
|
||||
var reloaded = await stateStore.GetAsync(state.RunId, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(reloaded);
|
||||
Assert.Equal(state.RunId, reloaded!.RunId);
|
||||
Assert.Equal(state.PlanHash, reloaded.PlanHash);
|
||||
Assert.Equal(state.Steps.Count, reloaded.Steps.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LogStore_Appends_And_Reads_In_Order()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var logStore = new MongoPackRunLogStore(context.Database, mongoOptions);
|
||||
|
||||
var runId = "mongo-log";
|
||||
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.created", "created", null, null), CancellationToken.None);
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow.AddSeconds(1), "warn", "step.retry", "retry", "step-a", new Dictionary<string, string> { ["attempt"] = "2" }), CancellationToken.None);
|
||||
|
||||
var entries = new List<PackRunLogEntry>();
|
||||
await foreach (var entry in logStore.ReadAsync(runId, CancellationToken.None))
|
||||
{
|
||||
entries.Add(entry);
|
||||
}
|
||||
|
||||
Assert.Equal(2, entries.Count);
|
||||
Assert.Equal("run.created", entries[0].EventType);
|
||||
Assert.Equal("step.retry", entries[1].EventType);
|
||||
Assert.Equal("step-a", entries[1].StepId);
|
||||
Assert.True(await logStore.ExistsAsync(runId, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApprovalStore_RoundTrips_And_Updates()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var approvalStore = new MongoPackRunApprovalStore(context.Database, mongoOptions);
|
||||
|
||||
var runId = "mongo-approvals";
|
||||
var approval = new PackRunApprovalState(
|
||||
"security-review",
|
||||
new[] { "packs.approve" },
|
||||
new[] { "step-plan" },
|
||||
Array.Empty<string>(),
|
||||
reasonTemplate: "Security approval required.",
|
||||
DateTimeOffset.UtcNow,
|
||||
PackRunApprovalStatus.Pending);
|
||||
|
||||
await approvalStore.SaveAsync(runId, new[] { approval }, CancellationToken.None);
|
||||
|
||||
var approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
|
||||
Assert.Single(approvals);
|
||||
|
||||
var updated = approval.Approve("approver", DateTimeOffset.UtcNow, "Approved");
|
||||
await approvalStore.UpdateAsync(runId, updated, CancellationToken.None);
|
||||
|
||||
approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
|
||||
Assert.Single(approvals);
|
||||
Assert.Equal(PackRunApprovalStatus.Approved, approvals[0].Status);
|
||||
Assert.Equal("approver", approvals[0].ActorId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ArtifactUploader_Persists_Metadata()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var database = context.Database;
|
||||
|
||||
var artifactUploader = new MongoPackRunArtifactUploader(
|
||||
database,
|
||||
mongoOptions,
|
||||
TimeProvider.System,
|
||||
NullLogger<MongoPackRunArtifactUploader>.Instance);
|
||||
|
||||
var plan = CreatePlanWithOutputs(out var outputFile);
|
||||
try
|
||||
{
|
||||
var executionContext = new PackRunExecutionContext("mongo-artifacts", plan, DateTimeOffset.UtcNow);
|
||||
var graph = new PackRunExecutionGraphBuilder().Build(plan);
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
|
||||
|
||||
await artifactUploader.UploadAsync(executionContext, state, plan.Outputs, CancellationToken.None);
|
||||
|
||||
var documents = await database
|
||||
.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(mongoOptions.ArtifactsCollection)
|
||||
.Find(Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Empty)
|
||||
.ToListAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var bundleDocument = Assert.Single(documents, d => string.Equals(d.Name, "bundlePath", StringComparison.Ordinal));
|
||||
Assert.Equal("file", bundleDocument.Type);
|
||||
Assert.Equal(outputFile, bundleDocument.SourcePath);
|
||||
Assert.Equal("referenced", bundleDocument.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(outputFile))
|
||||
{
|
||||
File.Delete(outputFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static TaskPackPlan CreatePlan()
|
||||
{
|
||||
var manifest = TestManifests.Load(TestManifests.Sample);
|
||||
var planner = new TaskPackPlanner();
|
||||
var result = planner.Plan(manifest);
|
||||
if (!result.Success || result.Plan is null)
|
||||
{
|
||||
Assert.Skip("Failed to build task pack plan for Mongo tests.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
|
||||
return result.Plan;
|
||||
}
|
||||
|
||||
private static TaskPackPlan CreatePlanWithOutputs(out string outputFile)
|
||||
{
|
||||
var manifest = TestManifests.Load(TestManifests.Output);
|
||||
var planner = new TaskPackPlanner();
|
||||
var result = planner.Plan(manifest);
|
||||
if (!result.Success || result.Plan is null)
|
||||
{
|
||||
Assert.Skip("Failed to build output plan for Mongo tests.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
|
||||
// Materialize a fake output file referenced by the plan.
|
||||
outputFile = Path.Combine(Path.GetTempPath(), $"taskrunner-output-{Guid.NewGuid():N}.txt");
|
||||
File.WriteAllText(outputFile, "fixture");
|
||||
|
||||
// Update the plan output path parameter to point at the file we just created.
|
||||
var originalPlan = result.Plan;
|
||||
|
||||
var resolvedFile = outputFile;
|
||||
|
||||
var outputs = originalPlan.Outputs
|
||||
.Select(output =>
|
||||
{
|
||||
if (!string.Equals(output.Name, "bundlePath", StringComparison.Ordinal))
|
||||
{
|
||||
return output;
|
||||
}
|
||||
|
||||
var node = JsonNode.Parse($"\"{resolvedFile.Replace("\\", "\\\\")}\"");
|
||||
var parameter = new TaskPackPlanParameterValue(node, null, null, false);
|
||||
return output with { Path = parameter };
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
return new TaskPackPlan(
|
||||
originalPlan.Metadata,
|
||||
originalPlan.Inputs,
|
||||
originalPlan.Steps,
|
||||
originalPlan.Hash,
|
||||
originalPlan.Approvals,
|
||||
originalPlan.Secrets,
|
||||
outputs,
|
||||
originalPlan.FailurePolicy);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
internal sealed class MongoTaskRunnerTestContext : IAsyncDisposable, IDisposable
|
||||
{
|
||||
private readonly MongoDbRunner? runner;
|
||||
private readonly string databaseName;
|
||||
private readonly IMongoClient client;
|
||||
private readonly string connectionString;
|
||||
|
||||
private MongoTaskRunnerTestContext(
|
||||
IMongoClient client,
|
||||
IMongoDatabase database,
|
||||
MongoDbRunner? runner,
|
||||
string databaseName,
|
||||
string connectionString)
|
||||
{
|
||||
this.client = client;
|
||||
Database = database;
|
||||
this.runner = runner;
|
||||
this.databaseName = databaseName;
|
||||
this.connectionString = connectionString;
|
||||
}
|
||||
|
||||
public IMongoDatabase Database { get; }
|
||||
|
||||
public static MongoTaskRunnerTestContext Create()
|
||||
{
|
||||
OpenSslLegacyShim.EnsureOpenSsl11();
|
||||
|
||||
var uri = Environment.GetEnvironmentVariable("STELLAOPS_TEST_MONGO_URI");
|
||||
if (!string.IsNullOrWhiteSpace(uri))
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = MongoUrl.Create(uri);
|
||||
var client = new MongoClient(url);
|
||||
var databaseName = string.IsNullOrWhiteSpace(url.DatabaseName)
|
||||
? $"taskrunner-tests-{Guid.NewGuid():N}"
|
||||
: url.DatabaseName;
|
||||
var database = client.GetDatabase(databaseName);
|
||||
return new MongoTaskRunnerTestContext(client, database, runner: null, databaseName, uri);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Assert.Skip($"Failed to connect to MongoDB using STELLAOPS_TEST_MONGO_URI: {ex.Message}");
|
||||
throw new InvalidOperationException(); // Unreachable
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var runner = MongoDbRunner.Start(singleNodeReplSet: false);
|
||||
var client = new MongoClient(runner.ConnectionString);
|
||||
var databaseName = $"taskrunner-tests-{Guid.NewGuid():N}";
|
||||
var database = client.GetDatabase(databaseName);
|
||||
return new MongoTaskRunnerTestContext(client, database, runner, databaseName, runner.ConnectionString);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Assert.Skip($"Unable to start embedded MongoDB (Mongo2Go): {ex.Message}");
|
||||
throw new InvalidOperationException(); // Unreachable
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await client.DropDatabaseAsync(databaseName);
|
||||
runner?.Dispose();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
client.DropDatabase(databaseName);
|
||||
runner?.Dispose();
|
||||
}
|
||||
|
||||
public TaskRunnerMongoOptions CreateMongoOptions()
|
||||
=> new()
|
||||
{
|
||||
ConnectionString = connectionString,
|
||||
Database = databaseName
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
using StellaOps.TaskRunner.Core.TaskPacks;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class PackRunStateFactoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void CreateInitialState_AssignsGateReasons()
|
||||
{
|
||||
var manifest = TestManifests.Load(TestManifests.Sample);
|
||||
var planner = new TaskPackPlanner();
|
||||
var planResult = planner.Plan(manifest);
|
||||
|
||||
Assert.True(planResult.Success);
|
||||
Assert.NotNull(planResult.Plan);
|
||||
var plan = planResult.Plan!;
|
||||
|
||||
var context = new PackRunExecutionContext("run-state-factory", plan, DateTimeOffset.UtcNow);
|
||||
var graphBuilder = new PackRunExecutionGraphBuilder();
|
||||
var graph = graphBuilder.Build(plan);
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
|
||||
var timestamp = DateTimeOffset.UtcNow;
|
||||
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, timestamp);
|
||||
|
||||
Assert.Equal("run-state-factory", state.RunId);
|
||||
Assert.Equal(plan.Hash, state.PlanHash);
|
||||
|
||||
Assert.True(state.Steps.TryGetValue("approval", out var approvalStep));
|
||||
Assert.Equal(PackRunStepExecutionStatus.Pending, approvalStep.Status);
|
||||
Assert.Equal("requires-approval", approvalStep.StatusReason);
|
||||
|
||||
Assert.True(state.Steps.TryGetValue("plan-step", out var planStep));
|
||||
Assert.Equal(PackRunStepExecutionStatus.Pending, planStep.Status);
|
||||
Assert.Null(planStep.StatusReason);
|
||||
}
|
||||
}
|
||||
@@ -1,136 +1,46 @@
|
||||
<?xml version="1.0" ?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<PropertyGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
<OutputType>Exe</OutputType>
|
||||
|
||||
|
||||
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
|
||||
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
|
||||
<UseConcelierTestInfra>false</UseConcelierTestInfra>
|
||||
|
||||
|
||||
<LangVersion>preview</LangVersion>
|
||||
|
||||
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
|
||||
|
||||
</PropertyGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1"/>
|
||||
|
||||
|
||||
|
||||
|
||||
<PackageReference Include="xunit.v3" Version="3.0.0"/>
|
||||
|
||||
|
||||
|
||||
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3"/>
|
||||
|
||||
|
||||
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest"/>
|
||||
|
||||
|
||||
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
<Using Include="Xunit"/>
|
||||
|
||||
|
||||
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj"/>
|
||||
|
||||
|
||||
|
||||
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj"/>
|
||||
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj"/>
|
||||
|
||||
|
||||
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</Project>
|
||||
<?xml version="1.0"?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<UseConcelierTestInfra>false</UseConcelierTestInfra>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsPackable>false</IsPackable>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="Mongo2Go" Version="4.1.0" />
|
||||
<PackageReference Include="xunit.v3" Version="3.0.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj" />
|
||||
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\..\..\tests\native/openssl-1.1/linux-x64/*"
|
||||
Link="native/linux-x64/%(Filename)%(Extension)"
|
||||
CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\..\tests\shared\OpenSslLegacyShim.cs">
|
||||
<Link>Shared\OpenSslLegacyShim.cs</Link>
|
||||
</Compile>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
@@ -11,20 +17,52 @@ using StellaOps.TaskRunner.WebService;
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
builder.Services.Configure<TaskRunnerServiceOptions>(builder.Configuration.GetSection("TaskRunner"));
|
||||
builder.Services.AddSingleton<TaskPackManifestLoader>();
|
||||
builder.Services.AddSingleton<TaskPackManifestLoader>();
|
||||
builder.Services.AddSingleton<TaskPackPlanner>();
|
||||
builder.Services.AddSingleton<PackRunSimulationEngine>();
|
||||
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
|
||||
var storageOptions = builder.Configuration.GetSection("TaskRunner:Storage").Get<TaskRunnerStorageOptions>() ?? new TaskRunnerStorageOptions();
|
||||
builder.Services.AddSingleton(storageOptions);
|
||||
|
||||
if (string.Equals(storageOptions.Mode, TaskRunnerStorageModes.Mongo, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunApprovalStore(options.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
builder.Services.AddSingleton(storageOptions.Mongo);
|
||||
builder.Services.AddSingleton<IMongoClient>(_ => new MongoClient(storageOptions.Mongo.ConnectionString));
|
||||
builder.Services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var mongoOptions = storageOptions.Mongo;
|
||||
var client = sp.GetRequiredService<IMongoClient>();
|
||||
var mongoUrl = MongoUrl.Create(mongoOptions.ConnectionString);
|
||||
var databaseName = !string.IsNullOrWhiteSpace(mongoOptions.Database)
|
||||
? mongoOptions.Database
|
||||
: mongoUrl.DatabaseName ?? "stellaops-taskrunner";
|
||||
return client.GetDatabase(databaseName);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
}
|
||||
else
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunStateStore(options.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunApprovalStore(options.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunStateStore(options.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunLogStore(options.LogsPath);
|
||||
});
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
@@ -77,8 +115,89 @@ app.MapPost("/v1/task-runner/simulations", async (
|
||||
var simulation = simulationEngine.Simulate(plan);
|
||||
var response = SimulationMapper.ToResponse(plan, simulation);
|
||||
return Results.Ok(response);
|
||||
}).WithName("SimulateTaskPack");
|
||||
|
||||
}).WithName("SimulateTaskPack");
|
||||
|
||||
app.MapPost("/v1/task-runner/runs", async (
|
||||
[FromBody] CreateRunRequest request,
|
||||
TaskPackManifestLoader loader,
|
||||
TaskPackPlanner planner,
|
||||
PackRunExecutionGraphBuilder executionGraphBuilder,
|
||||
PackRunSimulationEngine simulationEngine,
|
||||
IPackRunStateStore stateStore,
|
||||
IPackRunLogStore logStore,
|
||||
IPackRunJobScheduler scheduler,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request is null || string.IsNullOrWhiteSpace(request.Manifest))
|
||||
{
|
||||
return Results.BadRequest(new { error = "Manifest is required." });
|
||||
}
|
||||
|
||||
TaskPackManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = loader.Deserialize(request.Manifest);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = "Invalid manifest", detail = ex.Message });
|
||||
}
|
||||
|
||||
var inputs = ConvertInputs(request.Inputs);
|
||||
var planResult = planner.Plan(manifest, inputs);
|
||||
if (!planResult.Success || planResult.Plan is null)
|
||||
{
|
||||
return Results.BadRequest(new
|
||||
{
|
||||
errors = planResult.Errors.Select(error => new { error.Path, error.Message })
|
||||
});
|
||||
}
|
||||
|
||||
var plan = planResult.Plan;
|
||||
var runId = string.IsNullOrWhiteSpace(request.RunId)
|
||||
? Guid.NewGuid().ToString("n")
|
||||
: request.RunId!;
|
||||
|
||||
var existing = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
return Results.Conflict(new { error = "Run already exists." });
|
||||
}
|
||||
|
||||
var requestedAt = DateTimeOffset.UtcNow;
|
||||
var context = new PackRunExecutionContext(runId, plan, requestedAt);
|
||||
var graph = executionGraphBuilder.Build(plan);
|
||||
|
||||
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, requestedAt);
|
||||
await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
await scheduler.ScheduleAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await logStore.AppendAsync(
|
||||
runId,
|
||||
new PackRunLogEntry(DateTimeOffset.UtcNow, "error", "run.schedule-failed", ex.Message, null, null),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return Results.StatusCode(StatusCodes.Status500InternalServerError);
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
metadata["planHash"] = plan.Hash;
|
||||
metadata["requestedAt"] = requestedAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
|
||||
await logStore.AppendAsync(
|
||||
runId,
|
||||
new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.created", "Run created via API.", null, metadata),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var response = RunStateMapper.ToResponse(state);
|
||||
return Results.Created($"/v1/task-runner/runs/{runId}", response);
|
||||
}).WithName("CreatePackRun");
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}", async (
|
||||
string runId,
|
||||
IPackRunStateStore stateStore,
|
||||
@@ -94,10 +213,34 @@ app.MapGet("/v1/task-runner/runs/{runId}", async (
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
|
||||
return Results.Ok(RunStateMapper.ToResponse(state));
|
||||
}).WithName("GetRunState");
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}/logs", async (
|
||||
string runId,
|
||||
IPackRunLogStore logStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "runId is required." });
|
||||
}
|
||||
|
||||
if (!await logStore.ExistsAsync(runId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
return Results.Stream(async (stream, ct) =>
|
||||
{
|
||||
await foreach (var entry in logStore.ReadAsync(runId, ct).ConfigureAwait(false))
|
||||
{
|
||||
await RunLogMapper.WriteAsync(stream, entry, ct).ConfigureAwait(false);
|
||||
}
|
||||
}, "application/x-ndjson");
|
||||
}).WithName("StreamRunLogs");
|
||||
|
||||
app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", async (
|
||||
string runId,
|
||||
string approvalId,
|
||||
@@ -151,12 +294,14 @@ static IDictionary<string, JsonNode?>? ConvertInputs(JsonObject? node)
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
internal sealed record SimulationRequest(string Manifest, JsonObject? Inputs);
|
||||
|
||||
internal sealed record SimulationResponse(
|
||||
string PlanHash,
|
||||
FailurePolicyResponse FailurePolicy,
|
||||
IReadOnlyList<SimulationStepResponse> Steps,
|
||||
internal sealed record CreateRunRequest(string? RunId, string Manifest, JsonObject? Inputs);
|
||||
|
||||
internal sealed record SimulationRequest(string Manifest, JsonObject? Inputs);
|
||||
|
||||
internal sealed record SimulationResponse(
|
||||
string PlanHash,
|
||||
FailurePolicyResponse FailurePolicy,
|
||||
IReadOnlyList<SimulationStepResponse> Steps,
|
||||
IReadOnlyList<SimulationOutputResponse> Outputs,
|
||||
bool HasPendingApprovals);
|
||||
|
||||
@@ -206,9 +351,54 @@ internal sealed record RunStateStepResponse(
|
||||
string? StatusReason);
|
||||
|
||||
internal sealed record ApprovalDecisionDto(string Decision, string? ActorId, string? Summary);
|
||||
|
||||
internal static class SimulationMapper
|
||||
{
|
||||
|
||||
internal sealed record RunLogEntryResponse(
|
||||
DateTimeOffset Timestamp,
|
||||
string Level,
|
||||
string EventType,
|
||||
string Message,
|
||||
string? StepId,
|
||||
IReadOnlyDictionary<string, string>? Metadata);
|
||||
|
||||
internal static class RunLogMapper
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private static readonly byte[] NewLine = Encoding.UTF8.GetBytes("\n");
|
||||
|
||||
public static RunLogEntryResponse ToResponse(PackRunLogEntry entry)
|
||||
{
|
||||
IReadOnlyDictionary<string, string>? metadata = null;
|
||||
if (entry.Metadata is { Count: > 0 })
|
||||
{
|
||||
metadata = entry.Metadata
|
||||
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
|
||||
.ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
return new RunLogEntryResponse(
|
||||
entry.Timestamp,
|
||||
entry.Level,
|
||||
entry.EventType,
|
||||
entry.Message,
|
||||
entry.StepId,
|
||||
metadata);
|
||||
}
|
||||
|
||||
public static async Task WriteAsync(Stream stream, PackRunLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = ToResponse(entry);
|
||||
await JsonSerializer.SerializeAsync(stream, response, SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.WriteAsync(NewLine, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class SimulationMapper
|
||||
{
|
||||
public static SimulationResponse ToResponse(TaskPackPlan plan, PackRunSimulationResult result)
|
||||
{
|
||||
var failurePolicy = result.FailurePolicy ?? PackRunExecutionGraph.DefaultFailurePolicy;
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
namespace StellaOps.TaskRunner.WebService;
|
||||
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
|
||||
namespace StellaOps.TaskRunner.WebService;
|
||||
|
||||
public sealed class TaskRunnerServiceOptions
|
||||
{
|
||||
public string RunStatePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "state", "runs");
|
||||
public string ApprovalStorePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "approvals");
|
||||
public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
|
||||
|
||||
public TaskRunnerStorageOptions Storage { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.AirGap.Policy;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using StellaOps.TaskRunner.Worker.Services;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
|
||||
@@ -11,13 +13,8 @@ builder.Services.AddAirGapEgressPolicy(builder.Configuration, sectionName: "AirG
|
||||
builder.Services.Configure<PackRunWorkerOptions>(builder.Configuration.GetSection("Worker"));
|
||||
builder.Services.Configure<NotificationOptions>(builder.Configuration.GetSection("Notifications"));
|
||||
builder.Services.AddHttpClient("taskrunner-notifications");
|
||||
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunApprovalStore(options.Value.ApprovalStorePath);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
|
||||
builder.Services.AddSingleton(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
@@ -30,34 +27,71 @@ builder.Services.AddSingleton<IPackRunJobScheduler>(sp => sp.GetRequiredService<
|
||||
builder.Services.AddSingleton<IPackRunNotificationPublisher>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<NotificationOptions>>().Value;
|
||||
if (options.ApprovalEndpoint is not null || options.PolicyEndpoint is not null)
|
||||
{
|
||||
return new HttpPackRunNotificationPublisher(
|
||||
sp.GetRequiredService<IHttpClientFactory>(),
|
||||
sp.GetRequiredService<IOptions<NotificationOptions>>(),
|
||||
sp.GetRequiredService<ILogger<HttpPackRunNotificationPublisher>>());
|
||||
}
|
||||
if (options.ApprovalEndpoint is not null || options.PolicyEndpoint is not null)
|
||||
{
|
||||
return new HttpPackRunNotificationPublisher(
|
||||
sp.GetRequiredService<IHttpClientFactory>(),
|
||||
sp.GetRequiredService<IOptions<NotificationOptions>>(),
|
||||
sp.GetRequiredService<ILogger<HttpPackRunNotificationPublisher>>());
|
||||
}
|
||||
|
||||
return new LoggingPackRunNotificationPublisher(sp.GetRequiredService<ILogger<LoggingPackRunNotificationPublisher>>());
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunStateStore(options.Value.RunStatePath);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStepExecutor, NoopPackRunStepExecutor>();
|
||||
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
|
||||
builder.Services.AddSingleton<PackRunSimulationEngine>();
|
||||
builder.Services.AddSingleton<PackRunProcessor>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader>(sp =>
|
||||
|
||||
var workerStorageOptions = builder.Configuration.GetSection("Worker:Storage").Get<TaskRunnerStorageOptions>() ?? new TaskRunnerStorageOptions();
|
||||
builder.Services.AddSingleton(workerStorageOptions);
|
||||
|
||||
if (string.Equals(workerStorageOptions.Mode, TaskRunnerStorageModes.Mongo, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetService<TimeProvider>();
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
builder.Services.AddSingleton(workerStorageOptions.Mongo);
|
||||
builder.Services.AddSingleton<IMongoClient>(_ => new MongoClient(workerStorageOptions.Mongo.ConnectionString));
|
||||
builder.Services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var mongoOptions = workerStorageOptions.Mongo;
|
||||
var client = sp.GetRequiredService<IMongoClient>();
|
||||
var mongoUrl = MongoUrl.Create(mongoOptions.ConnectionString);
|
||||
var databaseName = !string.IsNullOrWhiteSpace(mongoOptions.Database)
|
||||
? mongoOptions.Database
|
||||
: mongoUrl.DatabaseName ?? "stellaops-taskrunner";
|
||||
return client.GetDatabase(databaseName);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader, MongoPackRunArtifactUploader>();
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunApprovalStore(options.Value.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunStateStore(options.Value.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunLogStore(options.Value.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
}
|
||||
|
||||
builder.Services.AddHostedService<PackRunWorkerService>();
|
||||
|
||||
var host = builder.Build();
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
namespace StellaOps.TaskRunner.Worker.Services;
|
||||
|
||||
public sealed class PackRunWorkerOptions
|
||||
{
|
||||
public TimeSpan IdleDelay { get; set; } = TimeSpan.FromSeconds(1);
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
|
||||
namespace StellaOps.TaskRunner.Worker.Services;
|
||||
|
||||
public sealed class PackRunWorkerOptions
|
||||
{
|
||||
public TimeSpan IdleDelay { get; set; } = TimeSpan.FromSeconds(1);
|
||||
|
||||
public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
|
||||
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
@@ -13,4 +15,8 @@ public sealed class PackRunWorkerOptions
|
||||
public string RunStatePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "state", "runs");
|
||||
|
||||
public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts");
|
||||
|
||||
public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
|
||||
|
||||
public TaskRunnerStorageOptions Storage { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
@@ -21,6 +22,7 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
private readonly PackRunSimulationEngine simulationEngine;
|
||||
private readonly IPackRunStepExecutor executor;
|
||||
private readonly IPackRunArtifactUploader artifactUploader;
|
||||
private readonly IPackRunLogStore logStore;
|
||||
private readonly ILogger<PackRunWorkerService> logger;
|
||||
|
||||
public PackRunWorkerService(
|
||||
@@ -31,6 +33,7 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
PackRunSimulationEngine simulationEngine,
|
||||
IPackRunStepExecutor executor,
|
||||
IPackRunArtifactUploader artifactUploader,
|
||||
IPackRunLogStore logStore,
|
||||
IOptions<PackRunWorkerOptions> options,
|
||||
ILogger<PackRunWorkerService> logger)
|
||||
{
|
||||
@@ -41,6 +44,7 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
this.simulationEngine = simulationEngine ?? throw new ArgumentNullException(nameof(simulationEngine));
|
||||
this.executor = executor ?? throw new ArgumentNullException(nameof(executor));
|
||||
this.artifactUploader = artifactUploader ?? throw new ArgumentNullException(nameof(artifactUploader));
|
||||
this.logStore = logStore ?? throw new ArgumentNullException(nameof(logStore));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
@@ -63,122 +67,126 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Unhandled exception while processing run {RunId}.", context.RunId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessRunAsync(PackRunExecutionContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
logger.LogInformation("Processing pack run {RunId}.", context.RunId);
|
||||
|
||||
var processorResult = await processor.ProcessNewRunAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var graph = graphBuilder.Build(context.Plan);
|
||||
|
||||
var state = await stateStore.GetAsync(context.RunId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null || !string.Equals(state.PlanHash, context.Plan.Hash, StringComparison.Ordinal))
|
||||
{
|
||||
state = await CreateInitialStateAsync(context, graph, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!processorResult.ShouldResumeImmediately)
|
||||
{
|
||||
logger.LogInformation("Run {RunId} awaiting approvals or policy gates.", context.RunId);
|
||||
return;
|
||||
}
|
||||
|
||||
var gateUpdate = PackRunGateStateUpdater.Apply(state, graph, processorResult.ApprovalCoordinator, DateTimeOffset.UtcNow);
|
||||
state = gateUpdate.State;
|
||||
|
||||
if (gateUpdate.HasBlockingFailure)
|
||||
{
|
||||
await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
logger.LogWarning("Run {RunId} halted because a gate failed.", context.RunId);
|
||||
return;
|
||||
}
|
||||
|
||||
var updatedState = await ExecuteGraphAsync(context, graph, state, cancellationToken).ConfigureAwait(false);
|
||||
await stateStore.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Unhandled exception while processing run {RunId}.", context.RunId);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["exceptionType"] = ex.GetType().FullName ?? ex.GetType().Name
|
||||
};
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"error",
|
||||
"run.failed",
|
||||
"Unhandled exception while processing run.",
|
||||
stoppingToken,
|
||||
metadata: metadata).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessRunAsync(PackRunExecutionContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
logger.LogInformation("Processing pack run {RunId}.", context.RunId);
|
||||
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"info",
|
||||
"run.received",
|
||||
"Run dequeued by worker.",
|
||||
cancellationToken,
|
||||
metadata: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["planHash"] = context.Plan.Hash
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
var processorResult = await processor.ProcessNewRunAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var graph = graphBuilder.Build(context.Plan);
|
||||
|
||||
var state = await stateStore.GetAsync(context.RunId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null || !string.Equals(state.PlanHash, context.Plan.Hash, StringComparison.Ordinal))
|
||||
{
|
||||
state = await CreateInitialStateAsync(context, graph, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!processorResult.ShouldResumeImmediately)
|
||||
{
|
||||
logger.LogInformation("Run {RunId} awaiting approvals or policy gates.", context.RunId);
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"info",
|
||||
"run.awaiting-approvals",
|
||||
"Run paused awaiting approvals or policy gates.",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var gateUpdate = PackRunGateStateUpdater.Apply(state, graph, processorResult.ApprovalCoordinator, DateTimeOffset.UtcNow);
|
||||
state = gateUpdate.State;
|
||||
|
||||
if (gateUpdate.HasBlockingFailure)
|
||||
{
|
||||
await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
logger.LogWarning("Run {RunId} halted because a gate failed.", context.RunId);
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"warn",
|
||||
"run.gate-blocked",
|
||||
"Run halted because a gate failed.",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var updatedState = await ExecuteGraphAsync(context, graph, state, cancellationToken).ConfigureAwait(false);
|
||||
await stateStore.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (updatedState.Steps.Values.All(step => step.Status is PackRunStepExecutionStatus.Succeeded or PackRunStepExecutionStatus.Skipped))
|
||||
{
|
||||
logger.LogInformation("Run {RunId} finished successfully.", context.RunId);
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"info",
|
||||
"run.completed",
|
||||
"Run finished successfully.",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await artifactUploader.UploadAsync(context, updatedState, context.Plan.Outputs, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogInformation("Run {RunId} paused with pending work.", context.RunId);
|
||||
await AppendLogAsync(
|
||||
context.RunId,
|
||||
"info",
|
||||
"run.paused",
|
||||
"Run paused with pending work.",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<PackRunState> CreateInitialStateAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunExecutionGraph graph,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timestamp = DateTimeOffset.UtcNow;
|
||||
var simulation = simulationEngine.Simulate(context.Plan);
|
||||
var simulationIndex = IndexSimulation(simulation.Steps);
|
||||
|
||||
var stepRecords = new Dictionary<string, PackRunStepStateRecord>(StringComparer.Ordinal);
|
||||
foreach (var step in EnumerateSteps(graph.Steps))
|
||||
{
|
||||
var simulationStatus = simulationIndex.TryGetValue(step.Id, out var node)
|
||||
? node.Status
|
||||
: PackRunSimulationStatus.Pending;
|
||||
|
||||
var status = step.Enabled ? PackRunStepExecutionStatus.Pending : PackRunStepExecutionStatus.Skipped;
|
||||
string? statusReason = null;
|
||||
if (!step.Enabled)
|
||||
{
|
||||
statusReason = "disabled";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.RequiresApproval)
|
||||
{
|
||||
statusReason = "requires-approval";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.RequiresPolicy)
|
||||
{
|
||||
statusReason = "requires-policy";
|
||||
}
|
||||
else if (simulationStatus == PackRunSimulationStatus.Skipped)
|
||||
{
|
||||
status = PackRunStepExecutionStatus.Skipped;
|
||||
statusReason = "condition-false";
|
||||
}
|
||||
|
||||
var record = new PackRunStepStateRecord(
|
||||
step.Id,
|
||||
step.Kind,
|
||||
step.Enabled,
|
||||
step.ContinueOnError,
|
||||
step.MaxParallel,
|
||||
step.ApprovalId,
|
||||
step.GateMessage,
|
||||
status,
|
||||
Attempts: 0,
|
||||
LastTransitionAt: null,
|
||||
NextAttemptAt: null,
|
||||
StatusReason: statusReason);
|
||||
|
||||
stepRecords[step.Id] = record;
|
||||
}
|
||||
|
||||
var failurePolicy = graph.FailurePolicy ?? PackRunExecutionGraph.DefaultFailurePolicy;
|
||||
var state = PackRunState.Create(
|
||||
context.RunId,
|
||||
context.Plan.Hash,
|
||||
context.Plan,
|
||||
failurePolicy,
|
||||
context.RequestedAt,
|
||||
stepRecords,
|
||||
timestamp);
|
||||
await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<PackRunState> CreateInitialStateAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunExecutionGraph graph,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var timestamp = DateTimeOffset.UtcNow;
|
||||
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, timestamp);
|
||||
await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
|
||||
return state;
|
||||
}
|
||||
|
||||
private Task AppendLogAsync(
|
||||
string runId,
|
||||
string level,
|
||||
string eventType,
|
||||
string message,
|
||||
CancellationToken cancellationToken,
|
||||
string? stepId = null,
|
||||
IReadOnlyDictionary<string, string>? metadata = null)
|
||||
{
|
||||
var entry = new PackRunLogEntry(DateTimeOffset.UtcNow, level, eventType, message, stepId, metadata);
|
||||
return logStore.AppendAsync(runId, entry, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<PackRunState> ExecuteGraphAsync(
|
||||
PackRunExecutionContext context,
|
||||
@@ -228,52 +236,83 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
return StepExecutionOutcome.Continue;
|
||||
}
|
||||
|
||||
if (record.NextAttemptAt is { } scheduled && scheduled > DateTimeOffset.UtcNow)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Run {RunId} step {StepId} waiting until {NextAttempt} for retry.",
|
||||
executionContext.RunId,
|
||||
record.StepId,
|
||||
scheduled);
|
||||
return StepExecutionOutcome.Defer;
|
||||
}
|
||||
|
||||
switch (step.Kind)
|
||||
{
|
||||
case PackRunStepKind.GateApproval:
|
||||
case PackRunStepKind.GatePolicy:
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = PackRunStepExecutionStatus.Succeeded,
|
||||
if (record.NextAttemptAt is { } scheduled && scheduled > DateTimeOffset.UtcNow)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Run {RunId} step {StepId} waiting until {NextAttempt} for retry.",
|
||||
executionContext.RunId,
|
||||
record.StepId,
|
||||
scheduled);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["nextAttemptAt"] = scheduled.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture),
|
||||
["attempts"] = record.Attempts.ToString(CultureInfo.InvariantCulture)
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"info",
|
||||
"step.awaiting-retry",
|
||||
$"Step {record.StepId} waiting for retry.",
|
||||
executionContext.CancellationToken,
|
||||
record.StepId,
|
||||
metadata).ConfigureAwait(false);
|
||||
return StepExecutionOutcome.Defer;
|
||||
}
|
||||
|
||||
switch (step.Kind)
|
||||
{
|
||||
case PackRunStepKind.GateApproval:
|
||||
case PackRunStepKind.GatePolicy:
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = PackRunStepExecutionStatus.Succeeded,
|
||||
StatusReason = null,
|
||||
LastTransitionAt = DateTimeOffset.UtcNow,
|
||||
NextAttemptAt = null
|
||||
};
|
||||
return StepExecutionOutcome.Continue;
|
||||
|
||||
case PackRunStepKind.Parallel:
|
||||
return await ExecuteParallelStepAsync(step, executionContext).ConfigureAwait(false);
|
||||
|
||||
case PackRunStepKind.Map:
|
||||
LastTransitionAt = DateTimeOffset.UtcNow,
|
||||
NextAttemptAt = null
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"info",
|
||||
step.Kind == PackRunStepKind.GateApproval ? "step.approval-satisfied" : "step.policy-satisfied",
|
||||
$"Gate {step.Id} satisfied.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id).ConfigureAwait(false);
|
||||
return StepExecutionOutcome.Continue;
|
||||
|
||||
case PackRunStepKind.Parallel:
|
||||
return await ExecuteParallelStepAsync(step, executionContext).ConfigureAwait(false);
|
||||
|
||||
case PackRunStepKind.Map:
|
||||
return await ExecuteMapStepAsync(step, executionContext).ConfigureAwait(false);
|
||||
|
||||
case PackRunStepKind.Run:
|
||||
return await ExecuteRunStepAsync(step, executionContext).ConfigureAwait(false);
|
||||
|
||||
default:
|
||||
logger.LogWarning("Run {RunId} encountered unsupported step kind '{Kind}' for step {StepId}. Marking as skipped.",
|
||||
executionContext.RunId,
|
||||
step.Kind,
|
||||
step.Id);
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = PackRunStepExecutionStatus.Skipped,
|
||||
StatusReason = "unsupported-kind",
|
||||
LastTransitionAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
return StepExecutionOutcome.Continue;
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.LogWarning("Run {RunId} encountered unsupported step kind '{Kind}' for step {StepId}. Marking as skipped.",
|
||||
executionContext.RunId,
|
||||
step.Kind,
|
||||
step.Id);
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = PackRunStepExecutionStatus.Skipped,
|
||||
StatusReason = "unsupported-kind",
|
||||
LastTransitionAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"warn",
|
||||
"step.skipped",
|
||||
"Step skipped because the step kind is unsupported.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id,
|
||||
new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["kind"] = step.Kind.ToString()
|
||||
}).ConfigureAwait(false);
|
||||
return StepExecutionOutcome.Continue;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<StepExecutionOutcome> ExecuteRunStepAsync(
|
||||
PackRunExecutionStep step,
|
||||
@@ -283,57 +322,124 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var currentState = new PackRunStepState(record.Status, record.Attempts, record.LastTransitionAt, record.NextAttemptAt);
|
||||
|
||||
if (currentState.Status == PackRunStepExecutionStatus.Pending)
|
||||
{
|
||||
currentState = PackRunStepStateMachine.Start(currentState, now);
|
||||
record = record with
|
||||
{
|
||||
Status = currentState.Status,
|
||||
LastTransitionAt = currentState.LastTransitionAt,
|
||||
NextAttemptAt = currentState.NextAttemptAt,
|
||||
StatusReason = null
|
||||
};
|
||||
executionContext.Steps[step.Id] = record;
|
||||
}
|
||||
|
||||
var result = await executor.ExecuteAsync(step, step.Parameters ?? PackRunExecutionStep.EmptyParameters, executionContext.CancellationToken).ConfigureAwait(false);
|
||||
if (result.Succeeded)
|
||||
{
|
||||
currentState = PackRunStepStateMachine.CompleteSuccess(currentState, DateTimeOffset.UtcNow);
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = currentState.Status,
|
||||
Attempts = currentState.Attempts,
|
||||
LastTransitionAt = currentState.LastTransitionAt,
|
||||
NextAttemptAt = currentState.NextAttemptAt,
|
||||
StatusReason = null
|
||||
};
|
||||
|
||||
return StepExecutionOutcome.Continue;
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Run {RunId} step {StepId} failed: {Error}",
|
||||
executionContext.RunId,
|
||||
step.Id,
|
||||
result.Error ?? "unknown error");
|
||||
|
||||
var failure = PackRunStepStateMachine.RegisterFailure(currentState, DateTimeOffset.UtcNow, executionContext.FailurePolicy);
|
||||
var updatedRecord = record with
|
||||
{
|
||||
Status = failure.State.Status,
|
||||
Attempts = failure.State.Attempts,
|
||||
LastTransitionAt = failure.State.LastTransitionAt,
|
||||
NextAttemptAt = failure.State.NextAttemptAt,
|
||||
StatusReason = result.Error
|
||||
};
|
||||
|
||||
executionContext.Steps[step.Id] = updatedRecord;
|
||||
|
||||
return failure.Outcome switch
|
||||
{
|
||||
PackRunStepFailureOutcome.Retry => StepExecutionOutcome.Defer,
|
||||
PackRunStepFailureOutcome.Abort when step.ContinueOnError => StepExecutionOutcome.Continue,
|
||||
if (currentState.Status == PackRunStepExecutionStatus.Pending)
|
||||
{
|
||||
currentState = PackRunStepStateMachine.Start(currentState, now);
|
||||
record = record with
|
||||
{
|
||||
Status = currentState.Status,
|
||||
LastTransitionAt = currentState.LastTransitionAt,
|
||||
NextAttemptAt = currentState.NextAttemptAt,
|
||||
StatusReason = null
|
||||
};
|
||||
executionContext.Steps[step.Id] = record;
|
||||
var startMetadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["attempt"] = currentState.Attempts.ToString(CultureInfo.InvariantCulture)
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"info",
|
||||
"step.started",
|
||||
$"Step {step.Id} started.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id,
|
||||
startMetadata).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var result = await executor.ExecuteAsync(step, step.Parameters ?? PackRunExecutionStep.EmptyParameters, executionContext.CancellationToken).ConfigureAwait(false);
|
||||
if (result.Succeeded)
|
||||
{
|
||||
currentState = PackRunStepStateMachine.CompleteSuccess(currentState, DateTimeOffset.UtcNow);
|
||||
executionContext.Steps[step.Id] = record with
|
||||
{
|
||||
Status = currentState.Status,
|
||||
Attempts = currentState.Attempts,
|
||||
LastTransitionAt = currentState.LastTransitionAt,
|
||||
NextAttemptAt = currentState.NextAttemptAt,
|
||||
StatusReason = null
|
||||
};
|
||||
|
||||
var successMetadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["attempt"] = currentState.Attempts.ToString(CultureInfo.InvariantCulture)
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"info",
|
||||
"step.succeeded",
|
||||
$"Step {step.Id} succeeded.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id,
|
||||
successMetadata).ConfigureAwait(false);
|
||||
|
||||
return StepExecutionOutcome.Continue;
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Run {RunId} step {StepId} failed: {Error}",
|
||||
executionContext.RunId,
|
||||
step.Id,
|
||||
result.Error ?? "unknown error");
|
||||
|
||||
var failure = PackRunStepStateMachine.RegisterFailure(currentState, DateTimeOffset.UtcNow, executionContext.FailurePolicy);
|
||||
var updatedRecord = record with
|
||||
{
|
||||
Status = failure.State.Status,
|
||||
Attempts = failure.State.Attempts,
|
||||
LastTransitionAt = failure.State.LastTransitionAt,
|
||||
NextAttemptAt = failure.State.NextAttemptAt,
|
||||
StatusReason = result.Error
|
||||
};
|
||||
|
||||
executionContext.Steps[step.Id] = updatedRecord;
|
||||
|
||||
var failureMetadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["attempt"] = failure.State.Attempts.ToString(CultureInfo.InvariantCulture)
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(result.Error))
|
||||
{
|
||||
failureMetadata["error"] = result.Error;
|
||||
}
|
||||
if (failure.State.NextAttemptAt is { } retryAt)
|
||||
{
|
||||
failureMetadata["nextAttemptAt"] = retryAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
var failureLevel = failure.Outcome == PackRunStepFailureOutcome.Abort && !step.ContinueOnError
|
||||
? "error"
|
||||
: "warn";
|
||||
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
failureLevel,
|
||||
"step.failed",
|
||||
$"Step {step.Id} failed.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id,
|
||||
failureMetadata).ConfigureAwait(false);
|
||||
|
||||
if (failure.Outcome == PackRunStepFailureOutcome.Retry)
|
||||
{
|
||||
var retryMetadata = new Dictionary<string, string>(failureMetadata, StringComparer.Ordinal)
|
||||
{
|
||||
["outcome"] = "retry"
|
||||
};
|
||||
await AppendLogAsync(
|
||||
executionContext.RunId,
|
||||
"info",
|
||||
"step.retry-scheduled",
|
||||
$"Step {step.Id} scheduled for retry.",
|
||||
executionContext.CancellationToken,
|
||||
step.Id,
|
||||
retryMetadata).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return failure.Outcome switch
|
||||
{
|
||||
PackRunStepFailureOutcome.Retry => StepExecutionOutcome.Defer,
|
||||
PackRunStepFailureOutcome.Abort when step.ContinueOnError => StepExecutionOutcome.Continue,
|
||||
PackRunStepFailureOutcome.Abort => StepExecutionOutcome.AbortRun,
|
||||
_ => StepExecutionOutcome.AbortRun
|
||||
};
|
||||
@@ -503,44 +609,11 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, PackRunSimulationNode> IndexSimulation(IReadOnlyList<PackRunSimulationNode> steps)
|
||||
{
|
||||
var result = new Dictionary<string, PackRunSimulationNode>(StringComparer.Ordinal);
|
||||
foreach (var node in steps)
|
||||
{
|
||||
result[node.Id] = node;
|
||||
if (node.Children.Count > 0)
|
||||
{
|
||||
foreach (var child in IndexSimulation(node.Children))
|
||||
{
|
||||
result[child.Key] = child.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static IEnumerable<PackRunExecutionStep> EnumerateSteps(IReadOnlyList<PackRunExecutionStep> steps)
|
||||
{
|
||||
foreach (var step in steps)
|
||||
{
|
||||
yield return step;
|
||||
if (step.Children.Count > 0)
|
||||
{
|
||||
foreach (var child in EnumerateSteps(step.Children))
|
||||
{
|
||||
yield return child;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record ExecutionContext(
|
||||
string RunId,
|
||||
TaskPackPlanFailurePolicy FailurePolicy,
|
||||
ConcurrentDictionary<string, PackRunStepStateRecord> Steps,
|
||||
CancellationToken CancellationToken);
|
||||
private sealed record ExecutionContext(
|
||||
string RunId,
|
||||
TaskPackPlanFailurePolicy FailurePolicy,
|
||||
ConcurrentDictionary<string, PackRunStepStateRecord> Steps,
|
||||
CancellationToken CancellationToken);
|
||||
|
||||
private enum StepExecutionOutcome
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user