Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management

- Added MongoPackRunApprovalStore for managing approval states with MongoDB.
- Introduced MongoPackRunArtifactUploader for uploading and storing artifacts.
- Created MongoPackRunLogStore to handle logging of pack run events.
- Developed MongoPackRunStateStore for persisting and retrieving pack run states.
- Implemented unit tests for MongoDB stores to ensure correct functionality.
- Added MongoTaskRunnerTestContext for setting up MongoDB test environment.
- Enhanced PackRunStateFactory to correctly initialize state with gate reasons.

src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/Program.cs

@@ -1,5 +1,11 @@
-using System.Text.Json.Nodes;
-using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using StellaOps.TaskRunner.Core.Execution;
 using StellaOps.TaskRunner.Core.Execution.Simulation;
@@ -11,20 +17,52 @@ using StellaOps.TaskRunner.WebService;
 var builder = WebApplication.CreateBuilder(args);
 
 builder.Services.Configure<TaskRunnerServiceOptions>(builder.Configuration.GetSection("TaskRunner"));
-builder.Services.AddSingleton<TaskPackManifestLoader>();
+builder.Services.AddSingleton<TaskPackManifestLoader>();
 builder.Services.AddSingleton<TaskPackPlanner>();
 builder.Services.AddSingleton<PackRunSimulationEngine>();
 builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
-builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
+
+var storageOptions = builder.Configuration.GetSection("TaskRunner:Storage").Get<TaskRunnerStorageOptions>() ?? new TaskRunnerStorageOptions();
+builder.Services.AddSingleton(storageOptions);
+
+if (string.Equals(storageOptions.Mode, TaskRunnerStorageModes.Mongo, StringComparison.OrdinalIgnoreCase))
 {
-    var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
-    return new FilePackRunApprovalStore(options.ApprovalStorePath);
-});
-builder.Services.AddSingleton<IPackRunStateStore>(sp =>
+    builder.Services.AddSingleton(storageOptions.Mongo);
+    builder.Services.AddSingleton<IMongoClient>(_ => new MongoClient(storageOptions.Mongo.ConnectionString));
+    builder.Services.AddSingleton<IMongoDatabase>(sp =>
+    {
+        var mongoOptions = storageOptions.Mongo;
+        var client = sp.GetRequiredService<IMongoClient>();
+        var mongoUrl = MongoUrl.Create(mongoOptions.ConnectionString);
+        var databaseName = !string.IsNullOrWhiteSpace(mongoOptions.Database)
+            ? mongoOptions.Database
+            : mongoUrl.DatabaseName ?? "stellaops-taskrunner";
+        return client.GetDatabase(databaseName);
+    });
+
+    builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
+    builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
+    builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
+}
+else
 {
-    var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
-    return new FilePackRunStateStore(options.RunStatePath);
-});
+    builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
+    {
+        var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
+        return new FilePackRunApprovalStore(options.ApprovalStorePath);
+    });
+    builder.Services.AddSingleton<IPackRunStateStore>(sp =>
+    {
+        var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
+        return new FilePackRunStateStore(options.RunStatePath);
+    });
+    builder.Services.AddSingleton<IPackRunLogStore>(sp =>
+    {
+        var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
+        return new FilePackRunLogStore(options.LogsPath);
+    });
+}
+
 builder.Services.AddSingleton(sp =>
 {
     var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
@@ -77,8 +115,89 @@ app.MapPost("/v1/task-runner/simulations", async (
     var simulation = simulationEngine.Simulate(plan);
     var response = SimulationMapper.ToResponse(plan, simulation);
     return Results.Ok(response);
-}).WithName("SimulateTaskPack");
-
+}).WithName("SimulateTaskPack");
+
+app.MapPost("/v1/task-runner/runs", async (
+    [FromBody] CreateRunRequest request,
+    TaskPackManifestLoader loader,
+    TaskPackPlanner planner,
+    PackRunExecutionGraphBuilder executionGraphBuilder,
+    PackRunSimulationEngine simulationEngine,
+    IPackRunStateStore stateStore,
+    IPackRunLogStore logStore,
+    IPackRunJobScheduler scheduler,
+    CancellationToken cancellationToken) =>
+{
+    if (request is null || string.IsNullOrWhiteSpace(request.Manifest))
+    {
+        return Results.BadRequest(new { error = "Manifest is required." });
+    }
+
+    TaskPackManifest manifest;
+    try
+    {
+        manifest = loader.Deserialize(request.Manifest);
+    }
+    catch (Exception ex)
+    {
+        return Results.BadRequest(new { error = "Invalid manifest", detail = ex.Message });
+    }
+
+    var inputs = ConvertInputs(request.Inputs);
+    var planResult = planner.Plan(manifest, inputs);
+    if (!planResult.Success || planResult.Plan is null)
+    {
+        return Results.BadRequest(new
+        {
+            errors = planResult.Errors.Select(error => new { error.Path, error.Message })
+        });
+    }
+
+    var plan = planResult.Plan;
+    var runId = string.IsNullOrWhiteSpace(request.RunId)
+        ? Guid.NewGuid().ToString("n")
+        : request.RunId!;
+
+    var existing = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
+    if (existing is not null)
+    {
+        return Results.Conflict(new { error = "Run already exists." });
+    }
+
+    var requestedAt = DateTimeOffset.UtcNow;
+    var context = new PackRunExecutionContext(runId, plan, requestedAt);
+    var graph = executionGraphBuilder.Build(plan);
+
+    var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, requestedAt);
+    await stateStore.SaveAsync(state, cancellationToken).ConfigureAwait(false);
+
+    try
+    {
+        await scheduler.ScheduleAsync(context, cancellationToken).ConfigureAwait(false);
+    }
+    catch (Exception ex)
+    {
+        await logStore.AppendAsync(
+            runId,
+            new PackRunLogEntry(DateTimeOffset.UtcNow, "error", "run.schedule-failed", ex.Message, null, null),
+            cancellationToken).ConfigureAwait(false);
+
+        return Results.StatusCode(StatusCodes.Status500InternalServerError);
+    }
+
+    var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
+    metadata["planHash"] = plan.Hash;
+    metadata["requestedAt"] = requestedAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
+
+    await logStore.AppendAsync(
+        runId,
+        new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.created", "Run created via API.", null, metadata),
+        cancellationToken).ConfigureAwait(false);
+
+    var response = RunStateMapper.ToResponse(state);
+    return Results.Created($"/v1/task-runner/runs/{runId}", response);
+}).WithName("CreatePackRun");
+
 app.MapGet("/v1/task-runner/runs/{runId}", async (
     string runId,
     IPackRunStateStore stateStore,
@@ -94,10 +213,34 @@ app.MapGet("/v1/task-runner/runs/{runId}", async (
     {
         return Results.NotFound();
     }
-
+
     return Results.Ok(RunStateMapper.ToResponse(state));
 }).WithName("GetRunState");
 
+app.MapGet("/v1/task-runner/runs/{runId}/logs", async (
+    string runId,
+    IPackRunLogStore logStore,
+    CancellationToken cancellationToken) =>
+{
+    if (string.IsNullOrWhiteSpace(runId))
+    {
+        return Results.BadRequest(new { error = "runId is required." });
+    }
+
+    if (!await logStore.ExistsAsync(runId, cancellationToken).ConfigureAwait(false))
+    {
+        return Results.NotFound();
+    }
+
+    return Results.Stream(async (stream, ct) =>
+    {
+        await foreach (var entry in logStore.ReadAsync(runId, ct).ConfigureAwait(false))
+        {
+            await RunLogMapper.WriteAsync(stream, entry, ct).ConfigureAwait(false);
+        }
+    }, "application/x-ndjson");
+}).WithName("StreamRunLogs");
+
 app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", async (
     string runId,
     string approvalId,
@@ -151,12 +294,14 @@ static IDictionary<string, JsonNode?>? ConvertInputs(JsonObject? node)
     return dictionary;
 }
 
-internal sealed record SimulationRequest(string Manifest, JsonObject? Inputs);
-
-internal sealed record SimulationResponse(
-    string PlanHash,
-    FailurePolicyResponse FailurePolicy,
-    IReadOnlyList<SimulationStepResponse> Steps,
+internal sealed record CreateRunRequest(string? RunId, string Manifest, JsonObject? Inputs);
+
+internal sealed record SimulationRequest(string Manifest, JsonObject? Inputs);
+
+internal sealed record SimulationResponse(
+    string PlanHash,
+    FailurePolicyResponse FailurePolicy,
+    IReadOnlyList<SimulationStepResponse> Steps,
     IReadOnlyList<SimulationOutputResponse> Outputs,
     bool HasPendingApprovals);
 
@@ -206,9 +351,54 @@ internal sealed record RunStateStepResponse(
     string? StatusReason);
 
 internal sealed record ApprovalDecisionDto(string Decision, string? ActorId, string? Summary);
-
-internal static class SimulationMapper
-{
+
+internal sealed record RunLogEntryResponse(
+    DateTimeOffset Timestamp,
+    string Level,
+    string EventType,
+    string Message,
+    string? StepId,
+    IReadOnlyDictionary<string, string>? Metadata);
+
+internal static class RunLogMapper
+{
+    private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
+    {
+        WriteIndented = false
+    };
+
+    private static readonly byte[] NewLine = Encoding.UTF8.GetBytes("\n");
+
+    public static RunLogEntryResponse ToResponse(PackRunLogEntry entry)
+    {
+        IReadOnlyDictionary<string, string>? metadata = null;
+        if (entry.Metadata is { Count: > 0 })
+        {
+            metadata = entry.Metadata
+                .OrderBy(pair => pair.Key, StringComparer.Ordinal)
+                .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
+        }
+
+        return new RunLogEntryResponse(
+            entry.Timestamp,
+            entry.Level,
+            entry.EventType,
+            entry.Message,
+            entry.StepId,
+            metadata);
+    }
+
+    public static async Task WriteAsync(Stream stream, PackRunLogEntry entry, CancellationToken cancellationToken)
+    {
+        var response = ToResponse(entry);
+        await JsonSerializer.SerializeAsync(stream, response, SerializerOptions, cancellationToken).ConfigureAwait(false);
+        await stream.WriteAsync(NewLine, cancellationToken).ConfigureAwait(false);
+        await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
+    }
+}
+
+internal static class SimulationMapper
+{
     public static SimulationResponse ToResponse(TaskPackPlan plan, PackRunSimulationResult result)
     {
         var failurePolicy = result.FailurePolicy ?? PackRunExecutionGraph.DefaultFailurePolicy;

src/TaskRunner/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/TaskRunnerServiceOptions.cs

@@ -1,9 +1,14 @@
-namespace StellaOps.TaskRunner.WebService;
-
+using StellaOps.TaskRunner.Core.Configuration;
+
+namespace StellaOps.TaskRunner.WebService;
+
 public sealed class TaskRunnerServiceOptions
 {
     public string RunStatePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "state", "runs");
     public string ApprovalStorePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "approvals");
     public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
     public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
+    public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
+
+    public TaskRunnerStorageOptions Storage { get; set; } = new();
 }