feat: Implement Filesystem and MongoDB provenance writers for PackRun execution context
- Added `FilesystemPackRunProvenanceWriter` to write provenance manifests to the filesystem. - Introduced `MongoPackRunArtifactReader` to read artifacts from MongoDB. - Created `MongoPackRunProvenanceWriter` to store provenance manifests in MongoDB. - Developed unit tests for filesystem and MongoDB provenance writers. - Established `ITimelineEventStore` and `ITimelineIngestionService` interfaces for timeline event handling. - Implemented `TimelineIngestionService` to validate and persist timeline events with hashing. - Created PostgreSQL schema and migration scripts for timeline indexing. - Added dependency injection support for timeline indexer services. - Developed tests for timeline ingestion and schema validation.
This commit is contained in:
@@ -19,10 +19,13 @@ Execute Task Packs safely and deterministically. Provide remote pack execution,
|
||||
## Required Reading
|
||||
- `docs/modules/platform/architecture.md`
|
||||
- `docs/modules/platform/architecture-overview.md`
|
||||
- `docs/modules/taskrunner/architecture.md`
|
||||
- `docs/product-advisories/29-Nov-2025 - Task Pack Orchestration and Automation.md`
|
||||
- `docs/task-packs/spec.md`, `docs/task-packs/authoring-guide.md`, `docs/task-packs/runbook.md`
|
||||
|
||||
## Working Agreement
|
||||
- 1. Update task status to `DOING`/`DONE` in both correspoding sprint file `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` when you start or finish work.
|
||||
- 2. Review this charter and the Required Reading documents before coding; confirm prerequisites are met.
|
||||
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations.
|
||||
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change.
|
||||
- 3. Keep changes deterministic (stable ordering, timestamps, hashes) and align with offline/air-gap expectations; enforce plan-hash binding for every run.
|
||||
- 4. Coordinate doc updates, tests, and cross-guild communication whenever contracts or workflows change; sync sprint Decisions/Risks when advisory-driven changes land.
|
||||
- 5. Revert to `TODO` if you pause the task without shipping changes; leave notes in commit/PR descriptions for context.
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
public interface IPackRunArtifactReader
|
||||
{
|
||||
Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record PackRunArtifactRecord(
|
||||
string Name,
|
||||
string Type,
|
||||
string? SourcePath,
|
||||
string? StoredPath,
|
||||
string Status,
|
||||
string? Notes,
|
||||
DateTimeOffset CapturedAt,
|
||||
string? ExpressionJson = null);
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
public interface IPackRunProvenanceWriter
|
||||
{
|
||||
Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -2,21 +2,24 @@ using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
public sealed class PackRunExecutionContext
|
||||
{
|
||||
public PackRunExecutionContext(string runId, TaskPackPlan plan, DateTimeOffset requestedAt)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
|
||||
RunId = runId;
|
||||
Plan = plan;
|
||||
RequestedAt = requestedAt;
|
||||
}
|
||||
|
||||
public string RunId { get; }
|
||||
|
||||
public TaskPackPlan Plan { get; }
|
||||
|
||||
public DateTimeOffset RequestedAt { get; }
|
||||
}
|
||||
public sealed class PackRunExecutionContext
|
||||
{
|
||||
public PackRunExecutionContext(string runId, TaskPackPlan plan, DateTimeOffset requestedAt, string? tenantId = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
|
||||
RunId = runId;
|
||||
Plan = plan;
|
||||
RequestedAt = requestedAt;
|
||||
TenantId = string.IsNullOrWhiteSpace(tenantId) ? null : tenantId.Trim();
|
||||
}
|
||||
|
||||
public string RunId { get; }
|
||||
|
||||
public TaskPackPlan Plan { get; }
|
||||
|
||||
public DateTimeOffset RequestedAt { get; }
|
||||
|
||||
public string? TenantId { get; }
|
||||
}
|
||||
|
||||
@@ -11,16 +11,18 @@ public sealed record PackRunState(
|
||||
DateTimeOffset RequestedAt,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset UpdatedAt,
|
||||
IReadOnlyDictionary<string, PackRunStepStateRecord> Steps)
|
||||
{
|
||||
public static PackRunState Create(
|
||||
string runId,
|
||||
IReadOnlyDictionary<string, PackRunStepStateRecord> Steps,
|
||||
string? TenantId = null)
|
||||
{
|
||||
public static PackRunState Create(
|
||||
string runId,
|
||||
string planHash,
|
||||
TaskPackPlan plan,
|
||||
TaskPackPlanFailurePolicy failurePolicy,
|
||||
DateTimeOffset requestedAt,
|
||||
IReadOnlyDictionary<string, PackRunStepStateRecord> steps,
|
||||
DateTimeOffset timestamp)
|
||||
DateTimeOffset timestamp,
|
||||
string? tenantId = null)
|
||||
=> new(
|
||||
runId,
|
||||
planHash,
|
||||
@@ -29,8 +31,9 @@ public sealed record PackRunState(
|
||||
requestedAt,
|
||||
timestamp,
|
||||
timestamp,
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(new Dictionary<string, PackRunStepStateRecord>(steps, StringComparer.Ordinal)));
|
||||
}
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(new Dictionary<string, PackRunStepStateRecord>(steps, StringComparer.Ordinal)),
|
||||
tenantId);
|
||||
}
|
||||
|
||||
public sealed record PackRunStepStateRecord(
|
||||
string StepId,
|
||||
|
||||
@@ -74,7 +74,8 @@ public static class PackRunStateFactory
|
||||
failurePolicy,
|
||||
context.RequestedAt,
|
||||
stepRecords,
|
||||
timestamp);
|
||||
timestamp,
|
||||
context.TenantId);
|
||||
}
|
||||
|
||||
private static Dictionary<string, PackRunSimulationNode> IndexSimulation(IReadOnlyList<PackRunSimulationNode> nodes)
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
public static class ProvenanceManifestFactory
|
||||
{
|
||||
public static ProvenanceManifest Create(PackRunExecutionContext context, PackRunState state, DateTimeOffset completedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var steps = state.Steps.Values
|
||||
.OrderBy(step => step.StepId, StringComparer.Ordinal)
|
||||
.Select(step => new ProvenanceStep(
|
||||
step.StepId,
|
||||
step.Kind.ToString(),
|
||||
step.Status.ToString(),
|
||||
step.Attempts,
|
||||
step.LastTransitionAt,
|
||||
step.StatusReason))
|
||||
.ToList();
|
||||
|
||||
var outputs = context.Plan.Outputs
|
||||
.Select(output => new ProvenanceOutput(output.Name, output.Type))
|
||||
.ToList();
|
||||
|
||||
return new ProvenanceManifest(
|
||||
context.RunId,
|
||||
context.TenantId,
|
||||
context.Plan.Hash,
|
||||
context.Plan.Metadata.Name,
|
||||
context.Plan.Metadata.Version,
|
||||
context.Plan.Metadata.Description,
|
||||
context.Plan.Metadata.Tags,
|
||||
context.RequestedAt,
|
||||
state.CreatedAt,
|
||||
completedAt,
|
||||
steps,
|
||||
outputs);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ProvenanceManifest(
|
||||
string RunId,
|
||||
string? TenantId,
|
||||
string PlanHash,
|
||||
string PackName,
|
||||
string PackVersion,
|
||||
string? PackDescription,
|
||||
IReadOnlyList<string> PackTags,
|
||||
DateTimeOffset RequestedAt,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset CompletedAt,
|
||||
IReadOnlyList<ProvenanceStep> Steps,
|
||||
IReadOnlyList<ProvenanceOutput> Outputs);
|
||||
|
||||
public sealed record ProvenanceStep(
|
||||
string Id,
|
||||
string Kind,
|
||||
string Status,
|
||||
int Attempts,
|
||||
DateTimeOffset? LastTransitionAt,
|
||||
string? StatusReason);
|
||||
|
||||
public sealed record ProvenanceOutput(string Name, string Type);
|
||||
@@ -118,7 +118,8 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
|
||||
DateTimeOffset RequestedAt,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset UpdatedAt,
|
||||
IReadOnlyList<StepDocument> Steps)
|
||||
IReadOnlyList<StepDocument> Steps,
|
||||
string? TenantId)
|
||||
{
|
||||
public static StateDocument FromDomain(PackRunState state)
|
||||
{
|
||||
@@ -147,11 +148,12 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
|
||||
state.RequestedAt,
|
||||
state.CreatedAt,
|
||||
state.UpdatedAt,
|
||||
steps);
|
||||
steps,
|
||||
state.TenantId);
|
||||
}
|
||||
|
||||
public PackRunState ToDomain()
|
||||
{
|
||||
public PackRunState ToDomain()
|
||||
{
|
||||
var steps = Steps.ToDictionary(
|
||||
step => step.StepId,
|
||||
step => new PackRunStepStateRecord(
|
||||
@@ -177,9 +179,10 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
|
||||
RequestedAt,
|
||||
CreatedAt,
|
||||
UpdatedAt,
|
||||
steps);
|
||||
steps,
|
||||
TenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record StepDocument(
|
||||
string StepId,
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
using System.Text.Json;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class FilesystemPackRunArtifactReader : IPackRunArtifactReader
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly string rootPath;
|
||||
|
||||
public FilesystemPackRunArtifactReader(string rootPath)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
|
||||
this.rootPath = Path.GetFullPath(rootPath);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var manifestPath = Path.Combine(rootPath, Sanitize(runId), "artifact-manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return Array.Empty<PackRunArtifactRecord>();
|
||||
}
|
||||
|
||||
await using var stream = File.OpenRead(manifestPath);
|
||||
var manifest = await JsonSerializer.DeserializeAsync<ArtifactManifest>(stream, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (manifest is null || manifest.Outputs is null)
|
||||
{
|
||||
return Array.Empty<PackRunArtifactRecord>();
|
||||
}
|
||||
|
||||
return manifest.Outputs
|
||||
.OrderBy(output => output.Name, StringComparer.Ordinal)
|
||||
.Select(output => new PackRunArtifactRecord(
|
||||
output.Name,
|
||||
output.Type,
|
||||
output.SourcePath,
|
||||
output.StoredPath,
|
||||
output.Status,
|
||||
output.Notes,
|
||||
manifest.UploadedAt,
|
||||
output.ExpressionJson))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string Sanitize(string value)
|
||||
{
|
||||
var safe = value.Trim();
|
||||
foreach (var invalid in Path.GetInvalidFileNameChars())
|
||||
{
|
||||
safe = safe.Replace(invalid, '_');
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(safe) ? "run" : safe;
|
||||
}
|
||||
|
||||
private sealed record ArtifactManifest(
|
||||
string RunId,
|
||||
DateTimeOffset UploadedAt,
|
||||
List<ArtifactRecord> Outputs);
|
||||
|
||||
private sealed record ArtifactRecord(
|
||||
string Name,
|
||||
string Type,
|
||||
string? SourcePath,
|
||||
string? StoredPath,
|
||||
string Status,
|
||||
string? Notes,
|
||||
string? ExpressionJson);
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
using System.Text.Json;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class FilesystemPackRunProvenanceWriter : IPackRunProvenanceWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly string rootPath;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public FilesystemPackRunProvenanceWriter(string rootPath, TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
|
||||
|
||||
this.rootPath = Path.GetFullPath(rootPath);
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var completedAt = timeProvider.GetUtcNow();
|
||||
var manifest = ProvenanceManifestFactory.Create(context, state, completedAt);
|
||||
var manifestPath = GetPath(context.RunId);
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(manifestPath)!);
|
||||
|
||||
await using var stream = File.Open(manifestPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
await JsonSerializer.SerializeAsync(stream, manifest, SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private string GetPath(string runId)
|
||||
{
|
||||
var safe = Sanitize(runId);
|
||||
return Path.Combine(rootPath, "provenance", $"{safe}.json");
|
||||
}
|
||||
|
||||
private static string Sanitize(string value)
|
||||
{
|
||||
var result = value.Trim();
|
||||
foreach (var invalid in Path.GetInvalidFileNameChars())
|
||||
{
|
||||
result = result.Replace(invalid, '_');
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -82,25 +82,25 @@ public sealed class MongoPackRunApprovalStore : IPackRunApprovalStore
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
|
||||
public static IEnumerable<CreateIndexModel<PackRunApprovalDocument>> GetIndexModels()
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.ApprovalId),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status))
|
||||
};
|
||||
yield return new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.ApprovalId),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_run_approvals_run_approval" });
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
yield return new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status),
|
||||
new CreateIndexOptions { Name = "pack_run_approvals_run_status" });
|
||||
}
|
||||
|
||||
private sealed class PackRunApprovalDocument
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunApprovalDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunArtifactReader : IPackRunArtifactReader
|
||||
{
|
||||
private readonly IMongoCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument> collection;
|
||||
|
||||
public MongoPackRunArtifactReader(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(options.ArtifactsCollection);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Eq(doc => doc.RunId, runId);
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.SortBy(doc => doc.Name)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(doc => new PackRunArtifactRecord(
|
||||
doc.Name,
|
||||
doc.Type,
|
||||
doc.SourcePath,
|
||||
doc.StoredPath,
|
||||
doc.Status,
|
||||
doc.Notes,
|
||||
new DateTimeOffset(doc.CapturedAt, TimeSpan.Zero),
|
||||
doc.Expression?.ToJson()))
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
@@ -149,24 +149,23 @@ public sealed class MongoPackRunArtifactUploader : IPackRunArtifactUploader
|
||||
return parameter.Value;
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
|
||||
public static IEnumerable<CreateIndexModel<PackRunArtifactDocument>> GetIndexModels()
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Name),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status))
|
||||
};
|
||||
yield return new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Name),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_artifacts_run_name" });
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
yield return new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId),
|
||||
new CreateIndexOptions { Name = "pack_artifacts_run" });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunArtifactDocument
|
||||
{
|
||||
[BsonId]
|
||||
|
||||
@@ -89,24 +89,24 @@ public sealed class MongoPackRunLogStore : IPackRunLogStore
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
|
||||
public static IEnumerable<CreateIndexModel<PackRunLogDocument>> GetIndexModels()
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Sequence),
|
||||
new CreateIndexOptions { Unique = true }),
|
||||
new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Timestamp))
|
||||
};
|
||||
yield return new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Sequence),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_run_logs_run_sequence" });
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
yield return new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Timestamp),
|
||||
new CreateIndexOptions { Name = "pack_run_logs_run_timestamp" });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunLogDocument
|
||||
{
|
||||
[BsonId]
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunProvenanceWriter : IPackRunProvenanceWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<ProvenanceDocument> collection;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public MongoPackRunProvenanceWriter(IMongoDatabase database, TaskRunnerMongoOptions options, TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<ProvenanceDocument>(options.ArtifactsCollection);
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var completedAt = timeProvider.GetUtcNow();
|
||||
var manifest = ProvenanceManifestFactory.Create(context, state, completedAt);
|
||||
var manifestJson = JsonSerializer.Serialize(manifest, SerializerOptions);
|
||||
var manifestDocument = BsonDocument.Parse(manifestJson);
|
||||
|
||||
var document = new ProvenanceDocument
|
||||
{
|
||||
RunId = context.RunId,
|
||||
Name = "provenance-manifest",
|
||||
Type = "object",
|
||||
Status = "materialized",
|
||||
CapturedAt = completedAt.UtcDateTime,
|
||||
Expression = manifestDocument
|
||||
};
|
||||
|
||||
var filter = Builders<ProvenanceDocument>.Filter.And(
|
||||
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.RunId, context.RunId),
|
||||
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.Name, document.Name));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private sealed class ProvenanceDocument
|
||||
{
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string Name { get; init; } = default!;
|
||||
|
||||
public string Type { get; init; } = default!;
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public DateTime CapturedAt { get; init; }
|
||||
|
||||
public BsonDocument Expression { get; init; } = default!;
|
||||
}
|
||||
}
|
||||
@@ -62,20 +62,23 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
|
||||
public static IEnumerable<CreateIndexModel<PackRunStateDocument>> GetIndexModels()
|
||||
{
|
||||
var models = new[]
|
||||
{
|
||||
new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt)),
|
||||
new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Ascending(document => document.PlanHash))
|
||||
};
|
||||
yield return new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt),
|
||||
new CreateIndexOptions { Name = "pack_runs_updatedAt_desc" });
|
||||
|
||||
target.Indexes.CreateMany(models);
|
||||
yield return new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys
|
||||
.Ascending(document => document.TenantId)
|
||||
.Descending(document => document.UpdatedAt),
|
||||
new CreateIndexOptions { Name = "pack_runs_tenant_updatedAt_desc", Sparse = true });
|
||||
}
|
||||
|
||||
private sealed class PackRunStateDocument
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunStateDocument
|
||||
{
|
||||
[BsonId]
|
||||
public string RunId { get; init; } = default!;
|
||||
@@ -94,6 +97,8 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
|
||||
public List<PackRunStepDocument> Steps { get; init; } = new();
|
||||
|
||||
public string? TenantId { get; init; }
|
||||
|
||||
public static PackRunStateDocument FromDomain(PackRunState state)
|
||||
{
|
||||
var planDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.Plan, SerializerOptions));
|
||||
@@ -113,7 +118,8 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
RequestedAt = state.RequestedAt.UtcDateTime,
|
||||
CreatedAt = state.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = state.UpdatedAt.UtcDateTime,
|
||||
Steps = steps
|
||||
Steps = steps,
|
||||
TenantId = state.TenantId
|
||||
};
|
||||
}
|
||||
|
||||
@@ -139,11 +145,12 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(CreatedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(UpdatedAt, TimeSpan.Zero),
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords));
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords),
|
||||
TenantId);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PackRunStepDocument
|
||||
public sealed class PackRunStepDocument
|
||||
{
|
||||
public string StepId { get; init; } = default!;
|
||||
|
||||
|
||||
@@ -48,6 +48,16 @@ public sealed class PackRunApprovalDecisionService
|
||||
return PackRunApprovalDecisionResult.NotFound;
|
||||
}
|
||||
|
||||
if (!string.Equals(state.PlanHash, request.PlanHash, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Approval decision for run {RunId} rejected – plan hash mismatch (expected {Expected}, got {Actual}).",
|
||||
runId,
|
||||
state.PlanHash,
|
||||
request.PlanHash);
|
||||
return PackRunApprovalDecisionResult.PlanHashMismatch;
|
||||
}
|
||||
|
||||
var requestedAt = state.RequestedAt != default ? state.RequestedAt : state.CreatedAt;
|
||||
var coordinator = PackRunApprovalCoordinator.Restore(state.Plan, approvals, requestedAt);
|
||||
|
||||
@@ -96,6 +106,7 @@ public sealed class PackRunApprovalDecisionService
|
||||
public sealed record PackRunApprovalDecisionRequest(
|
||||
string RunId,
|
||||
string ApprovalId,
|
||||
string PlanHash,
|
||||
PackRunApprovalDecisionType Decision,
|
||||
string? ActorId,
|
||||
string? Summary);
|
||||
@@ -110,6 +121,7 @@ public enum PackRunApprovalDecisionType
|
||||
public sealed record PackRunApprovalDecisionResult(string Status)
|
||||
{
|
||||
public static PackRunApprovalDecisionResult NotFound { get; } = new("not_found");
|
||||
public static PackRunApprovalDecisionResult PlanHashMismatch { get; } = new("plan_hash_mismatch");
|
||||
public static PackRunApprovalDecisionResult Applied { get; } = new("applied");
|
||||
public static PackRunApprovalDecisionResult Resumed { get; } = new("resumed");
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ public sealed class PackRunApprovalDecisionServiceTests
|
||||
NullLogger<PackRunApprovalDecisionService>.Instance);
|
||||
|
||||
var result = await service.ApplyAsync(
|
||||
new PackRunApprovalDecisionRequest("run-1", "security-review", PackRunApprovalDecisionType.Approved, "approver@example.com", "LGTM"),
|
||||
new PackRunApprovalDecisionRequest("run-1", "security-review", plan.Hash, PackRunApprovalDecisionType.Approved, "approver@example.com", "LGTM"),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("resumed", result.Status);
|
||||
@@ -62,13 +62,51 @@ public sealed class PackRunApprovalDecisionServiceTests
|
||||
NullLogger<PackRunApprovalDecisionService>.Instance);
|
||||
|
||||
var result = await service.ApplyAsync(
|
||||
new PackRunApprovalDecisionRequest("missing", "approval", PackRunApprovalDecisionType.Approved, "actor", null),
|
||||
new PackRunApprovalDecisionRequest("missing", "approval", "hash", PackRunApprovalDecisionType.Approved, "actor", null),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("not_found", result.Status);
|
||||
Assert.False(scheduler.ScheduledContexts.Any());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyAsync_ReturnsPlanHashMismatchWhenIncorrect()
|
||||
{
|
||||
var plan = TestPlanFactory.CreatePlan();
|
||||
var state = TestPlanFactory.CreateState("run-1", plan);
|
||||
var approval = new PackRunApprovalState(
|
||||
"security-review",
|
||||
new[] { "Packs.Approve" },
|
||||
new[] { "step-a" },
|
||||
Array.Empty<string>(),
|
||||
null,
|
||||
DateTimeOffset.UtcNow.AddMinutes(-5),
|
||||
PackRunApprovalStatus.Pending);
|
||||
|
||||
var approvalStore = new InMemoryApprovalStore(new Dictionary<string, IReadOnlyList<PackRunApprovalState>>
|
||||
{
|
||||
["run-1"] = new List<PackRunApprovalState> { approval }
|
||||
});
|
||||
var stateStore = new InMemoryStateStore(new Dictionary<string, PackRunState>
|
||||
{
|
||||
["run-1"] = state
|
||||
});
|
||||
var scheduler = new RecordingScheduler();
|
||||
|
||||
var service = new PackRunApprovalDecisionService(
|
||||
approvalStore,
|
||||
stateStore,
|
||||
scheduler,
|
||||
NullLogger<PackRunApprovalDecisionService>.Instance);
|
||||
|
||||
var result = await service.ApplyAsync(
|
||||
new PackRunApprovalDecisionRequest("run-1", "security-review", "wrong-hash", PackRunApprovalDecisionType.Approved, "actor", null),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal("plan_hash_mismatch", result.Status);
|
||||
Assert.False(scheduler.ScheduledContexts.Any());
|
||||
}
|
||||
|
||||
private sealed class InMemoryApprovalStore : IPackRunApprovalStore
|
||||
{
|
||||
private readonly Dictionary<string, List<PackRunApprovalState>> _approvals;
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
using StellaOps.TaskRunner.Core.TaskPacks;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class PackRunProvenanceWriterTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Filesystem_writer_emits_manifest()
|
||||
{
|
||||
var (context, state) = CreateRunState();
|
||||
var completedAt = new DateTimeOffset(2025, 11, 30, 12, 30, 0, TimeSpan.Zero);
|
||||
var temp = Directory.CreateTempSubdirectory();
|
||||
try
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var writer = new FilesystemPackRunProvenanceWriter(temp.FullName, new FixedTimeProvider(completedAt));
|
||||
await writer.WriteAsync(context, state, ct);
|
||||
|
||||
var path = Path.Combine(temp.FullName, "provenance", "run-test.json");
|
||||
Assert.True(File.Exists(path));
|
||||
|
||||
using var document = JsonDocument.Parse(await File.ReadAllTextAsync(path, ct));
|
||||
var root = document.RootElement;
|
||||
Assert.Equal("run-test", root.GetProperty("runId").GetString());
|
||||
Assert.Equal("tenant-alpha", root.GetProperty("tenantId").GetString());
|
||||
Assert.Equal(context.Plan.Hash, root.GetProperty("planHash").GetString());
|
||||
Assert.Equal(completedAt, root.GetProperty("completedAt").GetDateTimeOffset());
|
||||
}
|
||||
finally
|
||||
{
|
||||
temp.Delete(recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Mongo_writer_upserts_manifest()
|
||||
{
|
||||
await using var mongo = MongoTaskRunnerTestContext.Create();
|
||||
var (context, state) = CreateRunState();
|
||||
var completedAt = new DateTimeOffset(2025, 11, 30, 12, 0, 0, TimeSpan.Zero);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
var options = mongo.CreateMongoOptions();
|
||||
var writer = new MongoPackRunProvenanceWriter(mongo.Database, options, new FixedTimeProvider(completedAt));
|
||||
await writer.WriteAsync(context, state, ct);
|
||||
|
||||
var collection = mongo.Database.GetCollection<MongoDB.Bson.BsonDocument>(options.ArtifactsCollection);
|
||||
var saved = await collection
|
||||
.Find(Builders<MongoDB.Bson.BsonDocument>.Filter.Eq("RunId", context.RunId))
|
||||
.FirstOrDefaultAsync(ct);
|
||||
|
||||
Assert.NotNull(saved);
|
||||
var manifest = saved!["Expression"].AsBsonDocument;
|
||||
Assert.Equal("run-test", manifest["runId"].AsString);
|
||||
Assert.Equal("tenant-alpha", manifest["tenantId"].AsString);
|
||||
Assert.Equal(context.Plan.Hash, manifest["planHash"].AsString);
|
||||
}
|
||||
|
||||
private static (PackRunExecutionContext Context, PackRunState State) CreateRunState()
|
||||
{
|
||||
var loader = new TaskPackManifestLoader();
|
||||
var planner = new TaskPackPlanner();
|
||||
var manifest = loader.Deserialize(TestManifests.Sample);
|
||||
var plan = planner.Plan(manifest, new Dictionary<string, JsonNode?>()).Plan ?? throw new InvalidOperationException("Plan generation failed.");
|
||||
|
||||
var graphBuilder = new PackRunExecutionGraphBuilder();
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
var graph = graphBuilder.Build(plan);
|
||||
|
||||
var requestedAt = new DateTimeOffset(2025, 11, 30, 10, 0, 0, TimeSpan.Zero);
|
||||
var context = new PackRunExecutionContext("run-test", plan, requestedAt, "tenant-alpha");
|
||||
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, requestedAt);
|
||||
return (context, state);
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
{
|
||||
private readonly DateTimeOffset now;
|
||||
|
||||
public FixedTimeProvider(DateTimeOffset now)
|
||||
{
|
||||
this.now = now;
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => now;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,10 @@
|
||||
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="..\StellaOps.TaskRunner.WebService\OpenApiMetadataFactory.cs" Link="Web/OpenApiMetadataFactory.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Driver;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
@@ -22,6 +25,7 @@ builder.Services.AddSingleton<TaskPackManifestLoader>();
|
||||
builder.Services.AddSingleton<TaskPackPlanner>();
|
||||
builder.Services.AddSingleton<PackRunSimulationEngine>();
|
||||
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
|
||||
builder.Services.AddEndpointsApiExplorer();
|
||||
builder.Services.AddStellaOpsTelemetry(
|
||||
builder.Configuration,
|
||||
serviceName: "StellaOps.TaskRunner.WebService",
|
||||
@@ -52,6 +56,7 @@ if (string.Equals(storageOptions.Mode, TaskRunnerStorageModes.Mongo, StringCompa
|
||||
builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactReader, MongoPackRunArtifactReader>();
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -70,6 +75,11 @@ else
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunLogStore(options.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactReader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilesystemPackRunArtifactReader(options.ArtifactsPath);
|
||||
});
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(sp =>
|
||||
@@ -83,10 +93,7 @@ builder.Services.AddOpenApi();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
if (app.Environment.IsDevelopment())
|
||||
{
|
||||
app.MapOpenApi();
|
||||
}
|
||||
app.MapOpenApi("/openapi");
|
||||
|
||||
app.MapPost("/v1/task-runner/simulations", async (
|
||||
[FromBody] SimulationRequest request,
|
||||
@@ -126,7 +133,35 @@ app.MapPost("/v1/task-runner/simulations", async (
|
||||
return Results.Ok(response);
|
||||
}).WithName("SimulateTaskPack");
|
||||
|
||||
app.MapPost("/v1/task-runner/runs", async (
|
||||
app.MapPost("/v1/task-runner/runs", HandleCreateRun).WithName("CreatePackRun");
|
||||
app.MapPost("/api/runs", HandleCreateRun).WithName("CreatePackRunApi");
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}", HandleGetRunState).WithName("GetRunState");
|
||||
app.MapGet("/api/runs/{runId}", HandleGetRunState).WithName("GetRunStateApi");
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}/logs", HandleStreamRunLogs).WithName("StreamRunLogs");
|
||||
app.MapGet("/api/runs/{runId}/logs", HandleStreamRunLogs).WithName("StreamRunLogsApi");
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}/artifacts", HandleListArtifacts).WithName("ListRunArtifacts");
|
||||
app.MapGet("/api/runs/{runId}/artifacts", HandleListArtifacts).WithName("ListRunArtifactsApi");
|
||||
|
||||
app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", HandleApplyApprovalDecision).WithName("ApplyApprovalDecision");
|
||||
app.MapPost("/api/runs/{runId}/approvals/{approvalId}", HandleApplyApprovalDecision).WithName("ApplyApprovalDecisionApi");
|
||||
|
||||
app.MapPost("/v1/task-runner/runs/{runId}/cancel", HandleCancelRun).WithName("CancelRun");
|
||||
app.MapPost("/api/runs/{runId}/cancel", HandleCancelRun).WithName("CancelRunApi");
|
||||
|
||||
app.MapGet("/.well-known/openapi", (HttpResponse response) =>
|
||||
{
|
||||
var metadata = OpenApiMetadataFactory.Create("/openapi");
|
||||
response.Headers.ETag = metadata.ETag;
|
||||
response.Headers.Append("X-Signature", metadata.Signature);
|
||||
return Results.Ok(metadata);
|
||||
}).WithName("GetOpenApiMetadata");
|
||||
|
||||
app.MapGet("/", () => Results.Redirect("/openapi"));
|
||||
|
||||
async Task<IResult> HandleCreateRun(
|
||||
[FromBody] CreateRunRequest request,
|
||||
TaskPackManifestLoader loader,
|
||||
TaskPackPlanner planner,
|
||||
@@ -135,7 +170,7 @@ app.MapPost("/v1/task-runner/runs", async (
|
||||
IPackRunStateStore stateStore,
|
||||
IPackRunLogStore logStore,
|
||||
IPackRunJobScheduler scheduler,
|
||||
CancellationToken cancellationToken) =>
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (request is null || string.IsNullOrWhiteSpace(request.Manifest))
|
||||
{
|
||||
@@ -174,7 +209,7 @@ app.MapPost("/v1/task-runner/runs", async (
|
||||
}
|
||||
|
||||
var requestedAt = DateTimeOffset.UtcNow;
|
||||
var context = new PackRunExecutionContext(runId, plan, requestedAt);
|
||||
var context = new PackRunExecutionContext(runId, plan, requestedAt, request.TenantId);
|
||||
var graph = executionGraphBuilder.Build(plan);
|
||||
|
||||
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, requestedAt);
|
||||
@@ -194,9 +229,15 @@ app.MapPost("/v1/task-runner/runs", async (
|
||||
return Results.StatusCode(StatusCodes.Status500InternalServerError);
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
metadata["planHash"] = plan.Hash;
|
||||
metadata["requestedAt"] = requestedAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["planHash"] = plan.Hash,
|
||||
["requestedAt"] = requestedAt.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(context.TenantId))
|
||||
{
|
||||
metadata["tenantId"] = context.TenantId!;
|
||||
}
|
||||
|
||||
await logStore.AppendAsync(
|
||||
runId,
|
||||
@@ -205,31 +246,31 @@ app.MapPost("/v1/task-runner/runs", async (
|
||||
|
||||
var response = RunStateMapper.ToResponse(state);
|
||||
return Results.Created($"/v1/task-runner/runs/{runId}", response);
|
||||
}).WithName("CreatePackRun");
|
||||
}
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}", async (
|
||||
async Task<IResult> HandleGetRunState(
|
||||
string runId,
|
||||
IPackRunStateStore stateStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "runId is required." });
|
||||
}
|
||||
|
||||
var state = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "runId is required." });
|
||||
}
|
||||
|
||||
var state = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
return Results.Ok(RunStateMapper.ToResponse(state));
|
||||
}).WithName("GetRunState");
|
||||
}
|
||||
|
||||
app.MapGet("/v1/task-runner/runs/{runId}/logs", async (
|
||||
async Task<IResult> HandleStreamRunLogs(
|
||||
string runId,
|
||||
IPackRunLogStore logStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
@@ -248,14 +289,14 @@ app.MapGet("/v1/task-runner/runs/{runId}/logs", async (
|
||||
await RunLogMapper.WriteAsync(stream, entry, ct).ConfigureAwait(false);
|
||||
}
|
||||
}, "application/x-ndjson");
|
||||
}).WithName("StreamRunLogs");
|
||||
}
|
||||
|
||||
app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", async (
|
||||
async Task<IResult> HandleApplyApprovalDecision(
|
||||
string runId,
|
||||
string approvalId,
|
||||
[FromBody] ApprovalDecisionDto request,
|
||||
PackRunApprovalDecisionService decisionService,
|
||||
CancellationToken cancellationToken) =>
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
@@ -267,8 +308,13 @@ app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", async (
|
||||
return Results.BadRequest(new { error = "Invalid decision. Expected approved, rejected, or expired." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.PlanHash))
|
||||
{
|
||||
return Results.BadRequest(new { error = "planHash is required." });
|
||||
}
|
||||
|
||||
var result = await decisionService.ApplyAsync(
|
||||
new PackRunApprovalDecisionRequest(runId, approvalId, decisionType, request.ActorId, request.Summary),
|
||||
new PackRunApprovalDecisionRequest(runId, approvalId, request.PlanHash, decisionType, request.ActorId, request.Summary),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (ReferenceEquals(result, PackRunApprovalDecisionResult.NotFound))
|
||||
@@ -276,18 +322,105 @@ app.MapPost("/v1/task-runner/runs/{runId}/approvals/{approvalId}", async (
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
if (ReferenceEquals(result, PackRunApprovalDecisionResult.PlanHashMismatch))
|
||||
{
|
||||
return Results.Conflict(new { error = "Plan hash mismatch." });
|
||||
}
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
status = result.Status,
|
||||
resumed = result.ShouldResume
|
||||
});
|
||||
}).WithName("ApplyApprovalDecision");
|
||||
}
|
||||
|
||||
app.MapGet("/", () => Results.Redirect("/openapi"));
|
||||
|
||||
app.Run();
|
||||
|
||||
static IDictionary<string, JsonNode?>? ConvertInputs(JsonObject? node)
|
||||
async Task<IResult> HandleListArtifacts(
|
||||
string runId,
|
||||
IPackRunStateStore stateStore,
|
||||
IPackRunArtifactReader artifactReader,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "runId is required." });
|
||||
}
|
||||
|
||||
var state = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var artifacts = await artifactReader.ListAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
var response = artifacts
|
||||
.Select(artifact => new
|
||||
{
|
||||
artifact.Name,
|
||||
artifact.Type,
|
||||
artifact.SourcePath,
|
||||
artifact.StoredPath,
|
||||
artifact.Status,
|
||||
artifact.Notes,
|
||||
artifact.CapturedAt,
|
||||
artifact.ExpressionJson
|
||||
})
|
||||
.ToList();
|
||||
|
||||
return Results.Ok(response);
|
||||
}
|
||||
|
||||
async Task<IResult> HandleCancelRun(
|
||||
string runId,
|
||||
IPackRunStateStore stateStore,
|
||||
IPackRunLogStore logStore,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(runId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "runId is required." });
|
||||
}
|
||||
|
||||
var state = await stateStore.GetAsync(runId, cancellationToken).ConfigureAwait(false);
|
||||
if (state is null)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var updatedSteps = state.Steps.Values
|
||||
.Select(step => step.Status is PackRunStepExecutionStatus.Succeeded or PackRunStepExecutionStatus.Skipped
|
||||
? step
|
||||
: step with
|
||||
{
|
||||
Status = PackRunStepExecutionStatus.Skipped,
|
||||
StatusReason = "cancelled",
|
||||
LastTransitionAt = now,
|
||||
NextAttemptAt = null
|
||||
})
|
||||
.ToDictionary(step => step.StepId, step => step, StringComparer.Ordinal);
|
||||
|
||||
var updatedState = state with
|
||||
{
|
||||
UpdatedAt = now,
|
||||
Steps = new ReadOnlyDictionary<string, PackRunStepStateRecord>(updatedSteps)
|
||||
};
|
||||
|
||||
await stateStore.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["planHash"] = state.PlanHash
|
||||
};
|
||||
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(now, "warn", "run.cancel-requested", "Run cancellation requested.", null, metadata), cancellationToken).ConfigureAwait(false);
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.cancelled", "Run cancelled; remaining steps marked as skipped.", null, metadata), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return Results.Accepted($"/v1/task-runner/runs/{runId}", new { status = "cancelled" });
|
||||
}
|
||||
|
||||
app.Run();
|
||||
|
||||
static IDictionary<string, JsonNode?>? ConvertInputs(JsonObject? node)
|
||||
{
|
||||
if (node is null)
|
||||
{
|
||||
@@ -303,7 +436,7 @@ static IDictionary<string, JsonNode?>? ConvertInputs(JsonObject? node)
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
internal sealed record CreateRunRequest(string? RunId, string Manifest, JsonObject? Inputs);
|
||||
internal sealed record CreateRunRequest(string? RunId, string Manifest, JsonObject? Inputs, string? TenantId);
|
||||
|
||||
internal sealed record SimulationRequest(string Manifest, JsonObject? Inputs);
|
||||
|
||||
@@ -359,7 +492,7 @@ internal sealed record RunStateStepResponse(
|
||||
DateTimeOffset? NextAttemptAt,
|
||||
string? StatusReason);
|
||||
|
||||
internal sealed record ApprovalDecisionDto(string Decision, string? ActorId, string? Summary);
|
||||
internal sealed record ApprovalDecisionDto(string Decision, string PlanHash, string? ActorId, string? Summary);
|
||||
|
||||
internal sealed record RunLogEntryResponse(
|
||||
DateTimeOffset Timestamp,
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj"/>
|
||||
|
||||
<ProjectReference Include="..\..\Telemetry\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core.csproj"/>
|
||||
<ProjectReference Include="..\..\..\Telemetry\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core.csproj"/>
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ public sealed class TaskRunnerServiceOptions
|
||||
public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
|
||||
public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts");
|
||||
|
||||
public TaskRunnerStorageOptions Storage { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -73,6 +73,13 @@ if (string.Equals(workerStorageOptions.Mode, TaskRunnerStorageModes.Mongo, Strin
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader, MongoPackRunArtifactUploader>();
|
||||
builder.Services.AddSingleton<IPackRunProvenanceWriter>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var options = sp.GetRequiredService<TaskRunnerMongoOptions>();
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
return new MongoPackRunProvenanceWriter(db, options, timeProvider);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -98,6 +105,12 @@ else
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunProvenanceWriter>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
return new FilesystemPackRunProvenanceWriter(options.ArtifactsPath, timeProvider);
|
||||
});
|
||||
}
|
||||
|
||||
builder.Services.AddHostedService<PackRunWorkerService>();
|
||||
|
||||
@@ -24,6 +24,7 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
private readonly PackRunSimulationEngine simulationEngine;
|
||||
private readonly IPackRunStepExecutor executor;
|
||||
private readonly IPackRunArtifactUploader artifactUploader;
|
||||
private readonly IPackRunProvenanceWriter provenanceWriter;
|
||||
private readonly IPackRunLogStore logStore;
|
||||
private readonly ILogger<PackRunWorkerService> logger;
|
||||
private readonly UpDownCounter<long> runningSteps;
|
||||
@@ -36,17 +37,19 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
PackRunSimulationEngine simulationEngine,
|
||||
IPackRunStepExecutor executor,
|
||||
IPackRunArtifactUploader artifactUploader,
|
||||
IPackRunProvenanceWriter provenanceWriter,
|
||||
IPackRunLogStore logStore,
|
||||
IOptions<PackRunWorkerOptions> options,
|
||||
ILogger<PackRunWorkerService> logger)
|
||||
{
|
||||
this.dispatcher = dispatcher ?? throw new ArgumentNullException(nameof(dispatcher));
|
||||
this.processor = processor ?? throw new ArgumentNullException(nameof(processor));
|
||||
this.stateStore = stateStore ?? throw new ArgumentNullException(nameof(stateStore));
|
||||
this.stateStore = stateStore ?? throw new ArgumentNullException(nameof(stateStore));
|
||||
this.graphBuilder = graphBuilder ?? throw new ArgumentNullException(nameof(graphBuilder));
|
||||
this.simulationEngine = simulationEngine ?? throw new ArgumentNullException(nameof(simulationEngine));
|
||||
this.executor = executor ?? throw new ArgumentNullException(nameof(executor));
|
||||
this.artifactUploader = artifactUploader ?? throw new ArgumentNullException(nameof(artifactUploader));
|
||||
this.provenanceWriter = provenanceWriter ?? throw new ArgumentNullException(nameof(provenanceWriter));
|
||||
this.logStore = logStore ?? throw new ArgumentNullException(nameof(logStore));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
@@ -165,6 +168,7 @@ public sealed class PackRunWorkerService : BackgroundService
|
||||
"Run finished successfully.",
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await artifactUploader.UploadAsync(context, updatedState, context.Plan.Outputs, cancellationToken).ConfigureAwait(false);
|
||||
await provenanceWriter.WriteAsync(context, updatedState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj"/>
|
||||
|
||||
<ProjectReference Include="..\..\Telemetry\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core.csproj"/>
|
||||
<ProjectReference Include="..\..\..\Telemetry\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core.csproj"/>
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -2,16 +2,16 @@
|
||||
|
||||
| Task ID | Status | Sprint | Dependency | Notes |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TASKRUN-41-001 | BLOCKED | SPRINT_0157_0001_0001_taskrunner_i | — | Blocked: TaskRunner architecture/API contracts and Sprint 120/130/140 inputs not published. |
|
||||
| TASKRUN-AIRGAP-56-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Sealed-mode plan validation; depends on 41-001. |
|
||||
| TASKRUN-AIRGAP-56-002 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-001 | Bundle ingestion helpers; depends on 56-001. |
|
||||
| TASKRUN-AIRGAP-57-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-002 | Sealed install enforcement; depends on 56-002. |
|
||||
| TASKRUN-AIRGAP-58-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-57-001 | Evidence bundles for imports; depends on 57-001. |
|
||||
| TASKRUN-41-001 | DONE (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | — | Implemented run API, Mongo/file stores, approvals, provenance manifest per architecture contract. |
|
||||
| TASKRUN-AIRGAP-56-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Sealed-mode plan validation; depends on 41-001. |
|
||||
| TASKRUN-AIRGAP-56-002 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-001 | Bundle ingestion helpers; depends on 56-001. |
|
||||
| TASKRUN-AIRGAP-57-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-002 | Sealed install enforcement; depends on 56-002. |
|
||||
| TASKRUN-AIRGAP-58-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-57-001 | Evidence bundles for imports; depends on 57-001. |
|
||||
| TASKRUN-42-001 | BLOCKED (2025-11-25) | SPRINT_0157_0001_0001_taskrunner_i | — | Execution engine enhancements (loops/conditionals/maxParallel), simulation mode, policy gate integration. Blocked: loop/conditional semantics and policy-gate evaluation contract not published. |
|
||||
| TASKRUN-OAS-61-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Document APIs; depends on 41-001. |
|
||||
| TASKRUN-OAS-61-002 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-61-001 | Well-known OpenAPI endpoint; depends on 61-001. |
|
||||
| TASKRUN-OAS-62-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-61-002 | SDK examples; depends on 61-002. |
|
||||
| TASKRUN-OAS-63-001 | TODO | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-62-001 | Deprecation headers/notifications; depends on 62-001. |
|
||||
| TASKRUN-OAS-61-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Document APIs; depends on 41-001. |
|
||||
| TASKRUN-OAS-61-002 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-61-001 | Well-known OpenAPI endpoint; depends on 61-001. |
|
||||
| TASKRUN-OAS-62-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-61-002 | SDK examples; depends on 61-002. |
|
||||
| TASKRUN-OAS-63-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OAS-62-001 | Deprecation headers/notifications; depends on 62-001. |
|
||||
| TASKRUN-OBS-50-001 | DONE (2025-11-25) | SPRINT_0157_0001_0001_taskrunner_i | — | Telemetry core adoption. |
|
||||
| TASKRUN-OBS-51-001 | DONE (2025-11-25) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OBS-50-001 | Metrics/SLOs; depends on 50-001. |
|
||||
| TASKRUN-OBS-52-001 | BLOCKED (2025-11-25) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OBS-51-001 | Timeline events; blocked: schema/evidence-pointer contract not published. |
|
||||
|
||||
Reference in New Issue
Block a user