up
Some checks failed
Some checks failed
This commit is contained in:
@@ -20,6 +20,4 @@ public sealed class PackRunWorkerOptions
|
||||
public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts");
|
||||
|
||||
public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
|
||||
|
||||
public TaskRunnerStorageOptions Storage { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.TaskRunner.Core.Configuration;
|
||||
|
||||
public static class TaskRunnerStorageModes
|
||||
{
|
||||
public const string Filesystem = "filesystem";
|
||||
public const string Mongo = "mongo";
|
||||
}
|
||||
|
||||
public sealed class TaskRunnerStorageOptions
|
||||
{
|
||||
public string Mode { get; set; } = TaskRunnerStorageModes.Filesystem;
|
||||
|
||||
public TaskRunnerMongoOptions Mongo { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class TaskRunnerMongoOptions
|
||||
{
|
||||
public string ConnectionString { get; set; } = "mongodb://127.0.0.1:27017/stellaops-taskrunner";
|
||||
|
||||
public string? Database { get; set; }
|
||||
|
||||
public string RunsCollection { get; set; } = "pack_runs";
|
||||
|
||||
public string LogsCollection { get; set; } = "pack_run_logs";
|
||||
|
||||
public string ArtifactsCollection { get; set; } = "pack_artifacts";
|
||||
|
||||
public string ApprovalsCollection { get; set; } = "pack_run_approvals";
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunApprovalStore : IPackRunApprovalStore
|
||||
{
|
||||
private readonly IMongoCollection<PackRunApprovalDocument> collection;
|
||||
|
||||
public MongoPackRunApprovalStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunApprovalDocument>(options.ApprovalsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task SaveAsync(string runId, IReadOnlyList<PackRunApprovalState> approvals, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(approvals);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (approvals.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var documents = approvals
|
||||
.Select(approval => PackRunApprovalDocument.FromDomain(runId, approval))
|
||||
.ToList();
|
||||
|
||||
await collection.InsertManyAsync(documents, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunApprovalState>> GetAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.SortBy(document => document.ApprovalId)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(document => document.ToDomain())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public async Task UpdateAsync(string runId, PackRunApprovalState approval, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(approval);
|
||||
|
||||
var filter = Builders<PackRunApprovalDocument>.Filter.And(
|
||||
Builders<PackRunApprovalDocument>.Filter.Eq(document => document.RunId, runId),
|
||||
Builders<PackRunApprovalDocument>.Filter.Eq(document => document.ApprovalId, approval.ApprovalId));
|
||||
|
||||
var existingDocument = await collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (existingDocument is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Approval '{approval.ApprovalId}' not found for run '{runId}'.");
|
||||
}
|
||||
|
||||
var document = PackRunApprovalDocument.FromDomain(runId, approval, existingDocument.Id);
|
||||
await collection
|
||||
.ReplaceOneAsync(filter, document, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static IEnumerable<CreateIndexModel<PackRunApprovalDocument>> GetIndexModels()
|
||||
{
|
||||
yield return new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.ApprovalId),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_run_approvals_run_approval" });
|
||||
|
||||
yield return new CreateIndexModel<PackRunApprovalDocument>(
|
||||
Builders<PackRunApprovalDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Status),
|
||||
new CreateIndexOptions { Name = "pack_run_approvals_run_status" });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunApprovalDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string ApprovalId { get; init; } = default!;
|
||||
|
||||
public IReadOnlyList<string> RequiredGrants { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> StepIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> Messages { get; init; } = Array.Empty<string>();
|
||||
|
||||
public string? ReasonTemplate { get; init; }
|
||||
|
||||
public DateTime RequestedAt { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public string? ActorId { get; init; }
|
||||
|
||||
public DateTime? CompletedAt { get; init; }
|
||||
|
||||
public string? Summary { get; init; }
|
||||
|
||||
public static PackRunApprovalDocument FromDomain(string runId, PackRunApprovalState approval, ObjectId? id = null)
|
||||
=> new()
|
||||
{
|
||||
Id = id ?? ObjectId.GenerateNewId(),
|
||||
RunId = runId,
|
||||
ApprovalId = approval.ApprovalId,
|
||||
RequiredGrants = approval.RequiredGrants ?? Array.Empty<string>(),
|
||||
StepIds = approval.StepIds ?? Array.Empty<string>(),
|
||||
Messages = approval.Messages ?? Array.Empty<string>(),
|
||||
ReasonTemplate = approval.ReasonTemplate,
|
||||
RequestedAt = approval.RequestedAt.UtcDateTime,
|
||||
Status = approval.Status.ToString(),
|
||||
ActorId = approval.ActorId,
|
||||
CompletedAt = approval.CompletedAt?.UtcDateTime,
|
||||
Summary = approval.Summary
|
||||
};
|
||||
|
||||
public PackRunApprovalState ToDomain()
|
||||
{
|
||||
var status = Enum.Parse<PackRunApprovalStatus>(Status, ignoreCase: true);
|
||||
|
||||
return new PackRunApprovalState(
|
||||
ApprovalId,
|
||||
RequiredGrants?.ToList() ?? new List<string>(),
|
||||
StepIds?.ToList() ?? new List<string>(),
|
||||
Messages?.ToList() ?? new List<string>(),
|
||||
ReasonTemplate,
|
||||
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
|
||||
status,
|
||||
ActorId,
|
||||
CompletedAt is null ? null : new DateTimeOffset(CompletedAt.Value, TimeSpan.Zero),
|
||||
Summary);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunArtifactReader : IPackRunArtifactReader
|
||||
{
|
||||
private readonly IMongoCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument> collection;
|
||||
|
||||
public MongoPackRunArtifactReader(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(options.ArtifactsCollection);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Eq(doc => doc.RunId, runId);
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.SortBy(doc => doc.Name)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(doc => new PackRunArtifactRecord(
|
||||
doc.Name,
|
||||
doc.Type,
|
||||
doc.SourcePath,
|
||||
doc.StoredPath,
|
||||
doc.Status,
|
||||
doc.Notes,
|
||||
new DateTimeOffset(doc.CapturedAt, TimeSpan.Zero),
|
||||
doc.Expression?.ToJson(new JsonWriterSettings())))
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunArtifactUploader : IPackRunArtifactUploader
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<PackRunArtifactDocument> collection;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ILogger<MongoPackRunArtifactUploader> logger;
|
||||
|
||||
public MongoPackRunArtifactUploader(
|
||||
IMongoDatabase database,
|
||||
TaskRunnerMongoOptions options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<MongoPackRunArtifactUploader> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunArtifactDocument>(options.ArtifactsCollection);
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task UploadAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunState state,
|
||||
IReadOnlyList<TaskPackPlanOutput> outputs,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
ArgumentNullException.ThrowIfNull(outputs);
|
||||
|
||||
var filter = Builders<PackRunArtifactDocument>.Filter.Eq(document => document.RunId, context.RunId);
|
||||
await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (outputs.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var timestamp = timeProvider.GetUtcNow();
|
||||
var documents = new List<PackRunArtifactDocument>(outputs.Count);
|
||||
|
||||
foreach (var output in outputs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
documents.Add(ProcessOutput(context, output, timestamp));
|
||||
}
|
||||
|
||||
await collection.InsertManyAsync(documents, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private PackRunArtifactDocument ProcessOutput(
|
||||
PackRunExecutionContext context,
|
||||
TaskPackPlanOutput output,
|
||||
DateTimeOffset capturedAt)
|
||||
{
|
||||
var sourcePath = ResolveString(output.Path);
|
||||
var expressionNode = ResolveExpression(output.Expression);
|
||||
string status = "skipped";
|
||||
string? notes = null;
|
||||
string? storedPath = null;
|
||||
|
||||
if (IsFileOutput(output))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourcePath))
|
||||
{
|
||||
status = "unresolved";
|
||||
notes = "Output path requires runtime value.";
|
||||
}
|
||||
else if (!File.Exists(sourcePath))
|
||||
{
|
||||
status = "missing";
|
||||
notes = $"Source file '{sourcePath}' not found.";
|
||||
logger.LogWarning(
|
||||
"Pack run {RunId} output {Output} referenced missing file {Path}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
sourcePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
status = "referenced";
|
||||
storedPath = sourcePath;
|
||||
}
|
||||
}
|
||||
|
||||
BsonDocument? expressionDocument = null;
|
||||
if (expressionNode is not null)
|
||||
{
|
||||
var json = expressionNode.ToJsonString(SerializerOptions);
|
||||
expressionDocument = BsonDocument.Parse(json);
|
||||
status = status is "referenced" ? status : "materialized";
|
||||
}
|
||||
|
||||
return new PackRunArtifactDocument
|
||||
{
|
||||
Id = ObjectId.GenerateNewId(),
|
||||
RunId = context.RunId,
|
||||
Name = output.Name,
|
||||
Type = output.Type,
|
||||
SourcePath = sourcePath,
|
||||
StoredPath = storedPath,
|
||||
Status = status,
|
||||
Notes = notes,
|
||||
CapturedAt = capturedAt.UtcDateTime,
|
||||
Expression = expressionDocument
|
||||
};
|
||||
}
|
||||
|
||||
private static bool IsFileOutput(TaskPackPlanOutput output)
|
||||
=> string.Equals(output.Type, "file", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string? ResolveString(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue || parameter.Value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parameter.Value is JsonValue jsonValue && jsonValue.TryGetValue<string>(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static JsonNode? ResolveExpression(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return parameter.Value;
|
||||
}
|
||||
|
||||
public static IEnumerable<CreateIndexModel<PackRunArtifactDocument>> GetIndexModels()
|
||||
{
|
||||
yield return new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Name),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_artifacts_run_name" });
|
||||
|
||||
yield return new CreateIndexModel<PackRunArtifactDocument>(
|
||||
Builders<PackRunArtifactDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId),
|
||||
new CreateIndexOptions { Name = "pack_artifacts_run" });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunArtifactDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string Name { get; init; } = default!;
|
||||
|
||||
public string Type { get; init; } = default!;
|
||||
|
||||
public string? SourcePath { get; init; }
|
||||
|
||||
public string? StoredPath { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public string? Notes { get; init; }
|
||||
|
||||
public DateTime CapturedAt { get; init; }
|
||||
|
||||
public BsonDocument? Expression { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunLogStore : IPackRunLogStore
|
||||
{
|
||||
private readonly IMongoCollection<PackRunLogDocument> collection;
|
||||
|
||||
public MongoPackRunLogStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunLogDocument>(options.LogsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task AppendAsync(string runId, PackRunLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
for (var attempt = 0; attempt < 5; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var last = await collection
|
||||
.Find(filter)
|
||||
.SortByDescending(document => document.Sequence)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var nextSequence = last is null ? 1 : last.Sequence + 1;
|
||||
|
||||
var document = PackRunLogDocument.FromDomain(runId, nextSequence, entry);
|
||||
|
||||
try
|
||||
{
|
||||
await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Failed to append log entry for run '{runId}' after multiple attempts.");
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<PackRunLogEntry> ReadAsync(
|
||||
string runId,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
|
||||
using var cursor = await collection
|
||||
.Find(filter)
|
||||
.SortBy(document => document.Sequence)
|
||||
.ToCursorAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var document in cursor.Current)
|
||||
{
|
||||
yield return document.ToDomain();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> ExistsAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunLogDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
return await collection
|
||||
.Find(filter)
|
||||
.Limit(1)
|
||||
.AnyAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static IEnumerable<CreateIndexModel<PackRunLogDocument>> GetIndexModels()
|
||||
{
|
||||
yield return new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Sequence),
|
||||
new CreateIndexOptions { Unique = true, Name = "pack_run_logs_run_sequence" });
|
||||
|
||||
yield return new CreateIndexModel<PackRunLogDocument>(
|
||||
Builders<PackRunLogDocument>.IndexKeys
|
||||
.Ascending(document => document.RunId)
|
||||
.Ascending(document => document.Timestamp),
|
||||
new CreateIndexOptions { Name = "pack_run_logs_run_timestamp" });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunLogDocument
|
||||
{
|
||||
[BsonId]
|
||||
public ObjectId Id { get; init; }
|
||||
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public long Sequence { get; init; }
|
||||
|
||||
public DateTime Timestamp { get; init; }
|
||||
|
||||
public string Level { get; init; } = default!;
|
||||
|
||||
public string EventType { get; init; } = default!;
|
||||
|
||||
public string Message { get; init; } = default!;
|
||||
|
||||
public string? StepId { get; init; }
|
||||
|
||||
public Dictionary<string, string>? Metadata { get; init; }
|
||||
|
||||
public static PackRunLogDocument FromDomain(string runId, long sequence, PackRunLogEntry entry)
|
||||
=> new()
|
||||
{
|
||||
Id = ObjectId.GenerateNewId(),
|
||||
RunId = runId,
|
||||
Sequence = sequence,
|
||||
Timestamp = entry.Timestamp.UtcDateTime,
|
||||
Level = entry.Level,
|
||||
EventType = entry.EventType,
|
||||
Message = entry.Message,
|
||||
StepId = entry.StepId,
|
||||
Metadata = entry.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(entry.Metadata, StringComparer.Ordinal)
|
||||
};
|
||||
|
||||
public PackRunLogEntry ToDomain()
|
||||
{
|
||||
IReadOnlyDictionary<string, string>? metadata = Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(Metadata, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunLogEntry(
|
||||
new DateTimeOffset(Timestamp, TimeSpan.Zero),
|
||||
Level,
|
||||
EventType,
|
||||
Message,
|
||||
StepId,
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunProvenanceWriter : IPackRunProvenanceWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<ProvenanceDocument> collection;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public MongoPackRunProvenanceWriter(IMongoDatabase database, TaskRunnerMongoOptions options, TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<ProvenanceDocument>(options.ArtifactsCollection);
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var completedAt = timeProvider.GetUtcNow();
|
||||
var manifest = ProvenanceManifestFactory.Create(context, state, completedAt);
|
||||
var manifestJson = JsonSerializer.Serialize(manifest, SerializerOptions);
|
||||
var manifestDocument = BsonDocument.Parse(manifestJson);
|
||||
|
||||
var document = new ProvenanceDocument
|
||||
{
|
||||
RunId = context.RunId,
|
||||
Name = "provenance-manifest",
|
||||
Type = "object",
|
||||
Status = "materialized",
|
||||
CapturedAt = completedAt.UtcDateTime,
|
||||
Expression = manifestDocument
|
||||
};
|
||||
|
||||
var filter = Builders<ProvenanceDocument>.Filter.And(
|
||||
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.RunId, context.RunId),
|
||||
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.Name, document.Name));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private sealed class ProvenanceDocument
|
||||
{
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string Name { get; init; } = default!;
|
||||
|
||||
public string Type { get; init; } = default!;
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public DateTime CapturedAt { get; init; }
|
||||
|
||||
public BsonDocument Expression { get; init; } = default!;
|
||||
}
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
public sealed class MongoPackRunStateStore : IPackRunStateStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly IMongoCollection<PackRunStateDocument> collection;
|
||||
|
||||
public MongoPackRunStateStore(IMongoDatabase database, TaskRunnerMongoOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
collection = database.GetCollection<PackRunStateDocument>(options.RunsCollection);
|
||||
EnsureIndexes(collection);
|
||||
}
|
||||
|
||||
public async Task<PackRunState?> GetAsync(string runId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
|
||||
|
||||
var filter = Builders<PackRunStateDocument>.Filter.Eq(document => document.RunId, runId);
|
||||
var document = await collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return document?.ToDomain();
|
||||
}
|
||||
|
||||
public async Task SaveAsync(PackRunState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var document = PackRunStateDocument.FromDomain(state);
|
||||
var filter = Builders<PackRunStateDocument>.Filter.Eq(existing => existing.RunId, state.RunId);
|
||||
|
||||
await collection
|
||||
.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<PackRunState>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var documents = await collection
|
||||
.Find(FilterDefinition<PackRunStateDocument>.Empty)
|
||||
.SortByDescending(document => document.UpdatedAt)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents
|
||||
.Select(document => document.ToDomain())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public static IEnumerable<CreateIndexModel<PackRunStateDocument>> GetIndexModels()
|
||||
{
|
||||
yield return new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt),
|
||||
new CreateIndexOptions { Name = "pack_runs_updatedAt_desc" });
|
||||
|
||||
yield return new CreateIndexModel<PackRunStateDocument>(
|
||||
Builders<PackRunStateDocument>.IndexKeys
|
||||
.Ascending(document => document.TenantId)
|
||||
.Descending(document => document.UpdatedAt),
|
||||
new CreateIndexOptions { Name = "pack_runs_tenant_updatedAt_desc", Sparse = true });
|
||||
}
|
||||
|
||||
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
|
||||
=> target.Indexes.CreateMany(GetIndexModels());
|
||||
|
||||
public sealed class PackRunStateDocument
|
||||
{
|
||||
[BsonId]
|
||||
public string RunId { get; init; } = default!;
|
||||
|
||||
public string PlanHash { get; init; } = default!;
|
||||
|
||||
public BsonDocument Plan { get; init; } = default!;
|
||||
|
||||
public BsonDocument FailurePolicy { get; init; } = default!;
|
||||
|
||||
public DateTime RequestedAt { get; init; }
|
||||
|
||||
public DateTime CreatedAt { get; init; }
|
||||
|
||||
public DateTime UpdatedAt { get; init; }
|
||||
|
||||
public List<PackRunStepDocument> Steps { get; init; } = new();
|
||||
|
||||
public string? TenantId { get; init; }
|
||||
|
||||
public static PackRunStateDocument FromDomain(PackRunState state)
|
||||
{
|
||||
var planDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.Plan, SerializerOptions));
|
||||
var failurePolicyDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.FailurePolicy, SerializerOptions));
|
||||
|
||||
var steps = state.Steps.Values
|
||||
.OrderBy(step => step.StepId, StringComparer.Ordinal)
|
||||
.Select(PackRunStepDocument.FromDomain)
|
||||
.ToList();
|
||||
|
||||
return new PackRunStateDocument
|
||||
{
|
||||
RunId = state.RunId,
|
||||
PlanHash = state.PlanHash,
|
||||
Plan = planDocument,
|
||||
FailurePolicy = failurePolicyDocument,
|
||||
RequestedAt = state.RequestedAt.UtcDateTime,
|
||||
CreatedAt = state.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = state.UpdatedAt.UtcDateTime,
|
||||
Steps = steps,
|
||||
TenantId = state.TenantId
|
||||
};
|
||||
}
|
||||
|
||||
public PackRunState ToDomain()
|
||||
{
|
||||
var planJson = Plan.ToJson();
|
||||
var plan = JsonSerializer.Deserialize<TaskPackPlan>(planJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize stored TaskPackPlan.");
|
||||
|
||||
var failurePolicyJson = FailurePolicy.ToJson();
|
||||
var failurePolicy = JsonSerializer.Deserialize<TaskPackPlanFailurePolicy>(failurePolicyJson, SerializerOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize stored TaskPackPlanFailurePolicy.");
|
||||
|
||||
var stepRecords = Steps
|
||||
.Select(step => step.ToDomain())
|
||||
.ToDictionary(record => record.StepId, record => record, StringComparer.Ordinal);
|
||||
|
||||
return new PackRunState(
|
||||
RunId,
|
||||
PlanHash,
|
||||
plan,
|
||||
failurePolicy,
|
||||
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(CreatedAt, TimeSpan.Zero),
|
||||
new DateTimeOffset(UpdatedAt, TimeSpan.Zero),
|
||||
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords),
|
||||
TenantId);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class PackRunStepDocument
|
||||
{
|
||||
public string StepId { get; init; } = default!;
|
||||
|
||||
public string Kind { get; init; } = default!;
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
|
||||
public bool ContinueOnError { get; init; }
|
||||
|
||||
public int? MaxParallel { get; init; }
|
||||
|
||||
public string? ApprovalId { get; init; }
|
||||
|
||||
public string? GateMessage { get; init; }
|
||||
|
||||
public string Status { get; init; } = default!;
|
||||
|
||||
public int Attempts { get; init; }
|
||||
|
||||
public DateTime? LastTransitionAt { get; init; }
|
||||
|
||||
public DateTime? NextAttemptAt { get; init; }
|
||||
|
||||
public string? StatusReason { get; init; }
|
||||
|
||||
public static PackRunStepDocument FromDomain(PackRunStepStateRecord record)
|
||||
=> new()
|
||||
{
|
||||
StepId = record.StepId,
|
||||
Kind = record.Kind.ToString(),
|
||||
Enabled = record.Enabled,
|
||||
ContinueOnError = record.ContinueOnError,
|
||||
MaxParallel = record.MaxParallel,
|
||||
ApprovalId = record.ApprovalId,
|
||||
GateMessage = record.GateMessage,
|
||||
Status = record.Status.ToString(),
|
||||
Attempts = record.Attempts,
|
||||
LastTransitionAt = record.LastTransitionAt?.UtcDateTime,
|
||||
NextAttemptAt = record.NextAttemptAt?.UtcDateTime,
|
||||
StatusReason = record.StatusReason
|
||||
};
|
||||
|
||||
public PackRunStepStateRecord ToDomain()
|
||||
{
|
||||
var kind = Enum.Parse<PackRunStepKind>(Kind, ignoreCase: true);
|
||||
var status = Enum.Parse<PackRunStepExecutionStatus>(Status, ignoreCase: true);
|
||||
|
||||
return new PackRunStepStateRecord(
|
||||
StepId,
|
||||
kind,
|
||||
Enabled,
|
||||
ContinueOnError,
|
||||
MaxParallel,
|
||||
ApprovalId,
|
||||
GateMessage,
|
||||
status,
|
||||
Attempts,
|
||||
LastTransitionAt is null ? null : new DateTimeOffset(LastTransitionAt.Value, TimeSpan.Zero),
|
||||
NextAttemptAt is null ? null : new DateTimeOffset(NextAttemptAt.Value, TimeSpan.Zero),
|
||||
StatusReason);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class MongoIndexModelTests
|
||||
{
|
||||
[Fact]
|
||||
public void StateStore_indexes_match_contract()
|
||||
{
|
||||
var models = MongoPackRunStateStore.GetIndexModels().ToArray();
|
||||
|
||||
Assert.Collection(models,
|
||||
model => Assert.Equal("pack_runs_updatedAt_desc", model.Options.Name),
|
||||
model => Assert.Equal("pack_runs_tenant_updatedAt_desc", model.Options.Name));
|
||||
|
||||
Assert.True(models[1].Options.Sparse ?? false);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LogStore_indexes_match_contract()
|
||||
{
|
||||
var models = MongoPackRunLogStore.GetIndexModels().ToArray();
|
||||
|
||||
Assert.Collection(models,
|
||||
model =>
|
||||
{
|
||||
Assert.Equal("pack_run_logs_run_sequence", model.Options.Name);
|
||||
Assert.True(model.Options.Unique ?? false);
|
||||
},
|
||||
model => Assert.Equal("pack_run_logs_run_timestamp", model.Options.Name));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ArtifactStore_indexes_match_contract()
|
||||
{
|
||||
var models = MongoPackRunArtifactUploader.GetIndexModels().ToArray();
|
||||
|
||||
Assert.Collection(models,
|
||||
model =>
|
||||
{
|
||||
Assert.Equal("pack_artifacts_run_name", model.Options.Name);
|
||||
Assert.True(model.Options.Unique ?? false);
|
||||
},
|
||||
model => Assert.Equal("pack_artifacts_run", model.Options.Name));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApprovalStore_indexes_match_contract()
|
||||
{
|
||||
var models = MongoPackRunApprovalStore.GetIndexModels().ToArray();
|
||||
|
||||
Assert.Collection(models,
|
||||
model =>
|
||||
{
|
||||
Assert.Equal("pack_run_approvals_run_approval", model.Options.Name);
|
||||
Assert.True(model.Options.Unique ?? false);
|
||||
},
|
||||
model => Assert.Equal("pack_run_approvals_run_status", model.Options.Name));
|
||||
}
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
using StellaOps.TaskRunner.Core.TaskPacks;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using Xunit;
|
||||
using Xunit.Sdk;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class MongoPackRunStoresTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task StateStore_RoundTrips_State()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var stateStore = new MongoPackRunStateStore(context.Database, mongoOptions);
|
||||
|
||||
var plan = CreatePlan();
|
||||
var executionContext = new PackRunExecutionContext("mongo-run-state", plan, DateTimeOffset.UtcNow);
|
||||
var graph = new PackRunExecutionGraphBuilder().Build(plan);
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
|
||||
|
||||
await stateStore.SaveAsync(state, CancellationToken.None);
|
||||
|
||||
var reloaded = await stateStore.GetAsync(state.RunId, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(reloaded);
|
||||
Assert.Equal(state.RunId, reloaded!.RunId);
|
||||
Assert.Equal(state.PlanHash, reloaded.PlanHash);
|
||||
Assert.Equal(state.Steps.Count, reloaded.Steps.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LogStore_Appends_And_Reads_In_Order()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var logStore = new MongoPackRunLogStore(context.Database, mongoOptions);
|
||||
|
||||
var runId = "mongo-log";
|
||||
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.created", "created", null, null), CancellationToken.None);
|
||||
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow.AddSeconds(1), "warn", "step.retry", "retry", "step-a", new Dictionary<string, string> { ["attempt"] = "2" }), CancellationToken.None);
|
||||
|
||||
var entries = new List<PackRunLogEntry>();
|
||||
await foreach (var entry in logStore.ReadAsync(runId, CancellationToken.None))
|
||||
{
|
||||
entries.Add(entry);
|
||||
}
|
||||
|
||||
Assert.Equal(2, entries.Count);
|
||||
Assert.Equal("run.created", entries[0].EventType);
|
||||
Assert.Equal("step.retry", entries[1].EventType);
|
||||
Assert.Equal("step-a", entries[1].StepId);
|
||||
Assert.True(await logStore.ExistsAsync(runId, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApprovalStore_RoundTrips_And_Updates()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var approvalStore = new MongoPackRunApprovalStore(context.Database, mongoOptions);
|
||||
|
||||
var runId = "mongo-approvals";
|
||||
var approval = new PackRunApprovalState(
|
||||
"security-review",
|
||||
new[] { "packs.approve" },
|
||||
new[] { "step-plan" },
|
||||
Array.Empty<string>(),
|
||||
reasonTemplate: "Security approval required.",
|
||||
DateTimeOffset.UtcNow,
|
||||
PackRunApprovalStatus.Pending);
|
||||
|
||||
await approvalStore.SaveAsync(runId, new[] { approval }, CancellationToken.None);
|
||||
|
||||
var approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
|
||||
Assert.Single(approvals);
|
||||
|
||||
var updated = approval.Approve("approver", DateTimeOffset.UtcNow, "Approved");
|
||||
await approvalStore.UpdateAsync(runId, updated, CancellationToken.None);
|
||||
|
||||
approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
|
||||
Assert.Single(approvals);
|
||||
Assert.Equal(PackRunApprovalStatus.Approved, approvals[0].Status);
|
||||
Assert.Equal("approver", approvals[0].ActorId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ArtifactUploader_Persists_Metadata()
|
||||
{
|
||||
using var context = MongoTaskRunnerTestContext.Create();
|
||||
var mongoOptions = context.CreateMongoOptions();
|
||||
var database = context.Database;
|
||||
|
||||
var artifactUploader = new MongoPackRunArtifactUploader(
|
||||
database,
|
||||
mongoOptions,
|
||||
TimeProvider.System,
|
||||
NullLogger<MongoPackRunArtifactUploader>.Instance);
|
||||
|
||||
var plan = CreatePlanWithOutputs(out var outputFile);
|
||||
try
|
||||
{
|
||||
var executionContext = new PackRunExecutionContext("mongo-artifacts", plan, DateTimeOffset.UtcNow);
|
||||
var graph = new PackRunExecutionGraphBuilder().Build(plan);
|
||||
var simulationEngine = new PackRunSimulationEngine();
|
||||
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
|
||||
|
||||
await artifactUploader.UploadAsync(executionContext, state, plan.Outputs, CancellationToken.None);
|
||||
|
||||
var documents = await database
|
||||
.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(mongoOptions.ArtifactsCollection)
|
||||
.Find(Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Empty)
|
||||
.ToListAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var bundleDocument = Assert.Single(documents, d => string.Equals(d.Name, "bundlePath", StringComparison.Ordinal));
|
||||
Assert.Equal("file", bundleDocument.Type);
|
||||
Assert.Equal(outputFile, bundleDocument.SourcePath);
|
||||
Assert.Equal("referenced", bundleDocument.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(outputFile))
|
||||
{
|
||||
File.Delete(outputFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static TaskPackPlan CreatePlan()
|
||||
{
|
||||
var manifest = TestManifests.Load(TestManifests.Sample);
|
||||
var planner = new TaskPackPlanner();
|
||||
var result = planner.Plan(manifest);
|
||||
if (!result.Success || result.Plan is null)
|
||||
{
|
||||
Assert.Skip("Failed to build task pack plan for Mongo tests.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
|
||||
return result.Plan;
|
||||
}
|
||||
|
||||
private static TaskPackPlan CreatePlanWithOutputs(out string outputFile)
|
||||
{
|
||||
var manifest = TestManifests.Load(TestManifests.Output);
|
||||
var planner = new TaskPackPlanner();
|
||||
var result = planner.Plan(manifest);
|
||||
if (!result.Success || result.Plan is null)
|
||||
{
|
||||
Assert.Skip("Failed to build output plan for Mongo tests.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
|
||||
// Materialize a fake output file referenced by the plan.
|
||||
outputFile = Path.Combine(Path.GetTempPath(), $"taskrunner-output-{Guid.NewGuid():N}.txt");
|
||||
File.WriteAllText(outputFile, "fixture");
|
||||
|
||||
// Update the plan output path parameter to point at the file we just created.
|
||||
var originalPlan = result.Plan;
|
||||
|
||||
var resolvedFile = outputFile;
|
||||
|
||||
var outputs = originalPlan.Outputs
|
||||
.Select(output =>
|
||||
{
|
||||
if (!string.Equals(output.Name, "bundlePath", StringComparison.Ordinal))
|
||||
{
|
||||
return output;
|
||||
}
|
||||
|
||||
var node = JsonNode.Parse($"\"{resolvedFile.Replace("\\", "\\\\")}\"");
|
||||
var parameter = new TaskPackPlanParameterValue(node, null, null, false);
|
||||
return output with { Path = parameter };
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
return new TaskPackPlan(
|
||||
originalPlan.Metadata,
|
||||
originalPlan.Inputs,
|
||||
originalPlan.Steps,
|
||||
originalPlan.Hash,
|
||||
originalPlan.Approvals,
|
||||
originalPlan.Secrets,
|
||||
outputs,
|
||||
originalPlan.FailurePolicy);
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
internal sealed class MongoTaskRunnerTestContext : IAsyncDisposable, IDisposable
|
||||
{
|
||||
private readonly MongoDbRunner? runner;
|
||||
private readonly string databaseName;
|
||||
private readonly IMongoClient client;
|
||||
private readonly string connectionString;
|
||||
|
||||
private MongoTaskRunnerTestContext(
|
||||
IMongoClient client,
|
||||
IMongoDatabase database,
|
||||
MongoDbRunner? runner,
|
||||
string databaseName,
|
||||
string connectionString)
|
||||
{
|
||||
this.client = client;
|
||||
Database = database;
|
||||
this.runner = runner;
|
||||
this.databaseName = databaseName;
|
||||
this.connectionString = connectionString;
|
||||
}
|
||||
|
||||
public IMongoDatabase Database { get; }
|
||||
|
||||
public static MongoTaskRunnerTestContext Create()
|
||||
{
|
||||
OpenSslLegacyShim.EnsureOpenSsl11();
|
||||
|
||||
var uri = Environment.GetEnvironmentVariable("STELLAOPS_TEST_MONGO_URI");
|
||||
if (!string.IsNullOrWhiteSpace(uri))
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = MongoUrl.Create(uri);
|
||||
var client = new MongoClient(url);
|
||||
var databaseName = string.IsNullOrWhiteSpace(url.DatabaseName)
|
||||
? $"taskrunner-tests-{Guid.NewGuid():N}"
|
||||
: url.DatabaseName;
|
||||
var database = client.GetDatabase(databaseName);
|
||||
return new MongoTaskRunnerTestContext(client, database, runner: null, databaseName, uri);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Assert.Skip($"Failed to connect to MongoDB using STELLAOPS_TEST_MONGO_URI: {ex.Message}");
|
||||
throw new InvalidOperationException(); // Unreachable
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var runner = MongoDbRunner.Start(singleNodeReplSet: false);
|
||||
var client = new MongoClient(runner.ConnectionString);
|
||||
var databaseName = $"taskrunner-tests-{Guid.NewGuid():N}";
|
||||
var database = client.GetDatabase(databaseName);
|
||||
return new MongoTaskRunnerTestContext(client, database, runner, databaseName, runner.ConnectionString);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Assert.Skip($"Unable to start embedded MongoDB (Mongo2Go): {ex.Message}");
|
||||
throw new InvalidOperationException(); // Unreachable
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await client.DropDatabaseAsync(databaseName);
|
||||
runner?.Dispose();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
client.DropDatabase(databaseName);
|
||||
runner?.Dispose();
|
||||
}
|
||||
|
||||
public TaskRunnerMongoOptions CreateMongoOptions()
|
||||
=> new()
|
||||
{
|
||||
ConnectionString = connectionString,
|
||||
Database = databaseName
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
@@ -40,30 +39,6 @@ public sealed class PackRunProvenanceWriterTests
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Mongo_writer_upserts_manifest()
|
||||
{
|
||||
await using var mongo = MongoTaskRunnerTestContext.Create();
|
||||
var (context, state) = CreateRunState();
|
||||
var completedAt = new DateTimeOffset(2025, 11, 30, 12, 0, 0, TimeSpan.Zero);
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
|
||||
var options = mongo.CreateMongoOptions();
|
||||
var writer = new MongoPackRunProvenanceWriter(mongo.Database, options, new FixedTimeProvider(completedAt));
|
||||
await writer.WriteAsync(context, state, ct);
|
||||
|
||||
var collection = mongo.Database.GetCollection<MongoDB.Bson.BsonDocument>(options.ArtifactsCollection);
|
||||
var saved = await collection
|
||||
.Find(Builders<MongoDB.Bson.BsonDocument>.Filter.Eq("RunId", context.RunId))
|
||||
.FirstOrDefaultAsync(ct);
|
||||
|
||||
Assert.NotNull(saved);
|
||||
var manifest = saved!["Expression"].AsBsonDocument;
|
||||
Assert.Equal("run-test", manifest["runId"].AsString);
|
||||
Assert.Equal("tenant-alpha", manifest["tenantId"].AsString);
|
||||
Assert.Equal(context.Plan.Hash, manifest["planHash"].AsString);
|
||||
}
|
||||
|
||||
private static (PackRunExecutionContext Context, PackRunState State) CreateRunState()
|
||||
{
|
||||
var loader = new TaskPackManifestLoader();
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="Mongo2Go" Version="4.1.0" />
|
||||
<PackageReference Include="xunit.v3" Version="3.0.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3" />
|
||||
</ItemGroup>
|
||||
@@ -36,12 +35,6 @@
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\..\..\tests\native/openssl-1.1/linux-x64/*"
|
||||
Link="native/linux-x64/%(Filename)%(Extension)"
|
||||
CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -6,7 +6,6 @@ using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using MongoDB.Driver;
|
||||
using OpenTelemetry.Metrics;
|
||||
using OpenTelemetry.Trace;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
@@ -50,52 +49,26 @@ builder.Services.AddStellaOpsTelemetry(
|
||||
.AddRuntimeInstrumentation()
|
||||
.AddMeter(TaskRunnerTelemetry.MeterName));
|
||||
|
||||
var storageOptions = builder.Configuration.GetSection("TaskRunner:Storage").Get<TaskRunnerStorageOptions>() ?? new TaskRunnerStorageOptions();
|
||||
builder.Services.AddSingleton(storageOptions);
|
||||
|
||||
if (string.Equals(storageOptions.Mode, TaskRunnerStorageModes.Mongo, StringComparison.OrdinalIgnoreCase))
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
builder.Services.AddSingleton(storageOptions.Mongo);
|
||||
builder.Services.AddSingleton<IMongoClient>(_ => new MongoClient(storageOptions.Mongo.ConnectionString));
|
||||
builder.Services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var mongoOptions = storageOptions.Mongo;
|
||||
var client = sp.GetRequiredService<IMongoClient>();
|
||||
var mongoUrl = MongoUrl.Create(mongoOptions.ConnectionString);
|
||||
var databaseName = !string.IsNullOrWhiteSpace(mongoOptions.Database)
|
||||
? mongoOptions.Database
|
||||
: mongoUrl.DatabaseName ?? "stellaops-taskrunner";
|
||||
return client.GetDatabase(databaseName);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactReader, MongoPackRunArtifactReader>();
|
||||
}
|
||||
else
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunApprovalStore(options.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunApprovalStore(options.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunStateStore(options.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunLogStore(options.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactReader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilesystemPackRunArtifactReader(options.ArtifactsPath);
|
||||
});
|
||||
}
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunStateStore(options.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilePackRunLogStore(options.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactReader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<TaskRunnerServiceOptions>>().Value;
|
||||
return new FilesystemPackRunArtifactReader(options.ArtifactsPath);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton(sp =>
|
||||
{
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
|
||||
namespace StellaOps.TaskRunner.WebService;
|
||||
|
||||
public sealed class TaskRunnerServiceOptions
|
||||
@@ -10,6 +8,4 @@ public sealed class TaskRunnerServiceOptions
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
public string LogsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "logs", "runs");
|
||||
public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts");
|
||||
|
||||
public TaskRunnerStorageOptions Storage { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.AirGap.Policy;
|
||||
using StellaOps.TaskRunner.Core.Configuration;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
@@ -7,7 +6,7 @@ using StellaOps.TaskRunner.Core.Execution.Simulation;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using StellaOps.TaskRunner.Worker.Services;
|
||||
using StellaOps.Telemetry.Core;
|
||||
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
|
||||
builder.Services.AddAirGapEgressPolicy(builder.Configuration, sectionName: "AirGap");
|
||||
@@ -51,67 +50,34 @@ builder.Services.AddStellaOpsTelemetry(
|
||||
.AddRuntimeInstrumentation()
|
||||
.AddMeter(TaskRunnerTelemetry.MeterName));
|
||||
|
||||
var workerStorageOptions = builder.Configuration.GetSection("Worker:Storage").Get<TaskRunnerStorageOptions>() ?? new TaskRunnerStorageOptions();
|
||||
builder.Services.AddSingleton(workerStorageOptions);
|
||||
|
||||
if (string.Equals(workerStorageOptions.Mode, TaskRunnerStorageModes.Mongo, StringComparison.OrdinalIgnoreCase))
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
builder.Services.AddSingleton(workerStorageOptions.Mongo);
|
||||
builder.Services.AddSingleton<IMongoClient>(_ => new MongoClient(workerStorageOptions.Mongo.ConnectionString));
|
||||
builder.Services.AddSingleton<IMongoDatabase>(sp =>
|
||||
{
|
||||
var mongoOptions = workerStorageOptions.Mongo;
|
||||
var client = sp.GetRequiredService<IMongoClient>();
|
||||
var mongoUrl = MongoUrl.Create(mongoOptions.ConnectionString);
|
||||
var databaseName = !string.IsNullOrWhiteSpace(mongoOptions.Database)
|
||||
? mongoOptions.Database
|
||||
: mongoUrl.DatabaseName ?? "stellaops-taskrunner";
|
||||
return client.GetDatabase(databaseName);
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IPackRunStateStore, MongoPackRunStateStore>();
|
||||
builder.Services.AddSingleton<IPackRunLogStore, MongoPackRunLogStore>();
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore, MongoPackRunApprovalStore>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader, MongoPackRunArtifactUploader>();
|
||||
builder.Services.AddSingleton<IPackRunProvenanceWriter>(sp =>
|
||||
{
|
||||
var db = sp.GetRequiredService<IMongoDatabase>();
|
||||
var options = sp.GetRequiredService<TaskRunnerMongoOptions>();
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
return new MongoPackRunProvenanceWriter(db, options, timeProvider);
|
||||
});
|
||||
}
|
||||
else
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunApprovalStore(options.Value.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
builder.Services.AddSingleton<IPackRunApprovalStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunApprovalStore(options.Value.ApprovalStorePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunStateStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunStateStore(options.Value.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunLogStore(options.Value.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunProvenanceWriter>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
return new FilesystemPackRunProvenanceWriter(options.ArtifactsPath, timeProvider);
|
||||
});
|
||||
}
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunStateStore(options.Value.RunStatePath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunLogStore>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>();
|
||||
return new FilePackRunLogStore(options.Value.LogsPath);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
builder.Services.AddSingleton<IPackRunProvenanceWriter>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetRequiredService<TimeProvider>();
|
||||
return new FilesystemPackRunProvenanceWriter(options.ArtifactsPath, timeProvider);
|
||||
});
|
||||
|
||||
builder.Services.AddHostedService<PackRunWorkerService>();
|
||||
|
||||
|
||||
@@ -18,4 +18,8 @@
|
||||
| TASKRUN-OBS-53-001 | BLOCKED (2025-11-25) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-OBS-52-001 | Evidence locker snapshots; blocked: waiting on timeline schema/pointer contract. |
|
||||
| TASKRUN-GAPS-157-014 | DONE (2025-12-05) | SPRINT_0157_0001_0001_taskrunner_i | — | TP1–TP10 remediation: canonical plan-hash recipe, inputs.lock evidence, approval DSSE ledger, redaction, deterministic RNG/time, sandbox/egress quotas, registry signing + SBOM + revocation, offline bundle schema + verifier script, SLO/alerting, fail-closed gates. |
|
||||
|
||||
| MR-T10.7.1 | DONE (2025-12-11) | SPRINT_3410_0001_0001_mongodb_final_removal | ƒ?" | TaskRunner WebService now filesystem-only; removed Mongo wiring and dependencies. |
|
||||
| MR-T10.7.2 | DONE (2025-12-11) | SPRINT_3410_0001_0001_mongodb_final_removal | MR-T10.7.1 | TaskRunner Worker uses filesystem storage only; removed Mongo wiring and options. |
|
||||
| MR-T10.7.3 | DONE (2025-12-11) | SPRINT_3410_0001_0001_mongodb_final_removal | MR-T10.7.2 | Removed Mongo storage implementations/tests; dropped Mongo2Go dependency. |
|
||||
|
||||
Status source of truth: `docs/implplan/SPRINT_0157_0001_0001_taskrunner_i.md`. Update both files together. Keep UTC dates when advancing status.
|
||||
|
||||
Reference in New Issue
Block a user