Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added MongoPackRunApprovalStore for managing approval states with MongoDB.
- Introduced MongoPackRunArtifactUploader for uploading and storing artifacts.
- Created MongoPackRunLogStore to handle logging of pack run events.
- Developed MongoPackRunStateStore for persisting and retrieving pack run states.
- Implemented unit tests for MongoDB stores to ensure correct functionality.
- Added MongoTaskRunnerTestContext for setting up MongoDB test environment.
- Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
This commit is contained in:
master
2025-11-07 10:01:35 +02:00
parent e5ffcd6535
commit a1ce3f74fa
122 changed files with 8730 additions and 914 deletions

View File

@@ -0,0 +1,88 @@
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Infrastructure.Execution;
namespace StellaOps.TaskRunner.Tests;
public sealed class FilePackRunLogStoreTests : IDisposable
{
private readonly string rootPath;
public FilePackRunLogStoreTests()
{
rootPath = Path.Combine(Path.GetTempPath(), "StellaOps_TaskRunnerTests", Guid.NewGuid().ToString("n"));
}
[Fact]
public async Task AppendAndReadAsync_RoundTripsEntriesInOrder()
{
var store = new FilePackRunLogStore(rootPath);
var runId = "run-append-test";
var first = new PackRunLogEntry(
DateTimeOffset.UtcNow,
"info",
"run.created",
"Run created.",
StepId: null,
Metadata: new Dictionary<string, string>(StringComparer.Ordinal)
{
["planHash"] = "hash-1"
});
var second = new PackRunLogEntry(
DateTimeOffset.UtcNow.AddSeconds(1),
"info",
"step.started",
"Step started.",
StepId: "build",
Metadata: null);
await store.AppendAsync(runId, first, CancellationToken.None);
await store.AppendAsync(runId, second, CancellationToken.None);
var reloaded = new List<PackRunLogEntry>();
await foreach (var entry in store.ReadAsync(runId, CancellationToken.None))
{
reloaded.Add(entry);
}
Assert.Collection(
reloaded,
entry =>
{
Assert.Equal("run.created", entry.EventType);
Assert.NotNull(entry.Metadata);
Assert.Equal("hash-1", entry.Metadata!["planHash"]);
},
entry =>
{
Assert.Equal("step.started", entry.EventType);
Assert.Equal("build", entry.StepId);
});
}
[Fact]
public async Task ExistsAsync_ReturnsFalseWhenNoLogPresent()
{
var store = new FilePackRunLogStore(rootPath);
var exists = await store.ExistsAsync("missing-run", CancellationToken.None);
Assert.False(exists);
}
public void Dispose()
{
try
{
if (Directory.Exists(rootPath))
{
Directory.Delete(rootPath, recursive: true);
}
}
catch
{
// Ignore cleanup failures to keep tests deterministic.
}
}
}

View File

@@ -0,0 +1,196 @@
using System.Text.Json.Nodes;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Core.Execution.Simulation;
using StellaOps.TaskRunner.Core.Planning;
using StellaOps.TaskRunner.Core.TaskPacks;
using StellaOps.TaskRunner.Infrastructure.Execution;
using Xunit;
using Xunit.Sdk;
namespace StellaOps.TaskRunner.Tests;
public sealed class MongoPackRunStoresTests
{
[Fact]
public async Task StateStore_RoundTrips_State()
{
using var context = MongoTaskRunnerTestContext.Create();
var mongoOptions = context.CreateMongoOptions();
var stateStore = new MongoPackRunStateStore(context.Database, mongoOptions);
var plan = CreatePlan();
var executionContext = new PackRunExecutionContext("mongo-run-state", plan, DateTimeOffset.UtcNow);
var graph = new PackRunExecutionGraphBuilder().Build(plan);
var simulationEngine = new PackRunSimulationEngine();
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
await stateStore.SaveAsync(state, CancellationToken.None);
var reloaded = await stateStore.GetAsync(state.RunId, CancellationToken.None);
Assert.NotNull(reloaded);
Assert.Equal(state.RunId, reloaded!.RunId);
Assert.Equal(state.PlanHash, reloaded.PlanHash);
Assert.Equal(state.Steps.Count, reloaded.Steps.Count);
}
[Fact]
public async Task LogStore_Appends_And_Reads_In_Order()
{
using var context = MongoTaskRunnerTestContext.Create();
var mongoOptions = context.CreateMongoOptions();
var logStore = new MongoPackRunLogStore(context.Database, mongoOptions);
var runId = "mongo-log";
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow, "info", "run.created", "created", null, null), CancellationToken.None);
await logStore.AppendAsync(runId, new PackRunLogEntry(DateTimeOffset.UtcNow.AddSeconds(1), "warn", "step.retry", "retry", "step-a", new Dictionary<string, string> { ["attempt"] = "2" }), CancellationToken.None);
var entries = new List<PackRunLogEntry>();
await foreach (var entry in logStore.ReadAsync(runId, CancellationToken.None))
{
entries.Add(entry);
}
Assert.Equal(2, entries.Count);
Assert.Equal("run.created", entries[0].EventType);
Assert.Equal("step.retry", entries[1].EventType);
Assert.Equal("step-a", entries[1].StepId);
Assert.True(await logStore.ExistsAsync(runId, CancellationToken.None));
}
[Fact]
public async Task ApprovalStore_RoundTrips_And_Updates()
{
using var context = MongoTaskRunnerTestContext.Create();
var mongoOptions = context.CreateMongoOptions();
var approvalStore = new MongoPackRunApprovalStore(context.Database, mongoOptions);
var runId = "mongo-approvals";
var approval = new PackRunApprovalState(
"security-review",
new[] { "packs.approve" },
new[] { "step-plan" },
Array.Empty<string>(),
reasonTemplate: "Security approval required.",
DateTimeOffset.UtcNow,
PackRunApprovalStatus.Pending);
await approvalStore.SaveAsync(runId, new[] { approval }, CancellationToken.None);
var approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
Assert.Single(approvals);
var updated = approval.Approve("approver", DateTimeOffset.UtcNow, "Approved");
await approvalStore.UpdateAsync(runId, updated, CancellationToken.None);
approvals = await approvalStore.GetAsync(runId, CancellationToken.None);
Assert.Single(approvals);
Assert.Equal(PackRunApprovalStatus.Approved, approvals[0].Status);
Assert.Equal("approver", approvals[0].ActorId);
}
[Fact]
public async Task ArtifactUploader_Persists_Metadata()
{
using var context = MongoTaskRunnerTestContext.Create();
var mongoOptions = context.CreateMongoOptions();
var database = context.Database;
var artifactUploader = new MongoPackRunArtifactUploader(
database,
mongoOptions,
TimeProvider.System,
NullLogger<MongoPackRunArtifactUploader>.Instance);
var plan = CreatePlanWithOutputs(out var outputFile);
try
{
var executionContext = new PackRunExecutionContext("mongo-artifacts", plan, DateTimeOffset.UtcNow);
var graph = new PackRunExecutionGraphBuilder().Build(plan);
var simulationEngine = new PackRunSimulationEngine();
var state = PackRunStateFactory.CreateInitialState(executionContext, graph, simulationEngine, DateTimeOffset.UtcNow);
await artifactUploader.UploadAsync(executionContext, state, plan.Outputs, CancellationToken.None);
var documents = await database
.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(mongoOptions.ArtifactsCollection)
.Find(Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Empty)
.ToListAsync(TestContext.Current.CancellationToken);
var bundleDocument = Assert.Single(documents, d => string.Equals(d.Name, "bundlePath", StringComparison.Ordinal));
Assert.Equal("file", bundleDocument.Type);
Assert.Equal(outputFile, bundleDocument.SourcePath);
Assert.Equal("referenced", bundleDocument.Status);
}
finally
{
if (File.Exists(outputFile))
{
File.Delete(outputFile);
}
}
}
private static TaskPackPlan CreatePlan()
{
var manifest = TestManifests.Load(TestManifests.Sample);
var planner = new TaskPackPlanner();
var result = planner.Plan(manifest);
if (!result.Success || result.Plan is null)
{
Assert.Skip("Failed to build task pack plan for Mongo tests.");
throw new InvalidOperationException();
}
return result.Plan;
}
private static TaskPackPlan CreatePlanWithOutputs(out string outputFile)
{
var manifest = TestManifests.Load(TestManifests.Output);
var planner = new TaskPackPlanner();
var result = planner.Plan(manifest);
if (!result.Success || result.Plan is null)
{
Assert.Skip("Failed to build output plan for Mongo tests.");
throw new InvalidOperationException();
}
// Materialize a fake output file referenced by the plan.
outputFile = Path.Combine(Path.GetTempPath(), $"taskrunner-output-{Guid.NewGuid():N}.txt");
File.WriteAllText(outputFile, "fixture");
// Update the plan output path parameter to point at the file we just created.
var originalPlan = result.Plan;
var resolvedFile = outputFile;
var outputs = originalPlan.Outputs
.Select(output =>
{
if (!string.Equals(output.Name, "bundlePath", StringComparison.Ordinal))
{
return output;
}
var node = JsonNode.Parse($"\"{resolvedFile.Replace("\\", "\\\\")}\"");
var parameter = new TaskPackPlanParameterValue(node, null, null, false);
return output with { Path = parameter };
})
.ToArray();
return new TaskPackPlan(
originalPlan.Metadata,
originalPlan.Inputs,
originalPlan.Steps,
originalPlan.Hash,
originalPlan.Approvals,
originalPlan.Secrets,
outputs,
originalPlan.FailurePolicy);
}
}

View File

@@ -0,0 +1,89 @@
using Mongo2Go;
using MongoDB.Driver;
using StellaOps.TaskRunner.Core.Configuration;
using StellaOps.Testing;
using Xunit;
namespace StellaOps.TaskRunner.Tests;
internal sealed class MongoTaskRunnerTestContext : IAsyncDisposable, IDisposable
{
private readonly MongoDbRunner? runner;
private readonly string databaseName;
private readonly IMongoClient client;
private readonly string connectionString;
private MongoTaskRunnerTestContext(
IMongoClient client,
IMongoDatabase database,
MongoDbRunner? runner,
string databaseName,
string connectionString)
{
this.client = client;
Database = database;
this.runner = runner;
this.databaseName = databaseName;
this.connectionString = connectionString;
}
public IMongoDatabase Database { get; }
public static MongoTaskRunnerTestContext Create()
{
OpenSslLegacyShim.EnsureOpenSsl11();
var uri = Environment.GetEnvironmentVariable("STELLAOPS_TEST_MONGO_URI");
if (!string.IsNullOrWhiteSpace(uri))
{
try
{
var url = MongoUrl.Create(uri);
var client = new MongoClient(url);
var databaseName = string.IsNullOrWhiteSpace(url.DatabaseName)
? $"taskrunner-tests-{Guid.NewGuid():N}"
: url.DatabaseName;
var database = client.GetDatabase(databaseName);
return new MongoTaskRunnerTestContext(client, database, runner: null, databaseName, uri);
}
catch (Exception ex)
{
Assert.Skip($"Failed to connect to MongoDB using STELLAOPS_TEST_MONGO_URI: {ex.Message}");
throw new InvalidOperationException(); // Unreachable
}
}
try
{
var runner = MongoDbRunner.Start(singleNodeReplSet: false);
var client = new MongoClient(runner.ConnectionString);
var databaseName = $"taskrunner-tests-{Guid.NewGuid():N}";
var database = client.GetDatabase(databaseName);
return new MongoTaskRunnerTestContext(client, database, runner, databaseName, runner.ConnectionString);
}
catch (Exception ex)
{
Assert.Skip($"Unable to start embedded MongoDB (Mongo2Go): {ex.Message}");
throw new InvalidOperationException(); // Unreachable
}
}
public async ValueTask DisposeAsync()
{
await client.DropDatabaseAsync(databaseName);
runner?.Dispose();
}
public void Dispose()
{
client.DropDatabase(databaseName);
runner?.Dispose();
}
public TaskRunnerMongoOptions CreateMongoOptions()
=> new()
{
ConnectionString = connectionString,
Database = databaseName
};
}

View File

@@ -0,0 +1,40 @@
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Core.Execution.Simulation;
using StellaOps.TaskRunner.Core.Planning;
using StellaOps.TaskRunner.Core.TaskPacks;
namespace StellaOps.TaskRunner.Tests;
public sealed class PackRunStateFactoryTests
{
[Fact]
public void CreateInitialState_AssignsGateReasons()
{
var manifest = TestManifests.Load(TestManifests.Sample);
var planner = new TaskPackPlanner();
var planResult = planner.Plan(manifest);
Assert.True(planResult.Success);
Assert.NotNull(planResult.Plan);
var plan = planResult.Plan!;
var context = new PackRunExecutionContext("run-state-factory", plan, DateTimeOffset.UtcNow);
var graphBuilder = new PackRunExecutionGraphBuilder();
var graph = graphBuilder.Build(plan);
var simulationEngine = new PackRunSimulationEngine();
var timestamp = DateTimeOffset.UtcNow;
var state = PackRunStateFactory.CreateInitialState(context, graph, simulationEngine, timestamp);
Assert.Equal("run-state-factory", state.RunId);
Assert.Equal(plan.Hash, state.PlanHash);
Assert.True(state.Steps.TryGetValue("approval", out var approvalStep));
Assert.Equal(PackRunStepExecutionStatus.Pending, approvalStep.Status);
Assert.Equal("requires-approval", approvalStep.StatusReason);
Assert.True(state.Steps.TryGetValue("plan-step", out var planStep));
Assert.Equal(PackRunStepExecutionStatus.Pending, planStep.Status);
Assert.Null(planStep.StatusReason);
}
}

View File

@@ -1,136 +1,46 @@
<?xml version="1.0" ?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<IsPackable>false</IsPackable>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1"/>
<PackageReference Include="xunit.v3" Version="3.0.0"/>
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3"/>
</ItemGroup>
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest"/>
</ItemGroup>
<ItemGroup>
<Using Include="Xunit"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj"/>
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj"/>
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj"/>
</ItemGroup>
</Project>
<?xml version="1.0"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<IsPackable>false</IsPackable>
<OutputType>Exe</OutputType>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="Mongo2Go" Version="4.1.0" />
<PackageReference Include="xunit.v3" Version="3.0.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj" />
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj" />
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj" />
</ItemGroup>
<ItemGroup>
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<None Include="..\..\..\..\tests\native/openssl-1.1/linux-x64/*"
Link="native/linux-x64/%(Filename)%(Extension)"
CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\..\tests\shared\OpenSslLegacyShim.cs">
<Link>Shared\OpenSslLegacyShim.cs</Link>
</Compile>
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>