Add impact index fixture and filesystem artifact uploader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Introduced a sample BOM index JSON file for impact index testing. - Created unit tests for the impact index fixture to ensure proper loading of sample images. - Implemented the FilesystemPackRunArtifactUploader class to handle artifact uploads to the local filesystem. - Added comprehensive tests for the FilesystemPackRunArtifactUploader, covering file copying, missing files, and expression outputs.
This commit is contained in:
master
@@ -0,0 +1,239 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
/// <summary>
|
||||
/// Stores pack run artifacts on the local file system so they can be mirrored to the eventual remote store.
|
||||
/// </summary>
|
||||
public sealed class FilesystemPackRunArtifactUploader : IPackRunArtifactUploader
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
private readonly string rootPath;
|
||||
private readonly ILogger<FilesystemPackRunArtifactUploader> logger;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public FilesystemPackRunArtifactUploader(
|
||||
string rootPath,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<FilesystemPackRunArtifactUploader> logger)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
|
||||
|
||||
this.rootPath = Path.GetFullPath(rootPath);
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
|
||||
Directory.CreateDirectory(this.rootPath);
|
||||
}
|
||||
|
||||
public async Task UploadAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunState state,
|
||||
IReadOnlyList<TaskPackPlanOutput> outputs,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
ArgumentNullException.ThrowIfNull(outputs);
|
||||
|
||||
if (outputs.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var destinationRoot = Path.Combine(rootPath, SanitizeFileName(context.RunId));
|
||||
var filesRoot = Path.Combine(destinationRoot, "files");
|
||||
var expressionsRoot = Path.Combine(destinationRoot, "expressions");
|
||||
|
||||
Directory.CreateDirectory(destinationRoot);
|
||||
|
||||
var manifest = new ArtifactManifest(
|
||||
context.RunId,
|
||||
timeProvider.GetUtcNow(),
|
||||
new List<ArtifactRecord>(outputs.Count));
|
||||
|
||||
foreach (var output in outputs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var record = await ProcessOutputAsync(
|
||||
context,
|
||||
output,
|
||||
destinationRoot,
|
||||
filesRoot,
|
||||
expressionsRoot,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
manifest.Outputs.Add(record);
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(destinationRoot, "artifact-manifest.json");
|
||||
await using (var stream = File.Open(manifestPath, FileMode.Create, FileAccess.Write, FileShare.None))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, manifest, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Pack run {RunId} artifact manifest written to {Path} with {Count} output entries.",
|
||||
context.RunId,
|
||||
manifestPath,
|
||||
manifest.Outputs.Count);
|
||||
}
|
||||
|
||||
private async Task<ArtifactRecord> ProcessOutputAsync(
|
||||
PackRunExecutionContext context,
|
||||
TaskPackPlanOutput output,
|
||||
string destinationRoot,
|
||||
string filesRoot,
|
||||
string expressionsRoot,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var sourcePath = ResolveString(output.Path);
|
||||
var expressionNode = ResolveExpression(output.Expression);
|
||||
var status = "skipped";
|
||||
string? storedPath = null;
|
||||
string? notes = null;
|
||||
|
||||
if (IsFileOutput(output))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourcePath))
|
||||
{
|
||||
status = "unresolved";
|
||||
notes = "Output path requires runtime value.";
|
||||
}
|
||||
else if (!File.Exists(sourcePath))
|
||||
{
|
||||
status = "missing";
|
||||
notes = $"Source file '{sourcePath}' not found.";
|
||||
logger.LogWarning(
|
||||
"Pack run {RunId} output {Output} referenced missing file {Path}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
sourcePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.CreateDirectory(filesRoot);
|
||||
|
||||
var destinationPath = Path.Combine(filesRoot, DetermineDestinationFileName(output, sourcePath));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
|
||||
await CopyFileAsync(sourcePath, destinationPath, cancellationToken).ConfigureAwait(false);
|
||||
storedPath = GetRelativePath(destinationPath, destinationRoot);
|
||||
status = "copied";
|
||||
|
||||
logger.LogInformation(
|
||||
"Pack run {RunId} output {Output} copied to {Destination}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
destinationPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (expressionNode is not null)
|
||||
{
|
||||
Directory.CreateDirectory(expressionsRoot);
|
||||
|
||||
var expressionPath = Path.Combine(
|
||||
expressionsRoot,
|
||||
$"{SanitizeFileName(output.Name)}.json");
|
||||
|
||||
var json = expressionNode.ToJsonString(SerializerOptions);
|
||||
await File.WriteAllTextAsync(expressionPath, json, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
storedPath ??= GetRelativePath(expressionPath, destinationRoot);
|
||||
status = status == "copied" ? "copied" : "materialized";
|
||||
}
|
||||
|
||||
return new ArtifactRecord(
|
||||
output.Name,
|
||||
output.Type,
|
||||
sourcePath,
|
||||
storedPath,
|
||||
status,
|
||||
notes);
|
||||
}
|
||||
|
||||
private static async Task CopyFileAsync(string sourcePath, string destinationPath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var source = File.Open(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
await using var destination = File.Open(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static bool IsFileOutput(TaskPackPlanOutput output)
|
||||
=> string.Equals(output.Type, "file", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string DetermineDestinationFileName(TaskPackPlanOutput output, string sourcePath)
|
||||
{
|
||||
var extension = Path.GetExtension(sourcePath);
|
||||
var baseName = SanitizeFileName(output.Name);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(extension) &&
|
||||
!baseName.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return baseName + extension;
|
||||
}
|
||||
|
||||
return baseName;
|
||||
}
|
||||
|
||||
private static string? ResolveString(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue || parameter.Value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parameter.Value is JsonValue jsonValue && jsonValue.TryGetValue<string>(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static JsonNode? ResolveExpression(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return parameter.Value;
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
{
|
||||
var result = value;
|
||||
foreach (var invalid in Path.GetInvalidFileNameChars())
|
||||
{
|
||||
result = result.Replace(invalid, '_');
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(result) ? "output" : result;
|
||||
}
|
||||
|
||||
private static string GetRelativePath(string path, string root)
|
||||
=> Path.GetRelativePath(root, path)
|
||||
.Replace('\\', '/');
|
||||
|
||||
private sealed record ArtifactManifest(string RunId, DateTimeOffset UploadedAt, List<ArtifactRecord> Outputs);
|
||||
|
||||
private sealed record ArtifactRecord(
|
||||
string Name,
|
||||
string Type,
|
||||
string? SourcePath,
|
||||
string? StoredPath,
|
||||
string Status,
|
||||
string? Notes);
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
using StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TaskRunner.Tests;
|
||||
|
||||
public sealed class FilesystemPackRunArtifactUploaderTests : IDisposable
|
||||
{
|
||||
private readonly string artifactsRoot;
|
||||
|
||||
public FilesystemPackRunArtifactUploaderTests()
|
||||
{
|
||||
artifactsRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("n"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CopiesFileOutputs()
|
||||
{
|
||||
var sourceFile = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():n}.txt");
|
||||
await File.WriteAllTextAsync(sourceFile, "artifact-content", TestContext.Current.CancellationToken);
|
||||
|
||||
var uploader = CreateUploader();
|
||||
var output = CreateFileOutput("bundle", sourceFile);
|
||||
var context = CreateContext();
|
||||
var state = CreateState(context);
|
||||
|
||||
await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken);
|
||||
|
||||
var runPath = Path.Combine(artifactsRoot, context.RunId);
|
||||
var filesDirectory = Path.Combine(runPath, "files");
|
||||
var copiedFiles = Directory.GetFiles(filesDirectory);
|
||||
Assert.Single(copiedFiles);
|
||||
Assert.Equal("bundle.txt", Path.GetFileName(copiedFiles[0]));
|
||||
Assert.Equal("artifact-content", await File.ReadAllTextAsync(copiedFiles[0], TestContext.Current.CancellationToken));
|
||||
|
||||
var manifest = await ReadManifestAsync(runPath);
|
||||
Assert.Single(manifest.Outputs);
|
||||
Assert.Equal("copied", manifest.Outputs[0].Status);
|
||||
Assert.Equal("files/bundle.txt", manifest.Outputs[0].StoredPath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordsMissingFilesWithoutThrowing()
|
||||
{
|
||||
var uploader = CreateUploader();
|
||||
var output = CreateFileOutput("missing", Path.Combine(Path.GetTempPath(), "does-not-exist.txt"));
|
||||
var context = CreateContext();
|
||||
var state = CreateState(context);
|
||||
|
||||
await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken);
|
||||
|
||||
var manifest = await ReadManifestAsync(Path.Combine(artifactsRoot, context.RunId));
|
||||
Assert.Equal("missing", manifest.Outputs[0].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WritesExpressionOutputsAsJson()
|
||||
{
|
||||
var uploader = CreateUploader();
|
||||
var output = CreateExpressionOutput("metadata", JsonNode.Parse("""{"foo":"bar"}""")!);
|
||||
var context = CreateContext();
|
||||
var state = CreateState(context);
|
||||
|
||||
await uploader.UploadAsync(context, state, new[] { output }, TestContext.Current.CancellationToken);
|
||||
|
||||
var expressionPath = Path.Combine(artifactsRoot, context.RunId, "expressions", "metadata.json");
|
||||
Assert.True(File.Exists(expressionPath));
|
||||
|
||||
var manifest = await ReadManifestAsync(Path.Combine(artifactsRoot, context.RunId));
|
||||
Assert.Equal("materialized", manifest.Outputs[0].Status);
|
||||
Assert.Equal("expressions/metadata.json", manifest.Outputs[0].StoredPath);
|
||||
}
|
||||
|
||||
private FilesystemPackRunArtifactUploader CreateUploader()
|
||||
=> new(artifactsRoot, TimeProvider.System, NullLogger<FilesystemPackRunArtifactUploader>.Instance);
|
||||
|
||||
private static TaskPackPlanOutput CreateFileOutput(string name, string path)
|
||||
=> new(
|
||||
name,
|
||||
Type: "file",
|
||||
Path: new TaskPackPlanParameterValue(JsonValue.Create(path), null, null, false),
|
||||
Expression: null);
|
||||
|
||||
private static TaskPackPlanOutput CreateExpressionOutput(string name, JsonNode expression)
|
||||
=> new(
|
||||
name,
|
||||
Type: "object",
|
||||
Path: null,
|
||||
Expression: new TaskPackPlanParameterValue(expression, null, null, false));
|
||||
|
||||
private static PackRunExecutionContext CreateContext()
|
||||
=> new("run-" + Guid.NewGuid().ToString("n"), CreatePlan(), DateTimeOffset.UtcNow);
|
||||
|
||||
private static PackRunState CreateState(PackRunExecutionContext context)
|
||||
=> PackRunState.Create(
|
||||
runId: context.RunId,
|
||||
planHash: context.Plan.Hash,
|
||||
context.Plan,
|
||||
failurePolicy: new TaskPackPlanFailurePolicy(1, 1, false),
|
||||
requestedAt: DateTimeOffset.UtcNow,
|
||||
steps: new Dictionary<string, PackRunStepStateRecord>(StringComparer.Ordinal),
|
||||
timestamp: DateTimeOffset.UtcNow);
|
||||
|
||||
private static TaskPackPlan CreatePlan()
|
||||
{
|
||||
return new TaskPackPlan(
|
||||
new TaskPackPlanMetadata("sample-pack", "1.0.0", null, Array.Empty<string>()),
|
||||
new Dictionary<string, JsonNode?>(StringComparer.Ordinal),
|
||||
Array.Empty<TaskPackPlanStep>(),
|
||||
hash: "hash",
|
||||
approvals: Array.Empty<TaskPackPlanApproval>(),
|
||||
secrets: Array.Empty<TaskPackPlanSecret>(),
|
||||
outputs: Array.Empty<TaskPackPlanOutput>(),
|
||||
failurePolicy: new TaskPackPlanFailurePolicy(1, 1, false));
|
||||
}
|
||||
|
||||
private static async Task<ArtifactManifestModel> ReadManifestAsync(string runPath)
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(Path.Combine(runPath, "artifact-manifest.json"), TestContext.Current.CancellationToken);
|
||||
return JsonSerializer.Deserialize<ArtifactManifestModel>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(artifactsRoot))
|
||||
{
|
||||
Directory.Delete(artifactsRoot, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record ArtifactManifestModel(string RunId, DateTimeOffset UploadedAt, List<ArtifactRecordModel> Outputs);
|
||||
|
||||
private sealed record ArtifactRecordModel(string Name, string Type, string? SourcePath, string? StoredPath, string Status, string? Notes);
|
||||
}
|
||||
@@ -51,7 +51,13 @@ builder.Services.AddSingleton<IPackRunStepExecutor, NoopPackRunStepExecutor>();
|
||||
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
|
||||
builder.Services.AddSingleton<PackRunSimulationEngine>();
|
||||
builder.Services.AddSingleton<PackRunProcessor>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader, LoggingPackRunArtifactUploader>();
|
||||
builder.Services.AddSingleton<IPackRunArtifactUploader>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<PackRunWorkerOptions>>().Value;
|
||||
var timeProvider = sp.GetService<TimeProvider>();
|
||||
var logger = sp.GetRequiredService<ILogger<FilesystemPackRunArtifactUploader>>();
|
||||
return new FilesystemPackRunArtifactUploader(options.ArtifactsPath, timeProvider, logger);
|
||||
});
|
||||
builder.Services.AddHostedService<PackRunWorkerService>();
|
||||
|
||||
var host = builder.Build();
|
||||
|
||||
@@ -4,11 +4,13 @@ public sealed class PackRunWorkerOptions
|
||||
{
|
||||
public TimeSpan IdleDelay { get; set; } = TimeSpan.FromSeconds(1);
|
||||
|
||||
public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
|
||||
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
|
||||
public string QueuePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue");
|
||||
|
||||
public string ArchivePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "queue", "archive");
|
||||
|
||||
public string ApprovalStorePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "approvals");
|
||||
|
||||
public string RunStatePath { get; set; } = Path.Combine(AppContext.BaseDirectory, "state", "runs");
|
||||
|
||||
public string ArtifactsPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "artifacts");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user