Add impact index fixture and filesystem artifact uploader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Introduced a sample BOM index JSON file for impact index testing. - Created unit tests for the impact index fixture to ensure proper loading of sample images. - Implemented the FilesystemPackRunArtifactUploader class to handle artifact uploads to the local filesystem. - Added comprehensive tests for the FilesystemPackRunArtifactUploader, covering file copying, missing files, and expression outputs.
This commit is contained in:
master
@@ -0,0 +1,239 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.TaskRunner.Core.Execution;
|
||||
using StellaOps.TaskRunner.Core.Planning;
|
||||
|
||||
namespace StellaOps.TaskRunner.Infrastructure.Execution;
|
||||
|
||||
/// <summary>
|
||||
/// Stores pack run artifacts on the local file system so they can be mirrored to the eventual remote store.
|
||||
/// </summary>
|
||||
public sealed class FilesystemPackRunArtifactUploader : IPackRunArtifactUploader
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
private readonly string rootPath;
|
||||
private readonly ILogger<FilesystemPackRunArtifactUploader> logger;
|
||||
private readonly TimeProvider timeProvider;
|
||||
|
||||
public FilesystemPackRunArtifactUploader(
|
||||
string rootPath,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<FilesystemPackRunArtifactUploader> logger)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
|
||||
|
||||
this.rootPath = Path.GetFullPath(rootPath);
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.timeProvider = timeProvider ?? TimeProvider.System;
|
||||
|
||||
Directory.CreateDirectory(this.rootPath);
|
||||
}
|
||||
|
||||
public async Task UploadAsync(
|
||||
PackRunExecutionContext context,
|
||||
PackRunState state,
|
||||
IReadOnlyList<TaskPackPlanOutput> outputs,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
ArgumentNullException.ThrowIfNull(outputs);
|
||||
|
||||
if (outputs.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var destinationRoot = Path.Combine(rootPath, SanitizeFileName(context.RunId));
|
||||
var filesRoot = Path.Combine(destinationRoot, "files");
|
||||
var expressionsRoot = Path.Combine(destinationRoot, "expressions");
|
||||
|
||||
Directory.CreateDirectory(destinationRoot);
|
||||
|
||||
var manifest = new ArtifactManifest(
|
||||
context.RunId,
|
||||
timeProvider.GetUtcNow(),
|
||||
new List<ArtifactRecord>(outputs.Count));
|
||||
|
||||
foreach (var output in outputs)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var record = await ProcessOutputAsync(
|
||||
context,
|
||||
output,
|
||||
destinationRoot,
|
||||
filesRoot,
|
||||
expressionsRoot,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
manifest.Outputs.Add(record);
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(destinationRoot, "artifact-manifest.json");
|
||||
await using (var stream = File.Open(manifestPath, FileMode.Create, FileAccess.Write, FileShare.None))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, manifest, SerializerOptions, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Pack run {RunId} artifact manifest written to {Path} with {Count} output entries.",
|
||||
context.RunId,
|
||||
manifestPath,
|
||||
manifest.Outputs.Count);
|
||||
}
|
||||
|
||||
private async Task<ArtifactRecord> ProcessOutputAsync(
|
||||
PackRunExecutionContext context,
|
||||
TaskPackPlanOutput output,
|
||||
string destinationRoot,
|
||||
string filesRoot,
|
||||
string expressionsRoot,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var sourcePath = ResolveString(output.Path);
|
||||
var expressionNode = ResolveExpression(output.Expression);
|
||||
var status = "skipped";
|
||||
string? storedPath = null;
|
||||
string? notes = null;
|
||||
|
||||
if (IsFileOutput(output))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sourcePath))
|
||||
{
|
||||
status = "unresolved";
|
||||
notes = "Output path requires runtime value.";
|
||||
}
|
||||
else if (!File.Exists(sourcePath))
|
||||
{
|
||||
status = "missing";
|
||||
notes = $"Source file '{sourcePath}' not found.";
|
||||
logger.LogWarning(
|
||||
"Pack run {RunId} output {Output} referenced missing file {Path}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
sourcePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.CreateDirectory(filesRoot);
|
||||
|
||||
var destinationPath = Path.Combine(filesRoot, DetermineDestinationFileName(output, sourcePath));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
|
||||
await CopyFileAsync(sourcePath, destinationPath, cancellationToken).ConfigureAwait(false);
|
||||
storedPath = GetRelativePath(destinationPath, destinationRoot);
|
||||
status = "copied";
|
||||
|
||||
logger.LogInformation(
|
||||
"Pack run {RunId} output {Output} copied to {Destination}.",
|
||||
context.RunId,
|
||||
output.Name,
|
||||
destinationPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (expressionNode is not null)
|
||||
{
|
||||
Directory.CreateDirectory(expressionsRoot);
|
||||
|
||||
var expressionPath = Path.Combine(
|
||||
expressionsRoot,
|
||||
$"{SanitizeFileName(output.Name)}.json");
|
||||
|
||||
var json = expressionNode.ToJsonString(SerializerOptions);
|
||||
await File.WriteAllTextAsync(expressionPath, json, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
storedPath ??= GetRelativePath(expressionPath, destinationRoot);
|
||||
status = status == "copied" ? "copied" : "materialized";
|
||||
}
|
||||
|
||||
return new ArtifactRecord(
|
||||
output.Name,
|
||||
output.Type,
|
||||
sourcePath,
|
||||
storedPath,
|
||||
status,
|
||||
notes);
|
||||
}
|
||||
|
||||
private static async Task CopyFileAsync(string sourcePath, string destinationPath, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var source = File.Open(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
await using var destination = File.Open(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static bool IsFileOutput(TaskPackPlanOutput output)
|
||||
=> string.Equals(output.Type, "file", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static string DetermineDestinationFileName(TaskPackPlanOutput output, string sourcePath)
|
||||
{
|
||||
var extension = Path.GetExtension(sourcePath);
|
||||
var baseName = SanitizeFileName(output.Name);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(extension) &&
|
||||
!baseName.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return baseName + extension;
|
||||
}
|
||||
|
||||
return baseName;
|
||||
}
|
||||
|
||||
private static string? ResolveString(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue || parameter.Value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parameter.Value is JsonValue jsonValue && jsonValue.TryGetValue<string>(out var value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static JsonNode? ResolveExpression(TaskPackPlanParameterValue? parameter)
|
||||
{
|
||||
if (parameter is null || parameter.RequiresRuntimeValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return parameter.Value;
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
{
|
||||
var result = value;
|
||||
foreach (var invalid in Path.GetInvalidFileNameChars())
|
||||
{
|
||||
result = result.Replace(invalid, '_');
|
||||
}
|
||||
|
||||
return string.IsNullOrWhiteSpace(result) ? "output" : result;
|
||||
}
|
||||
|
||||
private static string GetRelativePath(string path, string root)
|
||||
=> Path.GetRelativePath(root, path)
|
||||
.Replace('\\', '/');
|
||||
|
||||
private sealed record ArtifactManifest(string RunId, DateTimeOffset UploadedAt, List<ArtifactRecord> Outputs);
|
||||
|
||||
private sealed record ArtifactRecord(
|
||||
string Name,
|
||||
string Type,
|
||||
string? SourcePath,
|
||||
string? StoredPath,
|
||||
string Status,
|
||||
string? Notes);
|
||||
}
|
||||
Reference in New Issue
Block a user