feat: Implement Filesystem and MongoDB provenance writers for PackRun execution context
Some checks failed
Airgap Sealed CI Smoke / sealed-smoke (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled

- Added `FilesystemPackRunProvenanceWriter` to write provenance manifests to the filesystem.
- Introduced `MongoPackRunArtifactReader` to read artifacts from MongoDB.
- Created `MongoPackRunProvenanceWriter` to store provenance manifests in MongoDB.
- Developed unit tests for filesystem and MongoDB provenance writers.
- Established `ITimelineEventStore` and `ITimelineIngestionService` interfaces for timeline event handling.
- Implemented `TimelineIngestionService` to validate and persist timeline events with hashing.
- Created PostgreSQL schema and migration scripts for timeline indexing.
- Added dependency injection support for timeline indexer services.
- Developed tests for timeline ingestion and schema validation.
This commit is contained in:
StellaOps Bot
2025-11-30 15:38:14 +02:00
parent 8f54ffa203
commit 17d45a6d30
276 changed files with 8618 additions and 688 deletions

View File

@@ -118,7 +118,8 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
DateTimeOffset RequestedAt,
DateTimeOffset CreatedAt,
DateTimeOffset UpdatedAt,
IReadOnlyList<StepDocument> Steps)
IReadOnlyList<StepDocument> Steps,
string? TenantId)
{
public static StateDocument FromDomain(PackRunState state)
{
@@ -147,11 +148,12 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
state.RequestedAt,
state.CreatedAt,
state.UpdatedAt,
steps);
steps,
state.TenantId);
}
public PackRunState ToDomain()
{
public PackRunState ToDomain()
{
var steps = Steps.ToDictionary(
step => step.StepId,
step => new PackRunStepStateRecord(
@@ -177,9 +179,10 @@ public sealed class FilePackRunStateStore : IPackRunStateStore
RequestedAt,
CreatedAt,
UpdatedAt,
steps);
steps,
TenantId);
}
}
}
private sealed record StepDocument(
string StepId,

View File

@@ -0,0 +1,75 @@
using System.Text.Json;
using StellaOps.TaskRunner.Core.Execution;
namespace StellaOps.TaskRunner.Infrastructure.Execution;
public sealed class FilesystemPackRunArtifactReader : IPackRunArtifactReader
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
private readonly string rootPath;
public FilesystemPackRunArtifactReader(string rootPath)
{
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
this.rootPath = Path.GetFullPath(rootPath);
}
public async Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
var manifestPath = Path.Combine(rootPath, Sanitize(runId), "artifact-manifest.json");
if (!File.Exists(manifestPath))
{
return Array.Empty<PackRunArtifactRecord>();
}
await using var stream = File.OpenRead(manifestPath);
var manifest = await JsonSerializer.DeserializeAsync<ArtifactManifest>(stream, SerializerOptions, cancellationToken)
.ConfigureAwait(false);
if (manifest is null || manifest.Outputs is null)
{
return Array.Empty<PackRunArtifactRecord>();
}
return manifest.Outputs
.OrderBy(output => output.Name, StringComparer.Ordinal)
.Select(output => new PackRunArtifactRecord(
output.Name,
output.Type,
output.SourcePath,
output.StoredPath,
output.Status,
output.Notes,
manifest.UploadedAt,
output.ExpressionJson))
.ToList();
}
private static string Sanitize(string value)
{
var safe = value.Trim();
foreach (var invalid in Path.GetInvalidFileNameChars())
{
safe = safe.Replace(invalid, '_');
}
return string.IsNullOrWhiteSpace(safe) ? "run" : safe;
}
private sealed record ArtifactManifest(
string RunId,
DateTimeOffset UploadedAt,
List<ArtifactRecord> Outputs);
private sealed record ArtifactRecord(
string Name,
string Type,
string? SourcePath,
string? StoredPath,
string Status,
string? Notes,
string? ExpressionJson);
}

View File

@@ -0,0 +1,56 @@
using System.Text.Json;
using StellaOps.TaskRunner.Core.Execution;
namespace StellaOps.TaskRunner.Infrastructure.Execution;
public sealed class FilesystemPackRunProvenanceWriter : IPackRunProvenanceWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false
};
private readonly string rootPath;
private readonly TimeProvider timeProvider;
public FilesystemPackRunProvenanceWriter(string rootPath, TimeProvider? timeProvider = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
this.rootPath = Path.GetFullPath(rootPath);
this.timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(state);
var completedAt = timeProvider.GetUtcNow();
var manifest = ProvenanceManifestFactory.Create(context, state, completedAt);
var manifestPath = GetPath(context.RunId);
Directory.CreateDirectory(Path.GetDirectoryName(manifestPath)!);
await using var stream = File.Open(manifestPath, FileMode.Create, FileAccess.Write, FileShare.None);
await JsonSerializer.SerializeAsync(stream, manifest, SerializerOptions, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private string GetPath(string runId)
{
var safe = Sanitize(runId);
return Path.Combine(rootPath, "provenance", $"{safe}.json");
}
private static string Sanitize(string value)
{
var result = value.Trim();
foreach (var invalid in Path.GetInvalidFileNameChars())
{
result = result.Replace(invalid, '_');
}
return result;
}
}

View File

@@ -82,25 +82,25 @@ public sealed class MongoPackRunApprovalStore : IPackRunApprovalStore
.ConfigureAwait(false);
}
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
public static IEnumerable<CreateIndexModel<PackRunApprovalDocument>> GetIndexModels()
{
var models = new[]
{
new CreateIndexModel<PackRunApprovalDocument>(
Builders<PackRunApprovalDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.ApprovalId),
new CreateIndexOptions { Unique = true }),
new CreateIndexModel<PackRunApprovalDocument>(
Builders<PackRunApprovalDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Status))
};
yield return new CreateIndexModel<PackRunApprovalDocument>(
Builders<PackRunApprovalDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.ApprovalId),
new CreateIndexOptions { Unique = true, Name = "pack_run_approvals_run_approval" });
target.Indexes.CreateMany(models);
yield return new CreateIndexModel<PackRunApprovalDocument>(
Builders<PackRunApprovalDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Status),
new CreateIndexOptions { Name = "pack_run_approvals_run_status" });
}
private sealed class PackRunApprovalDocument
private static void EnsureIndexes(IMongoCollection<PackRunApprovalDocument> target)
=> target.Indexes.CreateMany(GetIndexModels());
public sealed class PackRunApprovalDocument
{
[BsonId]
public ObjectId Id { get; init; }

View File

@@ -0,0 +1,42 @@
using MongoDB.Driver;
using StellaOps.TaskRunner.Core.Configuration;
using StellaOps.TaskRunner.Core.Execution;
namespace StellaOps.TaskRunner.Infrastructure.Execution;
public sealed class MongoPackRunArtifactReader : IPackRunArtifactReader
{
private readonly IMongoCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument> collection;
public MongoPackRunArtifactReader(IMongoDatabase database, TaskRunnerMongoOptions options)
{
ArgumentNullException.ThrowIfNull(database);
ArgumentNullException.ThrowIfNull(options);
collection = database.GetCollection<MongoPackRunArtifactUploader.PackRunArtifactDocument>(options.ArtifactsCollection);
}
public async Task<IReadOnlyList<PackRunArtifactRecord>> ListAsync(string runId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(runId);
var filter = Builders<MongoPackRunArtifactUploader.PackRunArtifactDocument>.Filter.Eq(doc => doc.RunId, runId);
var documents = await collection
.Find(filter)
.SortBy(doc => doc.Name)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents
.Select(doc => new PackRunArtifactRecord(
doc.Name,
doc.Type,
doc.SourcePath,
doc.StoredPath,
doc.Status,
doc.Notes,
new DateTimeOffset(doc.CapturedAt, TimeSpan.Zero),
doc.Expression?.ToJson()))
.ToList();
}
}

View File

@@ -149,24 +149,23 @@ public sealed class MongoPackRunArtifactUploader : IPackRunArtifactUploader
return parameter.Value;
}
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
public static IEnumerable<CreateIndexModel<PackRunArtifactDocument>> GetIndexModels()
{
var models = new[]
{
new CreateIndexModel<PackRunArtifactDocument>(
Builders<PackRunArtifactDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Name),
new CreateIndexOptions { Unique = true }),
new CreateIndexModel<PackRunArtifactDocument>(
Builders<PackRunArtifactDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Status))
};
yield return new CreateIndexModel<PackRunArtifactDocument>(
Builders<PackRunArtifactDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Name),
new CreateIndexOptions { Unique = true, Name = "pack_artifacts_run_name" });
target.Indexes.CreateMany(models);
yield return new CreateIndexModel<PackRunArtifactDocument>(
Builders<PackRunArtifactDocument>.IndexKeys
.Ascending(document => document.RunId),
new CreateIndexOptions { Name = "pack_artifacts_run" });
}
private static void EnsureIndexes(IMongoCollection<PackRunArtifactDocument> target)
=> target.Indexes.CreateMany(GetIndexModels());
public sealed class PackRunArtifactDocument
{
[BsonId]

View File

@@ -89,24 +89,24 @@ public sealed class MongoPackRunLogStore : IPackRunLogStore
.ConfigureAwait(false);
}
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
public static IEnumerable<CreateIndexModel<PackRunLogDocument>> GetIndexModels()
{
var models = new[]
{
new CreateIndexModel<PackRunLogDocument>(
Builders<PackRunLogDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Sequence),
new CreateIndexOptions { Unique = true }),
new CreateIndexModel<PackRunLogDocument>(
Builders<PackRunLogDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Timestamp))
};
yield return new CreateIndexModel<PackRunLogDocument>(
Builders<PackRunLogDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Sequence),
new CreateIndexOptions { Unique = true, Name = "pack_run_logs_run_sequence" });
target.Indexes.CreateMany(models);
yield return new CreateIndexModel<PackRunLogDocument>(
Builders<PackRunLogDocument>.IndexKeys
.Ascending(document => document.RunId)
.Ascending(document => document.Timestamp),
new CreateIndexOptions { Name = "pack_run_logs_run_timestamp" });
}
private static void EnsureIndexes(IMongoCollection<PackRunLogDocument> target)
=> target.Indexes.CreateMany(GetIndexModels());
public sealed class PackRunLogDocument
{
[BsonId]

View File

@@ -0,0 +1,67 @@
using System.Text.Json;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.TaskRunner.Core.Configuration;
using StellaOps.TaskRunner.Core.Execution;
namespace StellaOps.TaskRunner.Infrastructure.Execution;
public sealed class MongoPackRunProvenanceWriter : IPackRunProvenanceWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
private readonly IMongoCollection<ProvenanceDocument> collection;
private readonly TimeProvider timeProvider;
public MongoPackRunProvenanceWriter(IMongoDatabase database, TaskRunnerMongoOptions options, TimeProvider? timeProvider = null)
{
ArgumentNullException.ThrowIfNull(database);
ArgumentNullException.ThrowIfNull(options);
collection = database.GetCollection<ProvenanceDocument>(options.ArtifactsCollection);
this.timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task WriteAsync(PackRunExecutionContext context, PackRunState state, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(state);
var completedAt = timeProvider.GetUtcNow();
var manifest = ProvenanceManifestFactory.Create(context, state, completedAt);
var manifestJson = JsonSerializer.Serialize(manifest, SerializerOptions);
var manifestDocument = BsonDocument.Parse(manifestJson);
var document = new ProvenanceDocument
{
RunId = context.RunId,
Name = "provenance-manifest",
Type = "object",
Status = "materialized",
CapturedAt = completedAt.UtcDateTime,
Expression = manifestDocument
};
var filter = Builders<ProvenanceDocument>.Filter.And(
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.RunId, context.RunId),
Builders<ProvenanceDocument>.Filter.Eq(doc => doc.Name, document.Name));
var options = new ReplaceOptions { IsUpsert = true };
await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
private sealed class ProvenanceDocument
{
public string RunId { get; init; } = default!;
public string Name { get; init; } = default!;
public string Type { get; init; } = default!;
public string Status { get; init; } = default!;
public DateTime CapturedAt { get; init; }
public BsonDocument Expression { get; init; } = default!;
}
}

View File

@@ -62,20 +62,23 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
.ToList();
}
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
public static IEnumerable<CreateIndexModel<PackRunStateDocument>> GetIndexModels()
{
var models = new[]
{
new CreateIndexModel<PackRunStateDocument>(
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt)),
new CreateIndexModel<PackRunStateDocument>(
Builders<PackRunStateDocument>.IndexKeys.Ascending(document => document.PlanHash))
};
yield return new CreateIndexModel<PackRunStateDocument>(
Builders<PackRunStateDocument>.IndexKeys.Descending(document => document.UpdatedAt),
new CreateIndexOptions { Name = "pack_runs_updatedAt_desc" });
target.Indexes.CreateMany(models);
yield return new CreateIndexModel<PackRunStateDocument>(
Builders<PackRunStateDocument>.IndexKeys
.Ascending(document => document.TenantId)
.Descending(document => document.UpdatedAt),
new CreateIndexOptions { Name = "pack_runs_tenant_updatedAt_desc", Sparse = true });
}
private sealed class PackRunStateDocument
private static void EnsureIndexes(IMongoCollection<PackRunStateDocument> target)
=> target.Indexes.CreateMany(GetIndexModels());
public sealed class PackRunStateDocument
{
[BsonId]
public string RunId { get; init; } = default!;
@@ -94,6 +97,8 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
public List<PackRunStepDocument> Steps { get; init; } = new();
public string? TenantId { get; init; }
public static PackRunStateDocument FromDomain(PackRunState state)
{
var planDocument = BsonDocument.Parse(JsonSerializer.Serialize(state.Plan, SerializerOptions));
@@ -113,7 +118,8 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
RequestedAt = state.RequestedAt.UtcDateTime,
CreatedAt = state.CreatedAt.UtcDateTime,
UpdatedAt = state.UpdatedAt.UtcDateTime,
Steps = steps
Steps = steps,
TenantId = state.TenantId
};
}
@@ -139,11 +145,12 @@ public sealed class MongoPackRunStateStore : IPackRunStateStore
new DateTimeOffset(RequestedAt, TimeSpan.Zero),
new DateTimeOffset(CreatedAt, TimeSpan.Zero),
new DateTimeOffset(UpdatedAt, TimeSpan.Zero),
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords));
new ReadOnlyDictionary<string, PackRunStepStateRecord>(stepRecords),
TenantId);
}
}
private sealed class PackRunStepDocument
public sealed class PackRunStepDocument
{
public string StepId { get; init; } = default!;

View File

@@ -48,6 +48,16 @@ public sealed class PackRunApprovalDecisionService
return PackRunApprovalDecisionResult.NotFound;
}
if (!string.Equals(state.PlanHash, request.PlanHash, StringComparison.Ordinal))
{
_logger.LogWarning(
"Approval decision for run {RunId} rejected plan hash mismatch (expected {Expected}, got {Actual}).",
runId,
state.PlanHash,
request.PlanHash);
return PackRunApprovalDecisionResult.PlanHashMismatch;
}
var requestedAt = state.RequestedAt != default ? state.RequestedAt : state.CreatedAt;
var coordinator = PackRunApprovalCoordinator.Restore(state.Plan, approvals, requestedAt);
@@ -96,6 +106,7 @@ public sealed class PackRunApprovalDecisionService
public sealed record PackRunApprovalDecisionRequest(
string RunId,
string ApprovalId,
string PlanHash,
PackRunApprovalDecisionType Decision,
string? ActorId,
string? Summary);
@@ -110,6 +121,7 @@ public enum PackRunApprovalDecisionType
public sealed record PackRunApprovalDecisionResult(string Status)
{
public static PackRunApprovalDecisionResult NotFound { get; } = new("not_found");
public static PackRunApprovalDecisionResult PlanHashMismatch { get; } = new("plan_hash_mismatch");
public static PackRunApprovalDecisionResult Applied { get; } = new("applied");
public static PackRunApprovalDecisionResult Resumed { get; } = new("resumed");