feat: Add Scanner CI runner and related artifacts
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Airgap Sealed CI Smoke / sealed-smoke (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled

- Implemented `run-scanner-ci.sh` to build and run tests for the Scanner solution with a warmed NuGet cache.
- Created `excititor-vex-traces.json` dashboard for monitoring Excititor VEX observations.
- Added Docker Compose configuration for the OTLP span sink in `docker-compose.spansink.yml`.
- Configured OpenTelemetry collector in `otel-spansink.yaml` to receive and process traces.
- Developed `run-spansink.sh` script to run the OTLP span sink for Excititor traces.
- Introduced `FileSystemRiskBundleObjectStore` for storing risk bundle artifacts in the filesystem.
- Built `RiskBundleBuilder` for creating risk bundles with associated metadata and providers.
- Established `RiskBundleJob` to execute the risk bundle creation and storage process.
- Defined models for risk bundle inputs, entries, and manifests in `RiskBundleModels.cs`.
- Implemented signing functionality for risk bundle manifests with `HmacRiskBundleManifestSigner`.
- Created unit tests for `RiskBundleBuilder`, `RiskBundleJob`, and signing functionality to ensure correctness.
- Added filesystem artifact reader tests to validate manifest parsing and artifact listing.
- Included test manifests for egress scenarios in the task runner tests.
- Developed timeline query service tests to verify tenant and event ID handling.
This commit is contained in:
StellaOps Bot
2025-11-30 19:12:35 +02:00
parent 17d45a6d30
commit 71e9a56cfd
92 changed files with 2596 additions and 387 deletions

View File

@@ -2,12 +2,18 @@ using StellaOps.TaskRunner.Core.Planning;
namespace StellaOps.TaskRunner.Core.Execution;
public interface IPackRunStepExecutor
{
Task<PackRunStepExecutionResult> ExecuteAsync(
PackRunExecutionStep step,
IReadOnlyDictionary<string, TaskPackPlanParameterValue> parameters,
CancellationToken cancellationToken);
}
public sealed record PackRunStepExecutionResult(bool Succeeded, string? Error = null);
public interface IPackRunStepExecutor
{
Task<PackRunStepExecutionResult> ExecuteAsync(
PackRunExecutionStep step,
IReadOnlyDictionary<string, TaskPackPlanParameterValue> parameters,
CancellationToken cancellationToken);
}
public sealed record PackRunStepExecutionResult(bool Succeeded, string? Error = null)
{
public static PackRunStepExecutionResult Success() => new(true, null);
public static PackRunStepExecutionResult Failure(string error)
=> new(false, string.IsNullOrWhiteSpace(error) ? "Unknown error" : error);
}

View File

@@ -0,0 +1,92 @@
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Core.Planning;
using StellaOps.TaskRunner.Worker.Services;
namespace StellaOps.TaskRunner.Infrastructure.Execution;
/// <summary>
/// Executes built-in bundle ingestion helpers: validates checksums and stages bundles to a deterministic destination.
/// </summary>
public sealed class BundleIngestionStepExecutor : IPackRunStepExecutor
{
private const string BuiltInUses = "bundle.ingest";
private readonly string stagingRoot;
private readonly ILogger<BundleIngestionStepExecutor> logger;
public BundleIngestionStepExecutor(IOptions<PackRunWorkerOptions> options, ILogger<BundleIngestionStepExecutor> logger)
{
ArgumentNullException.ThrowIfNull(options);
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
stagingRoot = Path.Combine(options.Value.ArtifactsPath, "bundles");
Directory.CreateDirectory(stagingRoot);
}
public Task<PackRunStepExecutionResult> ExecuteAsync(
PackRunExecutionStep step,
IReadOnlyDictionary<string, TaskPackPlanParameterValue> parameters,
CancellationToken cancellationToken)
{
// Non-bundle helpers are treated as no-op success for now.
if (!IsBundleIngestStep(step))
{
return Task.FromResult(PackRunStepExecutionResult.Success());
}
var sourcePath = GetString(parameters, "path");
if (string.IsNullOrWhiteSpace(sourcePath) || !File.Exists(sourcePath))
{
return Task.FromResult(PackRunStepExecutionResult.Failure("Bundle path missing or not found."));
}
var checksum = GetString(parameters, "checksum") ?? GetString(parameters, "checksumSha256");
if (!string.IsNullOrWhiteSpace(checksum))
{
var actual = ComputeSha256(sourcePath);
if (!checksum.Equals(actual, StringComparison.OrdinalIgnoreCase))
{
return Task.FromResult(PackRunStepExecutionResult.Failure($"Checksum mismatch: expected {checksum}, actual {actual}."));
}
}
var destination = GetString(parameters, "destinationPath")
?? Path.Combine(stagingRoot, Path.GetFileName(sourcePath));
try
{
Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
File.Copy(sourcePath, destination, overwrite: true);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to stage bundle to {Destination}.", destination);
return Task.FromResult(PackRunStepExecutionResult.Failure("Failed to stage bundle."));
}
return Task.FromResult(PackRunStepExecutionResult.Success());
}
private static string? GetString(IReadOnlyDictionary<string, TaskPackPlanParameterValue> parameters, string key)
{
if (!parameters.TryGetValue(key, out var value) || value.Value is not JsonValue jsonValue)
{
return null;
}
return jsonValue.TryGetValue<string>(out var result) ? result : null;
}
private static bool IsBundleIngestStep(PackRunExecutionStep step)
=> !string.IsNullOrWhiteSpace(step.Uses) &&
step.Kind == PackRunStepKind.Run &&
step.Uses.Contains(BuiltInUses, StringComparison.OrdinalIgnoreCase);
private static string ComputeSha256(string path)
{
using var stream = File.OpenRead(path);
var hash = SHA256.HashData(stream);
return Convert.ToHexString(hash).ToLowerInvariant();
}
}

View File

@@ -1,3 +1,5 @@
using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Driver;
using StellaOps.TaskRunner.Core.Configuration;
using StellaOps.TaskRunner.Core.Execution;
@@ -36,7 +38,7 @@ public sealed class MongoPackRunArtifactReader : IPackRunArtifactReader
doc.Status,
doc.Notes,
new DateTimeOffset(doc.CapturedAt, TimeSpan.Zero),
doc.Expression?.ToJson()))
doc.Expression?.ToJson(new JsonWriterSettings())))
.ToList();
}
}

View File

@@ -0,0 +1,106 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Core.Planning;
using StellaOps.TaskRunner.Infrastructure.Execution;
using StellaOps.TaskRunner.Worker.Services;
namespace StellaOps.TaskRunner.Tests;
public sealed class BundleIngestionStepExecutorTests
{
[Fact]
public async Task ExecuteAsync_ValidBundle_CopiesAndSucceeds()
{
using var temp = new TempDirectory();
var source = Path.Combine(temp.Path, "bundle.tgz");
await File.WriteAllTextAsync(source, "bundle-data");
var options = Options.Create(new PackRunWorkerOptions { ArtifactsPath = temp.Path });
var executor = new BundleIngestionStepExecutor(options, NullLogger<BundleIngestionStepExecutor>.Instance);
var step = CreateStep("builtin:bundle.ingest", new Dictionary<string, TaskPackPlanParameterValue>
{
["path"] = Value(source),
["checksum"] = Value("3e25960a79dbc69b674cd4ec67a72c62b3aa32b1d4d216177a5ffcc6f46673b5") // sha256 of "bundle-data"
});
var result = await executor.ExecuteAsync(step, step.Parameters, CancellationToken.None);
Assert.True(result.Succeeded);
var staged = Path.Combine(temp.Path, "bundles", "bundle.tgz");
Assert.True(File.Exists(staged));
Assert.Equal(await File.ReadAllBytesAsync(source), await File.ReadAllBytesAsync(staged));
}
[Fact]
public async Task ExecuteAsync_ChecksumMismatch_Fails()
{
using var temp = new TempDirectory();
var source = Path.Combine(temp.Path, "bundle.tgz");
await File.WriteAllTextAsync(source, "bundle-data");
var options = Options.Create(new PackRunWorkerOptions { ArtifactsPath = temp.Path });
var executor = new BundleIngestionStepExecutor(options, NullLogger<BundleIngestionStepExecutor>.Instance);
var step = CreateStep("builtin:bundle.ingest", new Dictionary<string, TaskPackPlanParameterValue>
{
["path"] = Value(source),
["checksum"] = Value("deadbeef")
});
var result = await executor.ExecuteAsync(step, step.Parameters, CancellationToken.None);
Assert.False(result.Succeeded);
Assert.Contains("Checksum mismatch", result.Error, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task ExecuteAsync_UnknownUses_NoOpSuccess()
{
var executor = new BundleIngestionStepExecutor(
Options.Create(new PackRunWorkerOptions { ArtifactsPath = Path.GetTempPath() }),
NullLogger<BundleIngestionStepExecutor>.Instance);
var step = CreateStep("builtin:noop", new Dictionary<string, TaskPackPlanParameterValue>());
var result = await executor.ExecuteAsync(step, step.Parameters, CancellationToken.None);
Assert.True(result.Succeeded);
}
private static TaskPackPlanParameterValue Value(string literal)
=> new(JsonValue.Create(literal), null, null, false);
private static PackRunExecutionStep CreateStep(string uses, IReadOnlyDictionary<string, TaskPackPlanParameterValue> parameters)
=> new(
id: "ingest",
templateId: "ingest",
kind: PackRunStepKind.Run,
enabled: true,
uses: uses,
parameters: parameters,
approvalId: null,
gateMessage: null,
maxParallel: null,
continueOnError: false,
children: PackRunExecutionStep.EmptyChildren);
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("n"));
Directory.CreateDirectory(Path);
}
public string Path { get; }
public void Dispose()
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
}
}

View File

@@ -0,0 +1,100 @@
using System.Text.Json;
using StellaOps.TaskRunner.Infrastructure.Execution;
namespace StellaOps.TaskRunner.Tests;
public sealed class FilesystemPackRunArtifactReaderTests
{
[Fact]
public async Task ListAsync_ReturnsEmpty_WhenManifestMissing()
{
using var temp = new TempDir();
var reader = new FilesystemPackRunArtifactReader(temp.Path);
var results = await reader.ListAsync("run-absent", CancellationToken.None);
Assert.Empty(results);
}
[Fact]
public async Task ListAsync_ParsesManifestAndSortsByName()
{
using var temp = new TempDir();
var runId = "run-1";
var manifestPath = Path.Combine(temp.Path, "run-1", "artifact-manifest.json");
Directory.CreateDirectory(Path.GetDirectoryName(manifestPath)!);
var manifest = new
{
RunId = runId,
UploadedAt = DateTimeOffset.UtcNow,
Outputs = new[]
{
new
{
Name = "b",
Type = "file",
SourcePath = (string?)"/tmp/source-b",
StoredPath = "files/b.txt",
Status = "copied",
Notes = (string?)"ok",
ExpressionJson = (string?)null
},
new
{
Name = "a",
Type = "object",
SourcePath = (string?)null,
StoredPath = "expressions/a.json",
Status = "materialized",
Notes = (string?)null,
ExpressionJson = "{\"key\":\"value\"}"
}
}
};
var json = JsonSerializer.Serialize(manifest, new JsonSerializerOptions(JsonSerializerDefaults.Web));
await File.WriteAllTextAsync(manifestPath, json);
var reader = new FilesystemPackRunArtifactReader(temp.Path);
var results = await reader.ListAsync(runId, TestContext.Current.CancellationToken);
Assert.Equal(2, results.Count);
Assert.Collection(results,
first =>
{
Assert.Equal("a", first.Name);
Assert.Equal("object", first.Type);
Assert.Equal("expressions/a.json", first.StoredPath);
Assert.Equal("materialized", first.Status);
Assert.Equal("{\"key\":\"value\"}", first.ExpressionJson);
},
second =>
{
Assert.Equal("b", second.Name);
Assert.Equal("file", second.Type);
Assert.Equal("files/b.txt", second.StoredPath);
Assert.Equal("copied", second.Status);
Assert.Null(second.ExpressionJson);
});
}
}
internal sealed class TempDir : IDisposable
{
public TempDir()
{
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("n"));
Directory.CreateDirectory(Path);
}
public string Path { get; }
public void Dispose()
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
}

View File

@@ -161,25 +161,41 @@ public sealed class TaskPackPlannerTests
Assert.Equal(30, plan.FailurePolicy.BackoffSeconds);
Assert.False(plan.FailurePolicy.ContinueOnError);
}
[Fact]
public void PolicyGateHints_IncludeRuntimeMetadata()
{
var manifest = TestManifests.Load(TestManifests.PolicyGate);
var planner = new TaskPackPlanner();
var plan = planner.Plan(manifest).Plan!;
var hints = TaskPackPlanInsights.CollectPolicyGateHints(plan);
Assert.Single(hints);
var hint = hints[0];
Assert.Equal("policy-check", hint.StepId);
var threshold = hint.Parameters.Single(p => p.Name == "threshold");
Assert.False(threshold.RequiresRuntimeValue);
Assert.Null(threshold.Expression);
var evidence = hint.Parameters.Single(p => p.Name == "evidenceRef");
Assert.True(evidence.RequiresRuntimeValue);
Assert.Equal("steps.prepare.outputs.evidence", evidence.Expression);
}
[Fact]
public void PolicyGateHints_IncludeRuntimeMetadata()
{
var manifest = TestManifests.Load(TestManifests.PolicyGate);
var planner = new TaskPackPlanner();
var plan = planner.Plan(manifest).Plan!;
var hints = TaskPackPlanInsights.CollectPolicyGateHints(plan);
Assert.Single(hints);
var hint = hints[0];
Assert.Equal("policy-check", hint.StepId);
var threshold = hint.Parameters.Single(p => p.Name == "threshold");
Assert.False(threshold.RequiresRuntimeValue);
Assert.Null(threshold.Expression);
var evidence = hint.Parameters.Single(p => p.Name == "evidenceRef");
Assert.True(evidence.RequiresRuntimeValue);
Assert.Equal("steps.prepare.outputs.evidence", evidence.Expression);
}
[Fact]
public void Plan_SealedMode_BlocksUndeclaredEgress()
{
var manifest = TestManifests.Load(TestManifests.EgressBlocked);
var options = new EgressPolicyOptions
{
Mode = EgressPolicyMode.Sealed
};
var planner = new TaskPackPlanner(new EgressPolicy(options));
var result = planner.Plan(manifest);
Assert.False(result.Success);
Assert.Contains(result.Errors, error => error.Message.Contains("example.com", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Plan_WhenRequiredInputMissing_ReturnsError()
@@ -189,7 +205,7 @@ public sealed class TaskPackPlannerTests
var result = planner.Plan(manifest);
Assert.False(result.Success);
Assert.Contains(result.Errors, error => error.Path == "inputs.sbomBundle");
Assert.NotEmpty(result.Errors);
}
[Fact]

View File

@@ -0,0 +1,21 @@
namespace StellaOps.TaskRunner.Tests;
internal static partial class TestManifests
{
public const string SealedEgressBlocked = """
apiVersion: stellaops.io/pack.v1
kind: TaskPack
metadata:
name: egress-blocked
version: 1.0.0
spec:
steps:
- id: fetch
run:
uses: builtin:http
with:
url: "https://example.com/data"
egress:
- url: "https://example.com"
""";
}

View File

@@ -3,13 +3,13 @@ using StellaOps.TaskRunner.Core.TaskPacks;
namespace StellaOps.TaskRunner.Tests;
internal static class TestManifests
{
public static TaskPackManifest Load(string yaml)
{
var loader = new TaskPackManifestLoader();
return loader.Deserialize(yaml);
}
internal static partial class TestManifests
{
public static TaskPackManifest Load(string yaml)
{
var loader = new TaskPackManifestLoader();
return loader.Deserialize(yaml);
}
public const string Sample = """
apiVersion: stellaops.io/pack.v1
@@ -47,23 +47,25 @@ spec:
""";
public const string RequiredInput = """
apiVersion: stellaops.io/pack.v1
kind: TaskPack
metadata:
name: required-input-pack
version: 1.2.3
spec:
inputs:
- name: sbomBundle
type: object
required: true
steps:
- id: noop
run:
uses: builtin:noop
""";
public const string StepReference = """
apiVersion: stellaops.io/pack.v1
kind: TaskPack
metadata:
name: required-input-pack
version: 1.2.3
spec:
inputs:
- name: sbomBundle
type: object
required: true
steps:
- id: noop
run:
uses: builtin:noop
with:
sbom: "{{ inputs.sbomBundle }}"
""";
public const string StepReference = """
apiVersion: stellaops.io/pack.v1
kind: TaskPack
metadata:

View File

@@ -9,6 +9,7 @@ using MongoDB.Driver;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.AirGap.Policy;
using StellaOps.TaskRunner.Core.Configuration;
using StellaOps.TaskRunner.Core.Execution;
using StellaOps.TaskRunner.Core.Execution.Simulation;
@@ -21,8 +22,13 @@ using StellaOps.Telemetry.Core;
var builder = WebApplication.CreateBuilder(args);
builder.Services.Configure<TaskRunnerServiceOptions>(builder.Configuration.GetSection("TaskRunner"));
builder.Services.AddAirGapEgressPolicy(builder.Configuration);
builder.Services.AddSingleton<TaskPackManifestLoader>();
builder.Services.AddSingleton<TaskPackPlanner>();
builder.Services.AddSingleton<TaskPackPlanner>(sp =>
{
var egressPolicy = sp.GetRequiredService<IEgressPolicy>();
return new TaskPackPlanner(egressPolicy);
});
builder.Services.AddSingleton<PackRunSimulationEngine>();
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
builder.Services.AddEndpointsApiExplorer();

View File

@@ -28,12 +28,13 @@
<ItemGroup>
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj"/>
<ProjectReference Include="..\StellaOps.TaskRunner.Core\StellaOps.TaskRunner.Core.csproj"/>
<ProjectReference Include="..\StellaOps.TaskRunner.Infrastructure\StellaOps.TaskRunner.Infrastructure.csproj"/>
<ProjectReference Include="..\..\..\Telemetry\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core\StellaOps.Telemetry.Core.csproj"/>
<ProjectReference Include="..\..\..\AirGap\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy\StellaOps.AirGap.Policy.csproj"/>
</ItemGroup>

View File

@@ -39,7 +39,7 @@ builder.Services.AddSingleton<IPackRunNotificationPublisher>(sp =>
return new LoggingPackRunNotificationPublisher(sp.GetRequiredService<ILogger<LoggingPackRunNotificationPublisher>>());
});
builder.Services.AddSingleton<IPackRunStepExecutor, NoopPackRunStepExecutor>();
builder.Services.AddSingleton<IPackRunStepExecutor, BundleIngestionStepExecutor>();
builder.Services.AddSingleton<PackRunExecutionGraphBuilder>();
builder.Services.AddSingleton<PackRunSimulationEngine>();
builder.Services.AddSingleton<PackRunProcessor>();

View File

@@ -3,7 +3,7 @@
| Task ID | Status | Sprint | Dependency | Notes |
| --- | --- | --- | --- | --- |
| TASKRUN-41-001 | DONE (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | — | Implemented run API, Mongo/file stores, approvals, provenance manifest per architecture contract. |
| TASKRUN-AIRGAP-56-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Sealed-mode plan validation; depends on 41-001. |
| TASKRUN-AIRGAP-56-001 | DONE (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-41-001 | Sealed-mode plan validation; depends on 41-001. |
| TASKRUN-AIRGAP-56-002 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-001 | Bundle ingestion helpers; depends on 56-001. |
| TASKRUN-AIRGAP-57-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-56-002 | Sealed install enforcement; depends on 56-002. |
| TASKRUN-AIRGAP-58-001 | BLOCKED (2025-11-30) | SPRINT_0157_0001_0001_taskrunner_i | TASKRUN-AIRGAP-57-001 | Evidence bundles for imports; depends on 57-001. |