feat: Add UI benchmark driver and scenarios for graph interactions
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

- Introduced `ui_bench_driver.mjs` to read scenarios and fixture manifest, generating a deterministic run plan.
- Created `ui_bench_plan.md` outlining the purpose, scope, and next steps for the benchmark.
- Added `ui_bench_scenarios.json` containing various scenarios for graph UI interactions.
- Implemented tests for CLI commands, ensuring bundle verification and telemetry defaults.
- Developed schemas for orchestrator components, including replay manifests and event envelopes.
- Added mock API for risk management, including listing and statistics functionalities.
- Implemented models for risk profiles and query options to support the new API.
This commit is contained in:
StellaOps Bot
2025-12-02 01:28:17 +02:00
parent 909d9b6220
commit 44171930ff
94 changed files with 3606 additions and 271 deletions

View File

@@ -1,3 +1,5 @@
using StellaOps.Orchestrator.Core.Hashing;
namespace StellaOps.Orchestrator.Core.Domain;
/// <summary>
@@ -90,9 +92,22 @@ public sealed record AuditEntry(
var entryId = Guid.NewGuid();
var occurredAt = DateTimeOffset.UtcNow;
// Compute content hash from entry data
var contentToHash = $"{entryId}|{tenantId}|{eventType}|{resourceType}|{resourceId}|{actorId}|{actorType}|{description}|{oldState}|{newState}|{occurredAt:O}|{sequenceNumber}";
var contentHash = ComputeSha256(contentToHash);
// Compute canonical hash from immutable content
var contentHash = CanonicalJsonHasher.ComputeCanonicalSha256(new
{
entryId,
tenantId,
eventType,
resourceType,
resourceId,
actorId,
actorType,
description,
oldState,
newState,
occurredAt,
sequenceNumber
});
return new AuditEntry(
EntryId: entryId,
@@ -122,8 +137,21 @@ public sealed record AuditEntry(
/// </summary>
public bool VerifyIntegrity()
{
var contentToHash = $"{EntryId}|{TenantId}|{EventType}|{ResourceType}|{ResourceId}|{ActorId}|{ActorType}|{Description}|{OldState}|{NewState}|{OccurredAt:O}|{SequenceNumber}";
var computed = ComputeSha256(contentToHash);
var computed = CanonicalJsonHasher.ComputeCanonicalSha256(new
{
EntryId,
TenantId,
EventType,
ResourceType,
ResourceId,
ActorId,
ActorType,
Description,
OldState,
NewState,
OccurredAt,
SequenceNumber
});
return string.Equals(ContentHash, computed, StringComparison.OrdinalIgnoreCase);
}

View File

@@ -0,0 +1,72 @@
using System.Collections.Immutable;
using System.Text.Json.Serialization;
using StellaOps.Orchestrator.Core.Hashing;
namespace StellaOps.Orchestrator.Core.Domain.Replay;
/// <summary>
/// Deterministic replay manifest that captures all inputs required to faithfully re-run a job.
/// Aligns with replay-manifest.schema.json and is hashed via canonical JSON.
/// </summary>
public sealed record ReplayManifest(
[property: JsonPropertyName("schemaVersion")] string SchemaVersion,
[property: JsonPropertyName("jobId")] string JobId,
[property: JsonPropertyName("replayOf")] string ReplayOf,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("reason")] string? Reason,
[property: JsonPropertyName("inputs")] ReplayInputs Inputs,
[property: JsonPropertyName("artifacts")] ImmutableArray<ReplayArtifact> Artifacts)
{
public static ReplayManifest Create(
string jobId,
string replayOf,
ReplayInputs inputs,
IEnumerable<ReplayArtifact>? artifacts = null,
string schemaVersion = "orch.replay.v1",
string? reason = null,
DateTimeOffset? createdAt = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
ArgumentException.ThrowIfNullOrWhiteSpace(replayOf);
ArgumentNullException.ThrowIfNull(inputs);
return new ReplayManifest(
SchemaVersion: schemaVersion,
JobId: jobId,
ReplayOf: replayOf,
CreatedAt: createdAt ?? DateTimeOffset.UtcNow,
Reason: string.IsNullOrWhiteSpace(reason) ? null : reason,
Inputs: inputs,
Artifacts: artifacts is null ? ImmutableArray<ReplayArtifact>.Empty : ImmutableArray.CreateRange(artifacts));
}
/// <summary>
/// Deterministic SHA-256 over canonical JSON representation of the manifest.
/// </summary>
public string ComputeHash() => CanonicalJsonHasher.ComputeCanonicalSha256(this);
}
public sealed record ReplayInputs(
[property: JsonPropertyName("policyHash")] string PolicyHash,
[property: JsonPropertyName("graphRevisionId")] string GraphRevisionId,
[property: JsonPropertyName("latticeHash")] string? LatticeHash,
[property: JsonPropertyName("toolImages")] ImmutableArray<string> ToolImages,
[property: JsonPropertyName("seeds")] ReplaySeeds Seeds,
[property: JsonPropertyName("timeSource")] ReplayTimeSource TimeSource,
[property: JsonPropertyName("env")] ImmutableDictionary<string, string> Env);
public sealed record ReplaySeeds(
[property: JsonPropertyName("rng")] int? Rng,
[property: JsonPropertyName("sampling")] int? Sampling);
public sealed record ReplayArtifact(
[property: JsonPropertyName("name")] string Name,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("mediaType")] string? MediaType);
[JsonConverter(typeof(JsonStringEnumConverter))]
public enum ReplayTimeSource
{
monotonic,
wall
}

View File

@@ -0,0 +1,66 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
namespace StellaOps.Orchestrator.Core.Hashing;
/// <summary>
/// Produces deterministic, canonical JSON and hashes for orchestrator payloads (events, audit, manifests).
/// Keys are sorted lexicographically; arrays preserve order; nulls are retained; timestamps remain ISO 8601 with offsets.
/// </summary>
public static class CanonicalJsonHasher
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
WriteIndented = false,
PropertyNamingPolicy = null,
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }
};
/// <summary>
/// Serialize the value to canonical JSON (sorted object keys, stable formatting).
/// </summary>
public static string ToCanonicalJson<T>(T value)
{
var node = JsonSerializer.SerializeToNode(value, SerializerOptions) ?? new JsonObject();
var ordered = OrderNode(node);
return ordered.ToJsonString(SerializerOptions);
}
/// <summary>
/// Compute SHA-256 over canonical JSON (lowercase hex).
/// </summary>
public static string ComputeCanonicalSha256<T>(T value)
{
var canonicalJson = ToCanonicalJson(value);
var bytes = Encoding.UTF8.GetBytes(canonicalJson);
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static JsonNode OrderNode(JsonNode node)
{
switch (node)
{
case JsonObject obj:
var orderedObj = new JsonObject();
foreach (var kvp in obj.OrderBy(x => x.Key, StringComparer.Ordinal))
{
orderedObj.Add(kvp.Key, kvp.Value is null ? null : OrderNode(kvp.Value));
}
return orderedObj;
case JsonArray arr:
var orderedArr = new JsonArray();
foreach (var item in arr)
{
orderedArr.Add(item is null ? null : OrderNode(item));
}
return orderedArr;
default:
return node; // primitives stay as-is
}
}
}

View File

@@ -0,0 +1,51 @@
{
"$id": "https://stellaops.dev/orchestrator/schemas/audit-bundle.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Orchestrator Audit Bundle",
"type": "object",
"additionalProperties": false,
"required": ["manifestVersion", "bundleId", "createdAt", "entries"],
"properties": {
"manifestVersion": { "type": "string", "pattern": "^orch\.audit\.v[0-9]+$" },
"bundleId": { "type": "string", "minLength": 1 },
"createdAt": { "type": "string", "format": "date-time" },
"tenantId": { "type": "string" },
"hashAlgorithm": { "type": "string", "enum": ["sha256"] },
"entries": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["entryId", "contentHash", "sequenceNumber", "occurredAt"],
"properties": {
"entryId": { "type": "string" },
"contentHash": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
"previousEntryHash": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
"sequenceNumber": { "type": "integer", "minimum": 1 },
"occurredAt": { "type": "string", "format": "date-time" },
"resourceType": { "type": "string" },
"resourceId": { "type": "string" },
"eventType": { "type": "string" },
"actorId": { "type": "string" },
"actorType": { "type": "string" },
"oldState": { "type": "string" },
"newState": { "type": "string" },
"metadata": { "type": "string" }
}
}
},
"attachments": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["uri", "digest"],
"properties": {
"uri": { "type": "string" },
"digest": { "type": "string", "pattern": "^[a-z0-9]+:[A-Fa-f0-9]+$" },
"mediaType": { "type": "string" }
}
}
}
}
}

View File

@@ -0,0 +1,95 @@
{
"$id": "https://stellaops.dev/orchestrator/schemas/event-envelope.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Orchestrator Event Envelope",
"type": "object",
"additionalProperties": false,
"required": [
"schemaVersion",
"eventId",
"eventType",
"occurredAt",
"idempotencyKey",
"tenantId",
"actor",
"job"
],
"properties": {
"schemaVersion": { "type": "string", "pattern": "^orch\.event\.v[0-9]+$" },
"eventId": { "type": "string", "minLength": 1 },
"eventType": { "type": "string", "minLength": 1 },
"occurredAt": { "type": "string", "format": "date-time" },
"idempotencyKey": { "type": "string", "minLength": 1 },
"correlationId": { "type": "string" },
"tenantId": { "type": "string", "minLength": 1 },
"projectId": { "type": "string" },
"actor": {
"type": "object",
"additionalProperties": false,
"required": ["subject", "scopes"],
"properties": {
"subject": { "type": "string", "minLength": 1 },
"scopes": { "type": "array", "items": { "type": "string" } }
}
},
"job": {
"type": "object",
"additionalProperties": false,
"required": ["id", "type", "attempt", "status"],
"properties": {
"id": { "type": "string", "minLength": 1 },
"type": { "type": "string", "minLength": 1 },
"runId": { "type": "string" },
"attempt": { "type": "integer", "minimum": 0 },
"leaseId": { "type": "string" },
"taskRunnerId": { "type": "string" },
"status": { "type": "string" },
"reason": { "type": "string" },
"payloadDigest": { "type": "string" },
"artifacts": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["uri", "digest"],
"properties": {
"uri": { "type": "string" },
"digest": { "type": "string" },
"mime": { "type": "string" }
}
}
},
"provenance": {
"type": "object",
"additionalProperties": { "type": "string" }
}
}
},
"metrics": {
"type": "object",
"additionalProperties": false,
"properties": {
"durationSeconds": { "type": "number", "minimum": 0 },
"logStreamLagSeconds": { "type": "number", "minimum": 0 },
"backoffSeconds": { "type": "number", "minimum": 0 }
}
},
"notifier": {
"type": "object",
"additionalProperties": false,
"properties": {
"channel": { "type": "string" },
"delivery": { "type": "string" },
"replay": {
"type": "object",
"additionalProperties": false,
"required": ["ordinal", "total"],
"properties": {
"ordinal": { "type": "integer", "minimum": 0 },
"total": { "type": "integer", "minimum": 1 }
}
}
}
}
}
}

View File

@@ -0,0 +1,55 @@
{
"$id": "https://stellaops.dev/orchestrator/schemas/replay-manifest.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Replay Manifest",
"type": "object",
"additionalProperties": false,
"required": ["schemaVersion", "jobId", "replayOf", "inputs"],
"properties": {
"schemaVersion": { "type": "string", "pattern": "^orch\.replay\.v[0-9]+$" },
"jobId": { "type": "string" },
"replayOf": { "type": "string" },
"createdAt": { "type": "string", "format": "date-time" },
"reason": { "type": "string" },
"inputs": {
"type": "object",
"additionalProperties": false,
"required": ["policyHash", "graphRevisionId", "toolImages", "seeds"],
"properties": {
"policyHash": { "type": "string" },
"graphRevisionId": { "type": "string" },
"latticeHash": { "type": "string" },
"toolImages": {
"type": "array",
"items": { "type": "string" }
},
"seeds": {
"type": "object",
"additionalProperties": false,
"properties": {
"rng": { "type": "integer", "minimum": 0 },
"sampling": { "type": "integer", "minimum": 0 }
}
},
"timeSource": { "type": "string", "enum": ["monotonic", "wall"] },
"env": {
"type": "object",
"additionalProperties": { "type": "string" }
}
}
},
"artifacts": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "digest"],
"properties": {
"name": { "type": "string" },
"digest": { "type": "string" },
"mediaType": { "type": "string" }
}
}
}
}
}

View File

@@ -0,0 +1,51 @@
{
"$id": "https://stellaops.dev/orchestrator/schemas/taskrunner-integrity.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "TaskRunner Integrity Capsule",
"type": "object",
"additionalProperties": false,
"required": ["jobId", "runId", "attempt", "artifacts", "logs"],
"properties": {
"jobId": { "type": "string" },
"runId": { "type": "string" },
"attempt": { "type": "integer", "minimum": 0 },
"workerImage": { "type": "string" },
"configHash": { "type": "string" },
"artifacts": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "digest", "sizeBytes"],
"properties": {
"name": { "type": "string" },
"digest": { "type": "string", "pattern": "^[a-z0-9]+:[A-Fa-f0-9]+$" },
"sizeBytes": { "type": "integer", "minimum": 0 },
"mediaType": { "type": "string" }
}
}
},
"logs": {
"type": "object",
"additionalProperties": false,
"required": ["digest", "sizeBytes"],
"properties": {
"digest": { "type": "string", "pattern": "^[a-z0-9]+:[A-Fa-f0-9]+$" },
"sizeBytes": { "type": "integer", "minimum": 0 }
}
},
"timeline": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["type", "occurredAt"],
"properties": {
"type": { "type": "string" },
"occurredAt": { "type": "string", "format": "date-time" },
"traceId": { "type": "string" }
}
}
}
}
}

View File

@@ -0,0 +1,54 @@
using System.Text.Json;
using StellaOps.Orchestrator.Core.Domain;
using StellaOps.Orchestrator.Core.Hashing;
namespace StellaOps.Orchestrator.Tests;
public class CanonicalJsonHasherTests
{
[Fact]
public void ProducesStableHash_WhenObjectPropertyOrderDiffers()
{
var first = new { b = 1, a = 2 };
var second = new { a = 2, b = 1 };
var firstHash = CanonicalJsonHasher.ComputeCanonicalSha256(first);
var secondHash = CanonicalJsonHasher.ComputeCanonicalSha256(second);
Assert.Equal(firstHash, secondHash);
}
[Fact]
public void CanonicalJson_SortsKeysAndPreservesArrays()
{
var value = new
{
meta = new { z = 1, a = 2 },
items = new[] { "first", "second" }
};
var json = CanonicalJsonHasher.ToCanonicalJson(value);
// keys sorted inside meta
Assert.Equal("{\"items\":[\"first\",\"second\"],\"meta\":{\"a\":2,\"z\":1}}", json);
}
[Fact]
public void AuditEntry_UsesCanonicalHash()
{
var entry = AuditEntry.Create(
tenantId: "tenant-1",
eventType: AuditEventType.JobCreated,
resourceType: "job",
resourceId: Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"),
actorId: "user-1",
actorType: ActorType.User,
description: "created job");
Assert.True(entry.VerifyIntegrity());
// Changing description should invalidate hash
var tampered = entry with { Description = "tampered" };
Assert.False(tampered.VerifyIntegrity());
}
}

View File

@@ -203,7 +203,6 @@ public sealed class PackRunStreamCoordinatorTests
public override void Dispose()
{
_state = WebSocketState.Closed;
base.Dispose();
}
public override Task<WebSocketReceiveResult> ReceiveAsync(ArraySegment<byte> buffer, CancellationToken cancellationToken)

View File

@@ -0,0 +1,39 @@
using System.Collections.Immutable;
using StellaOps.Orchestrator.Core.Domain.Replay;
namespace StellaOps.Orchestrator.Tests;
public class ReplayManifestTests
{
[Fact]
public void ComputeHash_IsStableWithCanonicalOrdering()
{
var inputs = new ReplayInputs(
PolicyHash: "sha256:policy",
GraphRevisionId: "graph-42",
LatticeHash: "sha256:lattice",
ToolImages: ImmutableArray.Create("img:1", "img:2"),
Seeds: new ReplaySeeds(Rng: 1234, Sampling: 99),
TimeSource: ReplayTimeSource.monotonic,
Env: ImmutableDictionary.Create<string, string>().Add("TZ", "UTC"));
var manifestA = ReplayManifest.Create(
jobId: "job-123",
replayOf: "job-original",
inputs: inputs,
artifacts: new[] { new ReplayArtifact("ledger.ndjson", "sha256:abc", "application/x-ndjson") },
createdAt: new DateTimeOffset(2025, 12, 1, 0, 0, 0, TimeSpan.Zero));
var manifestB = ReplayManifest.Create(
jobId: "job-123",
replayOf: "job-original",
inputs: inputs,
artifacts: new[] { new ReplayArtifact("ledger.ndjson", "sha256:abc", "application/x-ndjson") },
createdAt: new DateTimeOffset(2025, 12, 1, 0, 0, 0, TimeSpan.Zero));
var hashA = manifestA.ComputeHash();
var hashB = manifestB.ComputeHash();
Assert.Equal(hashA, hashB);
}
}

View File

@@ -0,0 +1,56 @@
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Orchestrator.Core;
using StellaOps.Orchestrator.Core.Hashing;
namespace StellaOps.Orchestrator.Tests;
public class SchemaSmokeTests
{
[Theory]
[InlineData("event-envelope.schema.json")]
[InlineData("audit-bundle.schema.json")]
[InlineData("replay-manifest.schema.json")]
[InlineData("taskrunner-integrity.schema.json")]
public void Schemas_AreWellFormedJson(string schemaFile)
{
var path = Path.Combine(AppContext.BaseDirectory, "../../../StellaOps.Orchestrator.Core/Schemas", schemaFile);
Assert.True(File.Exists(path), $"Schema missing: {schemaFile}");
var text = File.ReadAllText(path);
using var doc = JsonDocument.Parse(text);
Assert.True(doc.RootElement.TryGetProperty("$id", out _));
Assert.True(doc.RootElement.TryGetProperty("title", out _));
}
[Fact]
public void CanonicalHash_For_EventEnvelope_IsStable()
{
var envelope = EventEnvelope.Create(
eventType: "job.completed",
tenantId: "tenant-a",
job: new EventJob(
Id: "job-123",
Type: "pack-run",
RunId: "run-1",
Attempt: 1,
LeaseId: "lease-9",
TaskRunnerId: "tr-1",
Status: "completed",
Reason: null,
PayloadDigest: "sha256:abc",
Artifacts: ImmutableArray.Create<EventArtifact>(),
Provenance: ImmutableDictionary<string, string>.Empty),
actor: new EventActor("worker-go", ImmutableArray.Create("orch:job")),
occurredAt: new DateTimeOffset(2025, 12, 1, 12, 0, 0, TimeSpan.Zero),
eventId: "evt-1",
idempotencyKey: "fixed");
var hash1 = CanonicalJsonHasher.ComputeCanonicalSha256(envelope);
var hash2 = CanonicalJsonHasher.ComputeCanonicalSha256(envelope);
Assert.Equal(hash1, hash2);
Assert.Equal(64, hash1.Length);
}
}