feat: Add UI benchmark driver and scenarios for graph interactions
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled

- Introduced `ui_bench_driver.mjs` to read scenarios and fixture manifest, generating a deterministic run plan.
- Created `ui_bench_plan.md` outlining the purpose, scope, and next steps for the benchmark.
- Added `ui_bench_scenarios.json` containing various scenarios for graph UI interactions.
- Implemented tests for CLI commands, ensuring bundle verification and telemetry defaults.
- Developed schemas for orchestrator components, including replay manifests and event envelopes.
- Added mock API for risk management, including listing and statistics functionalities.
- Implemented models for risk profiles and query options to support the new API.
This commit is contained in:
StellaOps Bot
2025-12-02 01:28:17 +02:00
parent 909d9b6220
commit 44171930ff
94 changed files with 3606 additions and 271 deletions

View File

@@ -257,6 +257,7 @@ internal sealed class SurfaceManifestPublisher : ISurfaceManifestPublisher
ArtifactDocumentFormat.ComponentFragmentJson => "layer.fragments",
ArtifactDocumentFormat.ObservationJson => "observation.json",
ArtifactDocumentFormat.SurfaceManifestJson => "surface.manifest",
ArtifactDocumentFormat.CompositionRecipeJson => "composition.recipe",
ArtifactDocumentFormat.CycloneDxJson => "cdx-json",
ArtifactDocumentFormat.CycloneDxProtobuf => "cdx-protobuf",
ArtifactDocumentFormat.SpdxJson => "spdx-json",

View File

@@ -265,8 +265,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
pins["policy"] = policy;
}
var (artifactHashes, merkle) = ComputeDeterminismHashes(payloads);
merkleRoot = merkle;
var (artifactHashes, recipeBytes, recipeSha256) = BuildCompositionRecipe(payloads);
merkleRoot = recipeSha256;
var report = new
{
@@ -277,12 +277,26 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
concurrencyLimit = _determinism.ConcurrencyLimit,
pins = pins,
artifacts = artifactHashes,
merkleRoot = merkle
merkleRoot = recipeSha256
};
var evidence = new Determinism.DeterminismEvidence(artifactHashes, merkle);
var evidence = new Determinism.DeterminismEvidence(artifactHashes, recipeSha256);
context.Analysis.Set(ScanAnalysisKeys.DeterminismEvidence, evidence);
// Publish composition recipe as a manifest artifact for offline replay.
payloads = payloads.ToList();
((List<SurfaceManifestPayload>)payloads).Add(new SurfaceManifestPayload(
ArtifactDocumentType.CompositionRecipe,
ArtifactDocumentFormat.CompositionRecipeJson,
Kind: "composition.recipe",
MediaType: "application/vnd.stellaops.composition.recipe+json",
Content: recipeBytes,
Metadata: new Dictionary<string, string>
{
["schema"] = "stellaops.composition.recipe@1",
["merkleRoot"] = recipeSha256,
}));
var json = JsonSerializer.Serialize(report, JsonOptions);
return new SurfaceManifestPayload(
ArtifactDocumentType.SurfaceObservation,
@@ -293,9 +307,9 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
View: "replay");
}
private static (Dictionary<string, string> Hashes, string MerkleRoot) ComputeDeterminismHashes(IEnumerable<SurfaceManifestPayload> payloads)
private static (Dictionary<string, string> Hashes, byte[] RecipeBytes, string RecipeSha256) BuildCompositionRecipe(IEnumerable<SurfaceManifestPayload> payloads)
{
var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var map = new SortedDictionary<string, string>(StringComparer.Ordinal);
using var sha = SHA256.Create();
foreach (var payload in payloads.OrderBy(p => p.Kind, StringComparer.Ordinal))
@@ -304,18 +318,18 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
map[payload.Kind] = digest;
}
// Build Merkle-like root by hashing the ordered list of kind:digest lines.
var builder = new StringBuilder();
foreach (var kvp in map.OrderBy(kv => kv.Key, StringComparer.Ordinal))
var recipe = new
{
builder.Append(kvp.Key).Append(':').Append(kvp.Value).Append('\n');
}
schema = "stellaops.composition.recipe@1",
artifacts = map, // already sorted
};
var rootBytes = Encoding.UTF8.GetBytes(builder.ToString());
var rootHash = sha.ComputeHash(rootBytes);
var recipeJson = JsonSerializer.Serialize(recipe, JsonOptions);
var recipeBytes = Encoding.UTF8.GetBytes(recipeJson);
var rootHash = sha.ComputeHash(recipeBytes);
var merkleRoot = Convert.ToHexString(rootHash).ToLowerInvariant();
return (map, merkleRoot);
return (new Dictionary<string, string>(map, StringComparer.OrdinalIgnoreCase), recipeBytes, merkleRoot);
}
private static string? GetReplayBundleUri(ScanJobContext context)

View File

@@ -24,11 +24,11 @@ public sealed class CycloneDxComposer
private const string InventoryMediaTypeProtobuf = "application/vnd.cyclonedx+protobuf; version=1.6";
private const string UsageMediaTypeProtobuf = "application/vnd.cyclonedx+protobuf; version=1.6; view=usage";
public SbomCompositionResult Compose(SbomCompositionRequest request)
{
ArgumentNullException.ThrowIfNull(request);
if (request.LayerFragments.IsDefaultOrEmpty)
{
public SbomCompositionResult Compose(SbomCompositionRequest request)
{
ArgumentNullException.ThrowIfNull(request);
if (request.LayerFragments.IsDefaultOrEmpty)
{
throw new ArgumentException("At least one layer fragment is required.", nameof(request));
}
@@ -48,9 +48,9 @@ public sealed class CycloneDxComposer
.Where(static component => component.Usage.UsedByEntrypoint)
.ToImmutableArray();
CycloneDxArtifact? usageArtifact = null;
if (!usageComponents.IsEmpty)
{
CycloneDxArtifact? usageArtifact = null;
if (!usageComponents.IsEmpty)
{
usageArtifact = BuildArtifact(
request,
graph,
@@ -59,15 +59,36 @@ public sealed class CycloneDxComposer
generatedAt,
UsageMediaTypeJson,
UsageMediaTypeProtobuf);
}
return new SbomCompositionResult
{
Inventory = inventoryArtifact,
Usage = usageArtifact,
Graph = graph,
};
}
}
var compositionRecipeJson = BuildCompositionRecipeJson(graph, generatedAt);
var compositionRecipeSha = ComputeSha256(compositionRecipeJson);
var compositionRecipeUri = $"cas://sbom/composition/{compositionRecipeSha}.json";
inventoryArtifact = inventoryArtifact with
{
MerkleRoot = compositionRecipeSha,
CompositionRecipeUri = compositionRecipeUri,
};
if (usageArtifact is not null)
{
usageArtifact = usageArtifact with
{
MerkleRoot = compositionRecipeSha,
CompositionRecipeUri = compositionRecipeUri,
};
}
return new SbomCompositionResult
{
Inventory = inventoryArtifact,
Usage = usageArtifact,
Graph = graph,
CompositionRecipeJson = compositionRecipeJson,
CompositionRecipeSha256 = compositionRecipeSha,
};
}
private CycloneDxArtifact BuildArtifact(
SbomCompositionRequest request,
@@ -92,6 +113,7 @@ public sealed class CycloneDxComposer
: null;
request.AdditionalProperties?.TryGetValue("stellaops:composition.manifest", out var compositionUri);
request.AdditionalProperties?.TryGetValue("stellaops:composition.recipe", out var compositionRecipeUri);
return new CycloneDxArtifact
{
@@ -104,12 +126,38 @@ public sealed class CycloneDxComposer
ContentHash = jsonHash,
MerkleRoot = merkleRoot,
CompositionUri = compositionUri,
CompositionRecipeUri = compositionRecipeUri,
JsonMediaType = jsonMediaType,
ProtobufBytes = protobufBytes,
ProtobufSha256 = protobufHash,
ProtobufMediaType = protobufMediaType,
};
}
private static byte[] BuildCompositionRecipeJson(ComponentGraph graph, DateTimeOffset generatedAt)
{
var recipe = new
{
schema = "stellaops.composition.recipe@1",
generatedAt = ScannerTimestamps.ToIso8601(generatedAt),
layers = graph.Layers.Select(layer => new
{
layer.LayerDigest,
components = layer.Components
.Select(component => component.Identity.Key)
.OrderBy(key => key, StringComparer.Ordinal)
.ToArray(),
}).OrderBy(entry => entry.LayerDigest, StringComparer.Ordinal).ToArray(),
};
var json = JsonSerializer.Serialize(recipe, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
});
return Encoding.UTF8.GetBytes(json);
}
private Bom BuildBom(
SbomCompositionRequest request,

View File

@@ -33,6 +33,11 @@ public sealed record CycloneDxArtifact
/// </summary>
public string? CompositionUri { get; init; }
/// <summary>
/// CAS URI of the layer composition recipe (_composition.json) if emitted.
/// </summary>
public string? CompositionRecipeUri { get; init; }
public required string JsonMediaType { get; init; }
public required byte[] ProtobufBytes { get; init; }
@@ -42,11 +47,21 @@ public sealed record CycloneDxArtifact
public required string ProtobufMediaType { get; init; }
}
public sealed record SbomCompositionResult
{
public required CycloneDxArtifact Inventory { get; init; }
public CycloneDxArtifact? Usage { get; init; }
public required ComponentGraph Graph { get; init; }
}
public sealed record SbomCompositionResult
{
public required CycloneDxArtifact Inventory { get; init; }
public CycloneDxArtifact? Usage { get; init; }
public required ComponentGraph Graph { get; init; }
/// <summary>
/// Composition recipe JSON bytes (canonical) capturing fragment ordering and hashes.
/// </summary>
public required byte[] CompositionRecipeJson { get; init; }
/// <summary>
/// SHA256 hex of the composition recipe JSON.
/// </summary>
public required string CompositionRecipeSha256 { get; init; }
}

View File

@@ -88,18 +88,26 @@ public sealed class ScannerArtifactPackageBuilder
descriptors.Add(CreateDescriptor(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxProtobuf, composition.Usage.ProtobufMediaType, composition.Usage.ProtobufBytes, composition.Usage.ProtobufSha256, SbomView.Usage));
}
descriptors.Add(CreateDescriptor(ArtifactDocumentType.Index, ArtifactDocumentFormat.BomIndex, "application/vnd.stellaops.bom-index.v1+binary", bomIndex.Bytes, bomIndex.Sha256, null));
descriptors.Add(CreateDescriptor(ArtifactDocumentType.Index, ArtifactDocumentFormat.BomIndex, "application/vnd.stellaops.bom-index.v1+binary", bomIndex.Bytes, bomIndex.Sha256, null));
descriptors.Add(CreateDescriptor(
ArtifactDocumentType.CompositionRecipe,
ArtifactDocumentFormat.CompositionRecipeJson,
"application/vnd.stellaops.composition.recipe+json",
composition.CompositionRecipeJson,
composition.CompositionRecipeSha256,
null));
var manifest = new ScannerArtifactManifest
{
ImageDigest = imageDigest.Trim(),
GeneratedAt = generatedAt,
Artifacts = descriptors
.Select(ToManifestEntry)
.OrderBy(entry => entry.Kind, StringComparer.Ordinal)
.ThenBy(entry => entry.Format)
.ToImmutableArray(),
};
var manifest = new ScannerArtifactManifest
{
ImageDigest = imageDigest.Trim(),
GeneratedAt = generatedAt,
Artifacts = descriptors
.Select(ToManifestEntry)
.OrderBy(entry => entry.Kind, StringComparer.Ordinal)
.ThenBy(entry => entry.Format)
.ToImmutableArray(),
};
return new ScannerArtifactPackage
{
@@ -136,9 +144,10 @@ public sealed class ScannerArtifactPackageBuilder
ArtifactDocumentType.ImageBom => "sbom-inventory",
ArtifactDocumentType.LayerBom => "layer-sbom",
ArtifactDocumentType.Diff => "diff",
ArtifactDocumentType.Attestation => "attestation",
_ => descriptor.Type.ToString().ToLowerInvariant(),
};
ArtifactDocumentType.Attestation => "attestation",
ArtifactDocumentType.CompositionRecipe => "composition-recipe",
_ => descriptor.Type.ToString().ToLowerInvariant(),
};
return new ScannerArtifactManifestEntry
{

View File

@@ -12,7 +12,8 @@ public enum ArtifactDocumentType
SurfaceManifest,
SurfaceEntryTrace,
SurfaceLayerFragment,
SurfaceObservation
SurfaceObservation,
CompositionRecipe
}
public enum ArtifactDocumentFormat
@@ -26,7 +27,8 @@ public enum ArtifactDocumentFormat
EntryTraceNdjson,
EntryTraceGraphJson,
ComponentFragmentJson,
ObservationJson
ObservationJson,
CompositionRecipeJson
}
[BsonIgnoreExtraElements]

View File

@@ -8,6 +8,8 @@
<IsPackable>false</IsPackable>
<!-- Stay scoped: disable implicit restore sources beyond local nugets -->
<RestoreSources>$(StellaOpsLocalNuGetSource)</RestoreSources>
<DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
<RestoreNoCache>true</RestoreNoCache>
</PropertyGroup>
<ItemGroup>

View File

@@ -79,8 +79,9 @@ public sealed class CycloneDxComposerTests
Assert.Equal(first.Inventory.ContentHash, first.Inventory.JsonSha256);
Assert.Equal(first.Inventory.ProtobufSha256, second.Inventory.ProtobufSha256);
Assert.Equal(first.Inventory.SerialNumber, second.Inventory.SerialNumber);
Assert.Null(first.Inventory.MerkleRoot);
Assert.False(string.IsNullOrWhiteSpace(first.Inventory.MerkleRoot));
Assert.Null(first.Inventory.CompositionUri);
Assert.Null(first.Inventory.CompositionRecipeUri);
Assert.NotNull(first.Usage);
Assert.NotNull(second.Usage);
@@ -88,8 +89,15 @@ public sealed class CycloneDxComposerTests
Assert.Equal(first.Usage.ContentHash, first.Usage.JsonSha256);
Assert.Equal(first.Usage.ProtobufSha256, second.Usage.ProtobufSha256);
Assert.Equal(first.Usage.SerialNumber, second.Usage.SerialNumber);
Assert.Null(first.Usage.MerkleRoot);
Assert.False(string.IsNullOrWhiteSpace(first.Usage.MerkleRoot));
Assert.Null(first.Usage.CompositionUri);
Assert.Null(first.Usage.CompositionRecipeUri);
Assert.Equal(first.Inventory.MerkleRoot, first.Usage.MerkleRoot);
Assert.Equal(first.Inventory.MerkleRoot, result.CompositionRecipeSha256);
Assert.Equal(first.Inventory.ContentHash.Length, first.Inventory.MerkleRoot!.Length);
Assert.Equal(result.CompositionRecipeSha256.Length, 64);
Assert.NotEmpty(result.CompositionRecipeJson);
}
private static SbomCompositionRequest BuildRequest()

View File

@@ -64,16 +64,16 @@ public sealed class ScannerArtifactPackageBuilderTests
var packageBuilder = new ScannerArtifactPackageBuilder();
var package = packageBuilder.Build(request.Image.ImageDigest, request.GeneratedAt, composition, bomIndex);
Assert.Equal(5, package.Artifacts.Length); // inventory JSON+PB, usage JSON+PB, index
var kinds = package.Manifest.Artifacts.Select(entry => entry.Kind).ToArray();
Assert.Equal(new[] { "bom-index", "sbom-inventory", "sbom-inventory", "sbom-usage", "sbom-usage" }, kinds);
var manifestJson = package.Manifest.ToJsonBytes();
using var document = JsonDocument.Parse(manifestJson);
var root = document.RootElement;
Assert.Equal("sha256:image", root.GetProperty("imageDigest").GetString());
Assert.Equal(5, root.GetProperty("artifacts").GetArrayLength());
Assert.Equal(6, package.Artifacts.Length); // inventory JSON+PB, usage JSON+PB, index, composition recipe
var kinds = package.Manifest.Artifacts.Select(entry => entry.Kind).ToArray();
Assert.Equal(new[] { "bom-index", "composition-recipe", "sbom-inventory", "sbom-inventory", "sbom-usage", "sbom-usage" }, kinds);
var manifestJson = package.Manifest.ToJsonBytes();
using var document = JsonDocument.Parse(manifestJson);
var root = document.RootElement;
Assert.Equal("sha256:image", root.GetProperty("imageDigest").GetString());
Assert.Equal(6, root.GetProperty("artifacts").GetArrayLength());
var usageEntry = root.GetProperty("artifacts").EnumerateArray().First(element => element.GetProperty("kind").GetString() == "sbom-usage");
Assert.Equal("application/vnd.cyclonedx+json; version=1.6; view=usage", usageEntry.GetProperty("mediaType").GetString());

View File

@@ -102,10 +102,12 @@ public sealed class SurfaceManifestStageExecutorTests
Assert.Equal(publisher.LastManifestDigest, result!.ManifestDigest);
Assert.Equal(result.DeterminismMerkleRoot, publisher.LastRequest!.DeterminismMerkleRoot);
Assert.Equal(4, cache.Entries.Count);
Assert.Equal(6, cache.Entries.Count);
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.graph" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.ndjson" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.layer.fragments" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.determinism.json" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.composition.recipe" && key.Tenant == "tenant-a");
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.manifests" && key.Tenant == "tenant-a");
var publishedMetrics = listener.Measurements
@@ -114,7 +116,7 @@ public sealed class SurfaceManifestStageExecutorTests
Assert.Single(publishedMetrics);
Assert.Equal(1, publishedMetrics[0].Value);
Assert.Equal("published", publishedMetrics[0]["surface.result"]);
Assert.Equal(3, Convert.ToInt32(publishedMetrics[0]["surface.payload_count"]));
Assert.Equal(5, Convert.ToInt32(publishedMetrics[0]["surface.payload_count"]));
var payloadMetrics = listener.Measurements
.Where(m => m.InstrumentName == "scanner_worker_surface_payload_persisted_total")
@@ -608,7 +610,8 @@ public sealed class SurfaceManifestStageExecutorTests
WorkerInstance = request.WorkerInstance,
Attempt = request.Attempt
},
Artifacts = artifacts
Artifacts = artifacts,
DeterminismMerkleRoot = request.DeterminismMerkleRoot
};
var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(document, _options);