up

src/Bench/StellaOps.Bench/AGENTS.md

@@ -8,6 +8,7 @@ Design and maintain deterministic benchmark suites that measure StellaOps perfor
 - ImpactIndex/Scheduler/Scanner/Policy Engine workload simulations referenced in tasks.
 - Benchmark configuration and warm-up scripts used by DevOps for regression tracking.
 - Documentation of benchmark methodology and expected baseline metrics.
+- Determinism bench harness lives at `Determinism/` with optional reachability hashing; CI wrapper at `scripts/bench/determinism-run.sh` (threshold via `BENCH_DETERMINISM_THRESHOLD`). Include feeds via `DET_EXTRA_INPUTS`; optional reachability hashes via `DET_REACH_GRAPHS`/`DET_REACH_RUNTIME`.
 
 ## Required Reading
 - `docs/modules/platform/architecture-overview.md`

src/Bench/StellaOps.Bench/Determinism/README.md

@@ -22,6 +22,7 @@ Outputs land in `out/`:
 - SBOMs: `inputs/sboms/*.json` (sample SPDX provided)
 - VEX: `inputs/vex/*.json` (sample OpenVEX provided)
 - Scanner config: `configs/scanners.json` (defaults to built-in mock scanner)
+- Sample manifest: `inputs/inputs.sha256` covers the bundled sample SBOM/VEX/config for quick offline verification; regenerate when inputs change.
 
 ## Adding real scanners
 1. Add an entry to `configs/scanners.json` with `kind: "command"` and a command array, e.g.:

src/Bench/StellaOps.Bench/Determinism/inputs/feeds/README.md

@@ -0,0 +1,15 @@
+# Frozen feed bundle placeholder
+
+Place hashed feed bundles here for determinism runs. Example:
+
+```
+# build feed bundle (offline)
+# touch feed-bundle.tar.gz
+sha256sum feed-bundle.tar.gz > feeds.sha256
+```
+
+Then run the wrapper with:
+```
+DET_EXTRA_INPUTS="src/Bench/StellaOps.Bench/Determinism/inputs/feeds/feed-bundle.tar.gz" \
+BENCH_DETERMINISM_THRESHOLD=0.95 scripts/bench/determinism-run.sh
+```

src/Bench/StellaOps.Bench/Determinism/inputs/inputs.sha256

@@ -0,0 +1,3 @@
+577f932bbb00dbd596e46b96d5fbb9561506c7730c097e381a6b34de40402329  inputs/sboms/sample-spdx.json
+1b54ce4087800cfe1d5ac439c10a1f131b7476b2093b79d8cd0a29169314291f  inputs/vex/sample-openvex.json
+38453c9c0e0a90d22d7048d3201bf1b5665eb483e6682db1a7112f8e4f4fa1e6  configs/scanners.json

src/Bench/StellaOps.Bench/Determinism/offline_run.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Offline runner for determinism (and optional reachability) benches.
+# Usage: ./offline_run.sh [--inputs DIR] [--output DIR] [--runs N] [--threshold FLOAT] [--no-verify]
+# Defaults: inputs=offline/inputs, output=offline/results, runs=10, threshold=0.95, verify manifests on.
+
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+INPUT_DIR="offline/inputs"
+OUTPUT_DIR="offline/results"
+RUNS=10
+THRESHOLD=0.95
+VERIFY=1
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --inputs) INPUT_DIR="$2"; shift 2;;
+    --output) OUTPUT_DIR="$2"; shift 2;;
+    --runs) RUNS="$2"; shift 2;;
+    --threshold) THRESHOLD="$2"; shift 2;;
+    --no-verify) VERIFY=0; shift 1;;
+    *) echo "Unknown arg: $1"; exit 1;;
+  esac
+done
+
+mkdir -p "$OUTPUT_DIR"
+cd "$ROOT"
+
+if [ $VERIFY -eq 1 ]; then
+  if [ -f "$INPUT_DIR/inputs.sha256" ]; then
+    sha256sum -c "$INPUT_DIR/inputs.sha256"
+  fi
+  if [ -f "$INPUT_DIR/dataset.sha256" ]; then
+    sha256sum -c "$INPUT_DIR/dataset.sha256"
+  fi
+fi
+
+python run_bench.py \
+  --sboms "$INPUT_DIR"/sboms/*.json \
+  --vex "$INPUT_DIR"/vex/*.json \
+  --config "$INPUT_DIR"/scanners.json \
+  --runs "$RUNS" \
+  --shuffle \
+  --output "$OUTPUT_DIR"
+
+det_rate=$(python -c "import json;print(json.load(open('$OUTPUT_DIR/summary.json'))['determinism_rate'])")
+awk -v rate="$det_rate" -v th="$THRESHOLD" 'BEGIN {if (rate+0 < th+0) {printf("determinism_rate %s is below threshold %s\n", rate, th); exit 1}}'
+
+graph_glob="$INPUT_DIR/graphs/*.json"
+runtime_glob="$INPUT_DIR/runtime/*.ndjson"
+if ls $graph_glob >/dev/null 2>&1; then
+  python run_reachability.py \
+    --graphs "$graph_glob" \
+    --runtime "$runtime_glob" \
+    --output "$OUTPUT_DIR"
+fi
+
+echo "Offline run complete -> $OUTPUT_DIR"