up

scripts/bench/README.md

@@ -0,0 +1,10 @@
+# Bench scripts
+
+- `determinism-run.sh`: runs BENCH-DETERMINISM-401-057 harness (`src/Bench/StellaOps.Bench/Determinism`), writes artifacts to `out/bench-determinism`, and enforces threshold via `BENCH_DETERMINISM_THRESHOLD` (default 0.95). Defaults to 10 runs per scanner/SBOM pair. Pass `DET_EXTRA_INPUTS` (space-separated globs) to include frozen feeds in `inputs.sha256`; `DET_RUN_EXTRA_ARGS` to forward extra args to the harness.
+
+Usage:
+```sh
+BENCH_DETERMINISM_THRESHOLD=0.97 \
+DET_EXTRA_INPUTS="offline/feeds/*.tar.gz" \
+scripts/bench/determinism-run.sh
+```

scripts/bench/determinism-run.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# BENCH-DETERMINISM-401-057: run determinism harness and collect artifacts
+
+ROOT="$(git rev-parse --show-toplevel)"
+HARNESS="${ROOT}/src/Bench/StellaOps.Bench/Determinism"
+OUT="${ROOT}/out/bench-determinism"
+THRESHOLD="${BENCH_DETERMINISM_THRESHOLD:-0.95}"
+mkdir -p "$OUT"
+
+cd "$HARNESS"
+
+python run_bench.py \
+  --sboms inputs/sboms/*.json \
+  --vex inputs/vex/*.json \
+  --config configs/scanners.json \
+  --runs 10 \
+  --shuffle \
+  --output results \
+  --manifest-extra "${DET_EXTRA_INPUTS:-}" \
+  ${DET_RUN_EXTRA_ARGS:-}
+
+cp -a results "$OUT"/
+det_rate=$(python -c "import json;print(json.load(open('results/summary.json'))['determinism_rate'])")
+printf "determinism_rate=%s\n" "$det_rate" > "$OUT/summary.txt"
+printf "timestamp=%s\n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> "$OUT/summary.txt"
+
+awk -v rate="$det_rate" -v th="$THRESHOLD" 'BEGIN {if (rate+0 < th+0) {printf("determinism_rate %s is below threshold %s\n", rate, th); exit 1}}'
+
+tar -C "$OUT" -czf "$OUT/bench-determinism-artifacts.tgz" .
+echo "[bench-determinism] artifacts at $OUT"