up

src/Bench/StellaOps.Bench/Determinism/offline_run.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Offline runner for determinism (and optional reachability) benches.
+# Usage: ./offline_run.sh [--inputs DIR] [--output DIR] [--runs N] [--threshold FLOAT] [--no-verify]
+# Defaults: inputs=offline/inputs, output=offline/results, runs=10, threshold=0.95, verify manifests on.
+
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+INPUT_DIR="offline/inputs"
+OUTPUT_DIR="offline/results"
+RUNS=10
+THRESHOLD=0.95
+VERIFY=1
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --inputs) INPUT_DIR="$2"; shift 2;;
+    --output) OUTPUT_DIR="$2"; shift 2;;
+    --runs) RUNS="$2"; shift 2;;
+    --threshold) THRESHOLD="$2"; shift 2;;
+    --no-verify) VERIFY=0; shift 1;;
+    *) echo "Unknown arg: $1"; exit 1;;
+  esac
+done
+
+mkdir -p "$OUTPUT_DIR"
+cd "$ROOT"
+
+if [ $VERIFY -eq 1 ]; then
+  if [ -f "$INPUT_DIR/inputs.sha256" ]; then
+    sha256sum -c "$INPUT_DIR/inputs.sha256"
+  fi
+  if [ -f "$INPUT_DIR/dataset.sha256" ]; then
+    sha256sum -c "$INPUT_DIR/dataset.sha256"
+  fi
+fi
+
+python run_bench.py \
+  --sboms "$INPUT_DIR"/sboms/*.json \
+  --vex "$INPUT_DIR"/vex/*.json \
+  --config "$INPUT_DIR"/scanners.json \
+  --runs "$RUNS" \
+  --shuffle \
+  --output "$OUTPUT_DIR"
+
+det_rate=$(python -c "import json;print(json.load(open('$OUTPUT_DIR/summary.json'))['determinism_rate'])")
+awk -v rate="$det_rate" -v th="$THRESHOLD" 'BEGIN {if (rate+0 < th+0) {printf("determinism_rate %s is below threshold %s\n", rate, th); exit 1}}'
+
+graph_glob="$INPUT_DIR/graphs/*.json"
+runtime_glob="$INPUT_DIR/runtime/*.ndjson"
+if ls $graph_glob >/dev/null 2>&1; then
+  python run_reachability.py \
+    --graphs "$graph_glob" \
+    --runtime "$runtime_glob" \
+    --output "$OUTPUT_DIR"
+fi
+
+echo "Offline run complete -> $OUTPUT_DIR"