59 lines
1.7 KiB
Bash
59 lines
1.7 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Offline runner for determinism (and optional reachability) benches.
|
|
# Usage: ./offline_run.sh [--inputs DIR] [--output DIR] [--runs N] [--threshold FLOAT] [--no-verify]
|
|
# Defaults: inputs=offline/inputs, output=offline/results, runs=10, threshold=0.95, verify manifests on.
|
|
|
|
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
|
INPUT_DIR="offline/inputs"
|
|
OUTPUT_DIR="offline/results"
|
|
RUNS=10
|
|
THRESHOLD=0.95
|
|
VERIFY=1
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--inputs) INPUT_DIR="$2"; shift 2;;
|
|
--output) OUTPUT_DIR="$2"; shift 2;;
|
|
--runs) RUNS="$2"; shift 2;;
|
|
--threshold) THRESHOLD="$2"; shift 2;;
|
|
--no-verify) VERIFY=0; shift 1;;
|
|
*) echo "Unknown arg: $1"; exit 1;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
cd "$ROOT"
|
|
|
|
if [ $VERIFY -eq 1 ]; then
|
|
if [ -f "$INPUT_DIR/inputs.sha256" ]; then
|
|
sha256sum -c "$INPUT_DIR/inputs.sha256"
|
|
fi
|
|
if [ -f "$INPUT_DIR/dataset.sha256" ]; then
|
|
sha256sum -c "$INPUT_DIR/dataset.sha256"
|
|
fi
|
|
fi
|
|
|
|
python run_bench.py \
|
|
--sboms "$INPUT_DIR"/sboms/*.json \
|
|
--vex "$INPUT_DIR"/vex/*.json \
|
|
--config "$INPUT_DIR"/scanners.json \
|
|
--runs "$RUNS" \
|
|
--shuffle \
|
|
--output "$OUTPUT_DIR"
|
|
|
|
det_rate=$(python -c "import json;print(json.load(open('$OUTPUT_DIR/summary.json'))['determinism_rate'])")
|
|
awk -v rate="$det_rate" -v th="$THRESHOLD" 'BEGIN {if (rate+0 < th+0) {printf("determinism_rate %s is below threshold %s\n", rate, th); exit 1}}'
|
|
|
|
graph_glob="$INPUT_DIR/graphs/*.json"
|
|
runtime_glob="$INPUT_DIR/runtime/*.ndjson"
|
|
if ls $graph_glob >/dev/null 2>&1; then
|
|
python run_reachability.py \
|
|
--graphs "$graph_glob" \
|
|
--runtime "$runtime_glob" \
|
|
--output "$OUTPUT_DIR"
|
|
fi
|
|
|
|
echo "Offline run complete -> $OUTPUT_DIR"
|