c34fb7256d
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
SDK Publish & Sign / sdk-publish (push) Has been cancelled
sdk-generator-smoke / sdk-smoke (push) Has been cancelled
56 lines
1.8 KiB
Bash
56 lines
1.8 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# BENCH-DETERMINISM-401-057: run determinism harness and collect artifacts
|
|
|
|
ROOT="$(git rev-parse --show-toplevel)"
|
|
HARNESS="${ROOT}/src/Bench/StellaOps.Bench/Determinism"
|
|
OUT="${ROOT}/out/bench-determinism"
|
|
THRESHOLD="${BENCH_DETERMINISM_THRESHOLD:-0.95}"
|
|
mkdir -p "$OUT"
|
|
|
|
cd "$HARNESS"
|
|
|
|
python run_bench.py \
|
|
--sboms inputs/sboms/*.json \
|
|
--vex inputs/vex/*.json \
|
|
--config configs/scanners.json \
|
|
--runs 10 \
|
|
--shuffle \
|
|
--output results \
|
|
--manifest-extra "${DET_EXTRA_INPUTS:-}" \
|
|
${DET_RUN_EXTRA_ARGS:-}
|
|
|
|
cp -a results "$OUT"/
|
|
det_rate=$(python -c "import json;print(json.load(open('results/summary.json'))['determinism_rate'])")
|
|
printf "determinism_rate=%s\n" "$det_rate" > "$OUT/summary.txt"
|
|
printf "timestamp=%s\n" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> "$OUT/summary.txt"
|
|
|
|
awk -v rate="$det_rate" -v th="$THRESHOLD" 'BEGIN {if (rate+0 < th+0) {printf("determinism_rate %s is below threshold %s\n", rate, th); exit 1}}'
|
|
|
|
if [ -n "${DET_REACH_GRAPHS:-}" ]; then
|
|
echo "[bench-determinism] running reachability dataset hash"
|
|
reach_graphs=${DET_REACH_GRAPHS}
|
|
reach_runtime=${DET_REACH_RUNTIME:-}
|
|
# prefix relative globs with repo root for consistency
|
|
case "$reach_graphs" in
|
|
/*) ;;
|
|
*) reach_graphs="${ROOT}/${reach_graphs}" ;;
|
|
esac
|
|
case "$reach_runtime" in
|
|
/*|"") ;;
|
|
*) reach_runtime="${ROOT}/${reach_runtime}" ;;
|
|
esac
|
|
python run_reachability.py \
|
|
--graphs ${reach_graphs} \
|
|
--runtime ${reach_runtime} \
|
|
--output results
|
|
# copy reachability outputs
|
|
cp results/results-reach.csv "$OUT"/ || true
|
|
cp results/results-reach.json "$OUT"/ || true
|
|
cp results/dataset.sha256 "$OUT"/ || true
|
|
fi
|
|
|
|
tar -C "$OUT" -czf "$OUT/bench-determinism-artifacts.tgz" .
|
|
echo "[bench-determinism] artifacts at $OUT"
|