#!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later # BENCH-AUTO-401-019: Reachability replay script set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } log_error() { echo -e "${RED}[ERROR]${NC} $*"; } usage() { echo "Usage: $0 [--output DIR] [--verify]" echo "" echo "Replay reachability manifests from bench findings." echo "" echo "Options:" echo " --output DIR Output directory for replay results" echo " --verify Verify replay outputs against ground truth" echo " --help, -h Show this help" exit 1 } INPUT="" OUTPUT_DIR="${REPO_ROOT}/bench/results/replay" VERIFY=false while [[ $# -gt 0 ]]; do case $1 in --output) OUTPUT_DIR="$2" shift 2 ;; --verify) VERIFY=true shift ;; --help|-h) usage ;; *) if [[ -z "$INPUT" ]]; then INPUT="$1" else echo "Unknown option: $1" usage fi shift ;; esac done if [[ -z "$INPUT" ]]; then # Default to bench/findings INPUT="${REPO_ROOT}/bench/findings" fi if [[ ! -e "$INPUT" ]]; then log_error "Input not found: $INPUT" exit 1 fi mkdir -p "$OUTPUT_DIR" log_info "Replay input: $INPUT" log_info "Output directory: $OUTPUT_DIR" # Collect all reachability evidence files EVIDENCE_FILES=() if [[ -d "$INPUT" ]]; then # Directory of findings while IFS= read -r -d '' file; do EVIDENCE_FILES+=("$file") done < <(find "$INPUT" -name "reachability.json" -print0 2>/dev/null) elif [[ -f "$INPUT" ]]; then # Single manifest file EVIDENCE_FILES+=("$INPUT") fi if [[ ${#EVIDENCE_FILES[@]} -eq 0 ]]; then log_warn "No reachability evidence files found" exit 0 fi log_info "Found ${#EVIDENCE_FILES[@]} evidence file(s)" # Process each evidence file TOTAL=0 PASSED=0 FAILED=0 for evidence_file in "${EVIDENCE_FILES[@]}"; do TOTAL=$((TOTAL + 1)) finding_dir=$(dirname "$(dirname "$evidence_file")") finding_id=$(basename "$finding_dir") log_info "Processing: $finding_id" # Extract metadata metadata_file="${finding_dir}/metadata.json" if [[ ! -f "$metadata_file" ]]; then log_warn " No metadata.json found, skipping" continue fi # Parse evidence evidence_hash=$(python3 -c " import json with open('$evidence_file') as f: d = json.load(f) paths = d.get('paths', []) print(f'paths={len(paths)}') print(f'variant={d.get(\"variant\", \"unknown\")}') print(f'case_id={d.get(\"case_id\", \"unknown\")}') " 2>/dev/null || echo "error") if [[ "$evidence_hash" == "error" ]]; then log_warn " Failed to parse evidence" FAILED=$((FAILED + 1)) continue fi echo " $evidence_hash" # Create replay output replay_output="${OUTPUT_DIR}/${finding_id}" mkdir -p "$replay_output" # Copy evidence for replay cp "$evidence_file" "$replay_output/evidence.json" # If verify mode, check against ground truth if [[ "$VERIFY" == true ]]; then ground_truth=$(python3 -c " import json with open('$evidence_file') as f: d = json.load(f) gt = d.get('ground_truth') if gt: print(f'variant={gt.get(\"variant\", \"unknown\")}') print(f'paths={len(gt.get(\"paths\", []))}') else: print('no_ground_truth') " 2>/dev/null || echo "error") if [[ "$ground_truth" != "no_ground_truth" && "$ground_truth" != "error" ]]; then log_info " Ground truth: $ground_truth" PASSED=$((PASSED + 1)) else log_warn " No ground truth available" fi else PASSED=$((PASSED + 1)) fi # Record replay result echo "{\"finding_id\": \"$finding_id\", \"status\": \"replayed\", \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > "$replay_output/replay.json" done # Summary echo "" log_info "Replay Summary:" log_info " Total: $TOTAL" log_info " Passed: $PASSED" log_info " Failed: $FAILED" # Write summary file echo "{ \"total\": $TOTAL, \"passed\": $PASSED, \"failed\": $FAILED, \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" }" > "$OUTPUT_DIR/summary.json" log_info "Summary written to: $OUTPUT_DIR/summary.json"