up

bench/tools/replay.sh

@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# BENCH-AUTO-401-019: Reachability replay script
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
+
+usage() {
+    echo "Usage: $0 <manifest-or-findings-dir> [--output DIR] [--verify]"
+    echo ""
+    echo "Replay reachability manifests from bench findings."
+    echo ""
+    echo "Options:"
+    echo "  --output DIR   Output directory for replay results"
+    echo "  --verify       Verify replay outputs against ground truth"
+    echo "  --help, -h     Show this help"
+    exit 1
+}
+
+INPUT=""
+OUTPUT_DIR="${REPO_ROOT}/bench/results/replay"
+VERIFY=false
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --output)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --verify)
+            VERIFY=true
+            shift
+            ;;
+        --help|-h)
+            usage
+            ;;
+        *)
+            if [[ -z "$INPUT" ]]; then
+                INPUT="$1"
+            else
+                echo "Unknown option: $1"
+                usage
+            fi
+            shift
+            ;;
+    esac
+done
+
+if [[ -z "$INPUT" ]]; then
+    # Default to bench/findings
+    INPUT="${REPO_ROOT}/bench/findings"
+fi
+
+if [[ ! -e "$INPUT" ]]; then
+    log_error "Input not found: $INPUT"
+    exit 1
+fi
+
+mkdir -p "$OUTPUT_DIR"
+
+log_info "Replay input: $INPUT"
+log_info "Output directory: $OUTPUT_DIR"
+
+# Collect all reachability evidence files
+EVIDENCE_FILES=()
+
+if [[ -d "$INPUT" ]]; then
+    # Directory of findings
+    while IFS= read -r -d '' file; do
+        EVIDENCE_FILES+=("$file")
+    done < <(find "$INPUT" -name "reachability.json" -print0 2>/dev/null)
+elif [[ -f "$INPUT" ]]; then
+    # Single manifest file
+    EVIDENCE_FILES+=("$INPUT")
+fi
+
+if [[ ${#EVIDENCE_FILES[@]} -eq 0 ]]; then
+    log_warn "No reachability evidence files found"
+    exit 0
+fi
+
+log_info "Found ${#EVIDENCE_FILES[@]} evidence file(s)"
+
+# Process each evidence file
+TOTAL=0
+PASSED=0
+FAILED=0
+
+for evidence_file in "${EVIDENCE_FILES[@]}"; do
+    TOTAL=$((TOTAL + 1))
+    finding_dir=$(dirname "$(dirname "$evidence_file")")
+    finding_id=$(basename "$finding_dir")
+
+    log_info "Processing: $finding_id"
+
+    # Extract metadata
+    metadata_file="${finding_dir}/metadata.json"
+    if [[ ! -f "$metadata_file" ]]; then
+        log_warn "  No metadata.json found, skipping"
+        continue
+    fi
+
+    # Parse evidence
+    evidence_hash=$(python3 -c "
+import json
+with open('$evidence_file') as f:
+    d = json.load(f)
+paths = d.get('paths', [])
+print(f'paths={len(paths)}')
+print(f'variant={d.get(\"variant\", \"unknown\")}')
+print(f'case_id={d.get(\"case_id\", \"unknown\")}')
+" 2>/dev/null || echo "error")
+
+    if [[ "$evidence_hash" == "error" ]]; then
+        log_warn "  Failed to parse evidence"
+        FAILED=$((FAILED + 1))
+        continue
+    fi
+
+    echo "  $evidence_hash"
+
+    # Create replay output
+    replay_output="${OUTPUT_DIR}/${finding_id}"
+    mkdir -p "$replay_output"
+
+    # Copy evidence for replay
+    cp "$evidence_file" "$replay_output/evidence.json"
+
+    # If verify mode, check against ground truth
+    if [[ "$VERIFY" == true ]]; then
+        ground_truth=$(python3 -c "
+import json
+with open('$evidence_file') as f:
+    d = json.load(f)
+gt = d.get('ground_truth')
+if gt:
+    print(f'variant={gt.get(\"variant\", \"unknown\")}')
+    print(f'paths={len(gt.get(\"paths\", []))}')
+else:
+    print('no_ground_truth')
+" 2>/dev/null || echo "error")
+
+        if [[ "$ground_truth" != "no_ground_truth" && "$ground_truth" != "error" ]]; then
+            log_info "  Ground truth: $ground_truth"
+            PASSED=$((PASSED + 1))
+        else
+            log_warn "  No ground truth available"
+        fi
+    else
+        PASSED=$((PASSED + 1))
+    fi
+
+    # Record replay result
+    echo "{\"finding_id\": \"$finding_id\", \"status\": \"replayed\", \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > "$replay_output/replay.json"
+done
+
+# Summary
+echo ""
+log_info "Replay Summary:"
+log_info "  Total: $TOTAL"
+log_info "  Passed: $PASSED"
+log_info "  Failed: $FAILED"
+
+# Write summary file
+echo "{
+  \"total\": $TOTAL,
+  \"passed\": $PASSED,
+  \"failed\": $FAILED,
+  \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"
+}" > "$OUTPUT_DIR/summary.json"
+
+log_info "Summary written to: $OUTPUT_DIR/summary.json"