git.stella-ops.org/bench/tools/replay.sh

#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
# BENCH-AUTO-401-019: Reachability replay script

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }

usage() {
    echo "Usage: $0 <manifest-or-findings-dir> [--output DIR] [--verify]"
    echo ""
    echo "Replay reachability manifests from bench findings."
    echo ""
    echo "Options:"
    echo "  --output DIR   Output directory for replay results"
    echo "  --verify       Verify replay outputs against ground truth"
    echo "  --help, -h     Show this help"
    exit 1
}

INPUT=""
OUTPUT_DIR="${REPO_ROOT}/bench/results/replay"
VERIFY=false

while [[ $# -gt 0 ]]; do
    case $1 in
        --output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        --verify)
            VERIFY=true
            shift
            ;;
        --help|-h)
            usage
            ;;
        *)
            if [[ -z "$INPUT" ]]; then
                INPUT="$1"
            else
                echo "Unknown option: $1"
                usage
            fi
            shift
            ;;
    esac
done

if [[ -z "$INPUT" ]]; then
    # Default to bench/findings
    INPUT="${REPO_ROOT}/bench/findings"
fi

if [[ ! -e "$INPUT" ]]; then
    log_error "Input not found: $INPUT"
    exit 1
fi

mkdir -p "$OUTPUT_DIR"

log_info "Replay input: $INPUT"
log_info "Output directory: $OUTPUT_DIR"

# Collect all reachability evidence files
EVIDENCE_FILES=()

if [[ -d "$INPUT" ]]; then
    # Directory of findings
    while IFS= read -r -d '' file; do
        EVIDENCE_FILES+=("$file")
    done < <(find "$INPUT" -name "reachability.json" -print0 2>/dev/null)
elif [[ -f "$INPUT" ]]; then
    # Single manifest file
    EVIDENCE_FILES+=("$INPUT")
fi

if [[ ${#EVIDENCE_FILES[@]} -eq 0 ]]; then
    log_warn "No reachability evidence files found"
    exit 0
fi

log_info "Found ${#EVIDENCE_FILES[@]} evidence file(s)"

# Process each evidence file
TOTAL=0
PASSED=0
FAILED=0

for evidence_file in "${EVIDENCE_FILES[@]}"; do
    TOTAL=$((TOTAL + 1))
    finding_dir=$(dirname "$(dirname "$evidence_file")")
    finding_id=$(basename "$finding_dir")

    log_info "Processing: $finding_id"

    # Extract metadata
    metadata_file="${finding_dir}/metadata.json"
    if [[ ! -f "$metadata_file" ]]; then
        log_warn "  No metadata.json found, skipping"
        continue
    fi

    # Parse evidence
    evidence_hash=$(python3 -c "
import json
with open('$evidence_file') as f:
    d = json.load(f)
paths = d.get('paths', [])
print(f'paths={len(paths)}')
print(f'variant={d.get(\"variant\", \"unknown\")}')
print(f'case_id={d.get(\"case_id\", \"unknown\")}')
" 2>/dev/null || echo "error")

    if [[ "$evidence_hash" == "error" ]]; then
        log_warn "  Failed to parse evidence"
        FAILED=$((FAILED + 1))
        continue
    fi

    echo "  $evidence_hash"

    # Create replay output
    replay_output="${OUTPUT_DIR}/${finding_id}"
    mkdir -p "$replay_output"

    # Copy evidence for replay
    cp "$evidence_file" "$replay_output/evidence.json"

    # If verify mode, check against ground truth
    if [[ "$VERIFY" == true ]]; then
        ground_truth=$(python3 -c "
import json
with open('$evidence_file') as f:
    d = json.load(f)
gt = d.get('ground_truth')
if gt:
    print(f'variant={gt.get(\"variant\", \"unknown\")}')
    print(f'paths={len(gt.get(\"paths\", []))}')
else:
    print('no_ground_truth')
" 2>/dev/null || echo "error")

        if [[ "$ground_truth" != "no_ground_truth" && "$ground_truth" != "error" ]]; then
            log_info "  Ground truth: $ground_truth"
            PASSED=$((PASSED + 1))
        else
            log_warn "  No ground truth available"
        fi
    else
        PASSED=$((PASSED + 1))
    fi

    # Record replay result
    echo "{\"finding_id\": \"$finding_id\", \"status\": \"replayed\", \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" > "$replay_output/replay.json"
done

# Summary
echo ""
log_info "Replay Summary:"
log_info "  Total: $TOTAL"
log_info "  Passed: $PASSED"
log_info "  Failed: $FAILED"

# Write summary file
echo "{
  \"total\": $TOTAL,
  \"passed\": $PASSED,
  \"failed\": $FAILED,
  \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"
}" > "$OUTPUT_DIR/summary.json"

log_info "Summary written to: $OUTPUT_DIR/summary.json"