set -e
set -u

MODEL_NAME=math-shepherd
MODEL_DIR=/path/to/math-shepherd
MODEL_EVAL_DIR=../outputs/trained_model_name

GEN_DIRS=(
    # $MODEL_EVAL_DIR/MATH500/rule_based/gen8_temp1.0_topp1/xxxxxx_timestamp
    # $MODEL_EVAL_DIR/AIME2024/rule_based/gen8_temp1.0_topp1/xxxxxx_timestamp
    # $MODEL_EVAL_DIR/AMC2023/rule_based/gen8_temp1.0_topp1/xxxxxx_timestamp
    # $MODEL_EVAL_DIR/minerva_math/rule_based/gen8_temp1.0_topp1/xxxxxx_timestamp
    # $MODEL_EVAL_DIR/olympiadbench/rule_based/gen8_temp1.0_topp1/xxxxxx_timestamp
)


for i in "${!GEN_DIRS[@]}"; do
    GEN_DIR=${GEN_DIRS[$i]}
    GEN_FILE=$GEN_DIR/generation.jsonl
    CONFIG_FILE=$GEN_DIR/config.json
    OUT_DIR=$GEN_DIR/${MODEL_NAME}_as_verifier
    mkdir -p $OUT_DIR

    python3 ../math_verify_shepherd.py \
        --model_name $MODEL_DIR \
        --generation_file $GEN_FILE \
        --config_file $CONFIG_FILE \
        --output_dir $OUT_DIR

    python3 ../math_eval.py compute_metrics \
        --config_file $OUT_DIR/config.json \
        --pass_ratio \
        --c_acc_ms # verification accuracy for math-shepherd (ms)
done
