ReMatch/experiments/public/eval/eval_rematch.sh at main · FireRedTeam/ReMatch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#!/usr/bin/env bash
set -euo pipefail

PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
cd "$PROJECT_ROOT"

export UCX_VFS_ENABLE=n
echo "==> Environment"
echo "conda location: $(which conda)"
echo "Python location: $(which python)"
echo ""

export EVAL_MAP_NUM_PROC="1"
export HF_ENDPOINT="https://huggingface.co"
# ==============================================================================
# Configuration
# ==============================================================================
BATCH_SIZE=${BATCH_SIZE:-16}
MODALITIES=${MODALITIES:-"image"}

DATA_BASEDIR=${DATA_BASEDIR:-"$PROJECT_ROOT/data/vlm2vec_eval"}
MODEL_BASEDIR=${MODEL_BASEDIR:-"$PROJECT_ROOT/outputs"}
OUTPUT_BASEDIR=${OUTPUT_BASEDIR:-"$PROJECT_ROOT/outputs/evals"}
DATA_CONFIG_DIR=${DATA_CONFIG_DIR:-"$PROJECT_ROOT/experiments/public/eval"}

GPU_NUM=${GPU_NUM:-8}
WORLD_SIZE=${WORLD_SIZE:-1}
RANK=${RANK:-0}
MASTER_PORT=${MASTER_PORT:-22442}
MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"}


TGT_PREFIX_INSTRUCTION=${TGT_PREFIX_INSTRUCTION:-True}
MODEL_NAME=${MODEL_NAME:-"Rematch_Qwen2.5vl_3B.image.autoresize.lora32.loraAlpha64.BS1024.IB64.GCq32p32NormTemp002.lr1e4.step3kwarm100.lrCosine.TgtInstruction.mmE5H1.Queries16.ResidualAvg.OrthTriu0.2.ChatTemplate.ITM.V1.Ratio0.1"}
CHECKPOINT_NAME=${CHECKPOINT_NAME:-"checkpoint-2200"}
MODEL_BACKBONE=${MODEL_BACKBONE:-"qwen2_5_vl"}
POOLING=${POOLING:-"eos"}
LORA=${LORA:-true}
CHECKPOINT_PATH=${CHECKPOINT_PATH:-}

NUM_QUERIES=${NUM_QUERIES:-16}
LEARNABLE_QUERIES=${LEARNABLE_QUERIES:-true}
RESIDUAL_EMBEDDING=${RESIDUAL_EMBEDDING:-true}
RESIDUAL_EMBEDDING_METHOD=${RESIDUAL_EMBEDDING_METHOD:-"avg"}
CHAT_TEMPLATE=${CHAT_TEMPLATE:-true}

# ==> Define models and their base output paths here
# Format: "MODEL_NAME;model_backbone (see src/model/vlm_backbone);BASE_OUTPUT_PATH"
declare -a MODEL_SPECS
if [[ -n "$CHECKPOINT_PATH" ]]; then
  OUTPUT_NAME=${OUTPUT_NAME:-"${MODEL_NAME//\//_}"}
  MODEL_SPECS+=( "$MODEL_NAME;$MODEL_BACKBONE;$OUTPUT_BASEDIR/$OUTPUT_NAME" )
else
  MODEL_SPECS+=( "$MODEL_BASEDIR/$MODEL_NAME/$CHECKPOINT_NAME;$MODEL_BACKBONE;$OUTPUT_BASEDIR/$MODEL_NAME/$CHECKPOINT_NAME" )
fi


# ==============================================================================
# Main Execution Loop
# ==============================================================================
# Loop through each model specification
for spec in "${MODEL_SPECS[@]}"; do
  # Parse the model name and base output path from the spec string
  IFS=';' read -r MODEL_NAME MODEL_BACKBONE BASE_OUTPUT_PATH <<< "$spec"

  echo "================================================="
  echo "Processing Model: $MODEL_NAME"
  echo "================================================="

  # Loop through each modality for the current model
  for MODALITY in $MODALITIES; do
    DATA_CONFIG_PATH="$DATA_CONFIG_DIR/$MODALITY.yaml"
    OUTPUT_PATH="$BASE_OUTPUT_PATH/$MODALITY/"

    echo "-------------------------------------------------"
    echo "  - Modality: $MODALITY"
    echo "  - Output Path: $OUTPUT_PATH"

    # Ensure the output directory exists
    mkdir -p "$OUTPUT_PATH"

    CHECKPOINT_ARG=""
    if [[ -n "$CHECKPOINT_PATH" ]]; then
      CHECKPOINT_ARG="--checkpoint_path \"$CHECKPOINT_PATH\""
    fi

    cmd="EVAL_MAP_NUM_PROC=$EVAL_MAP_NUM_PROC torchrun --nproc_per_node=$GPU_NUM --master_port=$MASTER_PORT --max_restarts=0 eval.py \
      --lora \"$LORA\" \
      --pooling $POOLING \
      --normalize true \
      --tgt_prefix_instruction \"$TGT_PREFIX_INSTRUCTION\" \
      --learnable_queries \"$LEARNABLE_QUERIES\" \
      --residual_embedding \"$RESIDUAL_EMBEDDING\" \
      --residual_embedding_method \"$RESIDUAL_EMBEDDING_METHOD\" \
      --enable_chat_template \"$CHAT_TEMPLATE\" \
      --num_queries $NUM_QUERIES \
      --per_device_eval_batch_size $BATCH_SIZE \
      --model_backbone \"$MODEL_BACKBONE\" \
      --model_name \"$MODEL_NAME\" \
      $CHECKPOINT_ARG \
      --dataset_config \"$DATA_CONFIG_PATH\" \
      --encode_output_path \"$OUTPUT_PATH\" \
      --data_basedir \"$DATA_BASEDIR\""

    echo "  - Executing command..."
    eval "$cmd"
    echo "  - Done."
    echo "-------------------------------------------------"
  done
done

echo "All jobs completed."