Merge branch 'main' of https://github.com/EvolvingLMMs-Lab/OneVision-Encoder into anxiang_v2

Author: anxiangsir

README.md

@@ -301,27 +301,11 @@ cd eval_encoder
 Then run the following command:
 
 ```bash
-torchrun --nproc_per_node=8 --master_port=29512 attentive_probe_codec.py \
-  --eval_freq 1 \
-  --default_lr_list 0.0001 \
-  --batch_size 4 \
-  --default_weight_decay 0 \
-  --dali_py_num_workers 8 \
-  --model_family ov_encoder_codec \
-  --dataset diving48 \
-  --model_weight lmms-lab-encoder/onevision-encoder-large \
-  --model_name ov_encoder_large \
-  --embedding_size 1024 \
-  --default_epoch 30 \
-  --K_keep 2048 \
-  --num_frames 64 \
-  --mv_compensate median
-
+bash shells_eval_ap/eval_ov_encoder_large_2kpatches_codec.sh
 ```
 
 **Codec-Specific Parameters:**
 - `K_keep`: Number of patches to keep.
-- `mv_compensate`: Motion vector compensation method (e.g., `median`).
 - `cache_dir` (optional): Directory for cached codec patches. Use this to specify where codec-selected patches are stored/loaded when you want to persist or reuse them.
 
 #### Shared Parameters

dataloader/ap_dataloader_dali_codec.py

@@ -1060,8 +1060,8 @@ def get_dali_dataloader_codec(
     center_prior: float = 0.3,
     center_sigma: float = 0.35,
     static_fallback: bool = False,
-    static_abs_thresh: float = 116.0,
-    static_rel_thresh: float = 0.55,
+    static_abs_thresh: float = 126.0,
+    static_rel_thresh: float = 0.38,
     static_uniform_frames: int = 4,
 ) -> DALIWarper:
     """

eval_encoder/attentive_probe_codec.py

@@ -77,17 +77,17 @@ def parse_args() -> argparse.Namespace:
                         help="Neighborhood size (odd >=3) for MV inconsistency (default: 3)")
     parser.add_argument("--res_use_grad", action="store_true",
                         help="Use gradient-based residual energy instead of |res-128| (default: False)")
-    parser.add_argument("--center_prior", type=float, default=0.0,
-                        help="Center Gaussian prior strength applied to fused energy map before top-k (0 disables) (default: 0.0)")
+    parser.add_argument("--center_prior", type=float, default=0.3,
+                        help="Center Gaussian prior strength applied to fused energy map before top-k (0 disables) (default: 0.3)")
     parser.add_argument("--center_sigma", type=float, default=0.35,
                         help="Center Gaussian sigma as a fraction of min(H,W) (default: 0.35)")
     # Static / low-motion fallback (hybrid: uniform few frames + remaining top-k)
     parser.add_argument("--static_fallback", type=int, default=1,
                         help="Enable static-video hybrid fallback (1 enables, 0 disables) (default: 1)")
-    parser.add_argument("--static_abs_thresh", type=float, default=2.0,
-                        help="Absolute low-energy threshold on patch mean intensity (~0..255) (default: 2.0)")
-    parser.add_argument("--static_rel_thresh", type=float, default=0.15,
-                        help="Relative contrast threshold (0..1), smaller means flatter distribution (default: 0.15)")
+    parser.add_argument("--static_abs_thresh", type=float, default=126,
+                        help="Absolute low-energy threshold on patch mean intensity (~0..255) (default: 126)")
+    parser.add_argument("--static_rel_thresh", type=float, default=0.38,
+                        help="Relative contrast threshold (0..1), smaller means flatter distribution (default: 0.38)")
     parser.add_argument("--static_uniform_frames", type=int, default=4,
                         help="Number of uniformly-picked frames used in hybrid fallback (default: 4)")
 

eval_encoder/shells_eval_ap/common_codec.sh

@@ -51,26 +51,6 @@ get_epochs() {
     fi
 }
 
-# ============================================================================
-# Get codec parameters based on dataset name
-# Args: $1 - dataset name
-# Returns: Sets CODEC_MV_COMPENSATE, CODEC_STATIC_ABS_THRESH, CODEC_STATIC_REL_THRESH
-# ============================================================================
-get_codec_params() {
-    local dataset="$1"
-    if [[ "$dataset" == "diving48" || "$dataset" == "perception_test" ]]; then
-        # Parameters for diving48 and perception_test
-        CODEC_MV_COMPENSATE="similarity"
-        CODEC_STATIC_ABS_THRESH="126"
-        CODEC_STATIC_REL_THRESH="0.38"
-    else
-        # Parameters for other datasets
-        CODEC_MV_COMPENSATE="similarity"
-        CODEC_STATIC_ABS_THRESH="116"
-        CODEC_STATIC_REL_THRESH="0.55"
-    fi
-}
-
 # ============================================================================
 # Run attentive_probe_codec evaluation
 # Required variables to set before calling:
@@ -81,16 +61,14 @@ get_codec_params() {
 #   - EMBEDDING_SIZE: embedding dimension (optional, default 768)
 #   - INPUT_SIZE: input size (optional, not passed if unset)
 #   - NUM_FRAMES: number of frames (optional, not passed if unset)
-#   - K_keep: number of top-K patches to keep (optional, default 2048)
 #   - DATASETS: dataset array (optional, uses DEFAULT_DATASETS if unset/empty)
-#   - REPORT_DIR_SUFFIX: report directory suffix (optional, e.g. "_64frames_codec")
+#   - REPORT_DIR_SUFFIX: report directory suffix (optional, e.g. "_16frames")
 # ============================================================================
 run_attentive_probe_codec() {
     # Set default values
     MODEL_WEIGHT="${MODEL_WEIGHT:-NULL}"
     FRAMES_TOKEN_NUM="${FRAMES_TOKEN_NUM:-196}"
     EMBEDDING_SIZE="${EMBEDDING_SIZE:-768}"
-    K_keep="${K_keep:-2048}"
     REPORT_DIR_SUFFIX="${REPORT_DIR_SUFFIX:-}"
 
     # Use custom datasets or default datasets
@@ -99,18 +77,14 @@ run_attentive_probe_codec() {
     fi
 
     # Build report directory
-    BASE_REPORT_DIR="result_attentive_probe/${MODEL_FAMILY}/${MODEL_NAME}${REPORT_DIR_SUFFIX}"
+    BASE_REPORT_DIR="result_attentive_probe_codec/${MODEL_FAMILY}/${MODEL_NAME}${REPORT_DIR_SUFFIX}"
 
     # Loop through each dataset for testing
     for DATASET in "${DATASETS[@]}"; do
         BATCH_SIZE=$(get_batch_size "$DATASET")
         EPOCHS=$(get_epochs "$DATASET")
-        
-        # Get codec-specific parameters for this dataset
-        get_codec_params "$DATASET"
 
         echo "DATASET=$DATASET, BATCH_SIZE=$BATCH_SIZE"
-        echo "Codec params: mv_compensate=${CODEC_MV_COMPENSATE}, static_abs_thresh=${CODEC_STATIC_ABS_THRESH}, static_rel_thresh=${CODEC_STATIC_REL_THRESH}"
 
         echo "========================================================"
         echo "Start testing dataset: ${DATASET}"
@@ -132,6 +106,11 @@ run_attentive_probe_codec() {
             EXTRA_ARGS="${EXTRA_ARGS} --num_frames ${NUM_FRAMES}"
         fi
 
+        # Add --mv_use_inconsistency for all datasets except diving48
+        if [[ "$DATASET" != "diving48" ]]; then
+            EXTRA_ARGS="${EXTRA_ARGS} --mv_use_inconsistency"
+        fi
+
         torchrun --nproc_per_node 8 --master_port 15555 \
             attentive_probe_codec.py \
             --eval_freq 1 \
@@ -147,11 +126,6 @@ run_attentive_probe_codec() {
             --save_report "${SAVE_DIR}" \
             --frames_token_num ${FRAMES_TOKEN_NUM} \
             --embedding_size ${EMBEDDING_SIZE} \
-            --K_keep ${K_keep} \
-            --mv_compensate ${CODEC_MV_COMPENSATE} \
-            --static_abs_thresh ${CODEC_STATIC_ABS_THRESH} \
-            --static_rel_thresh ${CODEC_STATIC_REL_THRESH} \
-            --static_fallback 1 \
             ${EXTRA_ARGS}
 
         echo "Finished testing ${DATASET}"