add qwen3-vl

Luciennnnnnn · Luciennnnnnn · commit a1e68b0a3ff0 · 2025-10-24T21:51:17.000+08:00
diff --git a/README.md b/README.md
@@ -127,7 +127,7 @@ pip install -e .
 #### ✅ (Recommended) Install Optional High-Performance Dependencies
 For the best performance, especially during inference, we highly recommend installing vllm.
 ```bash
-pip install vllm
+pip install -U vllm
 ```
 
 ---
diff --git a/editscore/__init__.py b/editscore/__init__.py
@@ -59,6 +59,27 @@ def __init__(
                 lora_path=lora_path,
                 cache_dir=cache_dir,
             )
+        elif self.backbone == "qwen3vl":
+            from .mllm_tools.qwen3vl import Qwen3VL
+            self.model = Qwen3VL(
+                vlm_model=model_name_or_path,
+                temperature=temperature,
+                seed=seed,
+                lora_path=lora_path,
+            )
+        elif self.backbone == "qwen3vl_vllm":
+            from .mllm_tools.qwen3vl_vllm import Qwen3VL
+            self.model = Qwen3VL(
+                vlm_model=model_name_or_path,
+                tensor_parallel_size=tensor_parallel_size,
+                max_model_len=max_model_len,
+                max_num_seqs=max_num_seqs,
+                max_num_batched_tokens=max_num_batched_tokens,
+                temperature=temperature,
+                seed=seed,
+                lora_path=lora_path,
+                cache_dir=cache_dir,
+            )
         elif self.backbone == "internvl3_5":
             from .mllm_tools.internvl35_lmdeploy import InternVL35
             self.model = InternVL35(model=model_name_or_path, tensor_parallel_size=tensor_parallel_size)
diff --git a/editscore/mllm_tools/qwen3vl.py b/editscore/mllm_tools/qwen3vl.py
@@ -0,0 +1,95 @@
+from typing import Optional
+import random
+import numpy as np
+import torch
+
+from transformers import Qwen3VLForConditionalGeneration, AutoProcessor
+from peft import PeftModel
+
+
+def set_seed(seed: int):
+    """
+    Args:
+    Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch`.
+        seed (`int`): The seed to set.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+def apply_chat_template(prompt, num_images: int = 2):
+    """
+    This is used since the bug of transformers which do not support vision id https://github.com/QwenLM/Qwen2.5-VL/issues/716#issuecomment-2723316100
+    """
+    template = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n"
+    template += "".join([f"<img{i}>: <|vision_start|><|image_pad|><|vision_end|>" for i in range(1, num_images + 1)])
+    template += f"{prompt}<|im_end|>\n<|im_start|>assistant\n"
+    return template
+
+
+class Qwen3VL():
+    def __init__(
+        self,
+        vlm_model,
+        temperature: float = 0.7,
+        seed: Optional[int] = None,
+        lora_path: Optional[str] = None,
+    ) -> None:
+        self.model = Qwen3VLForConditionalGeneration.from_pretrained(
+            vlm_model, torch_dtype=torch.bfloat16, device_map="auto"
+        )
+        if lora_path:
+            self.model = PeftModel.from_pretrained(self.model, lora_path)
+            self.model = self.model.merge_and_unload()
+
+        self.processor = AutoProcessor.from_pretrained(vlm_model)
+        self.temperature = temperature
+        self.seed = seed
+    
+    def prepare_input(self, images, text_prompt: str = ""):
+        if not isinstance(images, list):
+            images = [images]
+
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "image", "image": image} for image in images]
+                + [{"type": "text", "text": text_prompt}],
+            }
+        ]
+
+        inputs = self.processor.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_dict=True,
+            return_tensors="pt"
+        )
+
+        inputs = inputs.to("cuda")
+
+        return inputs
+
+    def inference(self, inputs, seed: Optional[int] = None):
+        seed = self.seed if seed is None else seed
+
+        set_seed(seed)
+        generated_ids = self.model.generate(
+            **inputs,
+            max_new_tokens=512,
+            do_sample=True,
+            temperature=self.temperature,
+            top_p=0.9,
+            top_k=20,
+        )
+        generated_ids_trimmed = [
+            out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+        ]
+        outputs = self.processor.batch_decode(
+            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+        outputs = [output.strip() for output in outputs]
+        return outputs[0]
diff --git a/editscore/mllm_tools/qwen3vl_vllm.py b/editscore/mllm_tools/qwen3vl_vllm.py
@@ -0,0 +1,141 @@
+from typing import Optional
+
+import os
+import hashlib
+import random
+import time
+import numpy as np
+import torch
+
+from vllm import LLM
+from vllm.sampling_params import SamplingParams
+
+from transformers import Qwen3VLForConditionalGeneration, AutoProcessor
+from peft import PeftModel
+
+from qwen_vl_utils import process_vision_info
+
+
+def set_seed(seed: int):
+    """
+    Args:
+    Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch`.
+        seed (`int`): The seed to set.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+def apply_chat_template(prompt, num_images: int = 2):
+    """
+    This is used since the bug of transformers which do not support vision id https://github.com/QwenLM/Qwen2.5-VL/issues/716#issuecomment-2723316100
+    """
+    template = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n"
+    template += "".join([f"<img{i}>: <|vision_start|><|image_pad|><|vision_end|>" for i in range(1, num_images + 1)])
+    template += f"{prompt}<|im_end|>\n<|im_start|>assistant\n"
+    return template
+
+
+class Qwen3VL():
+    def __init__(
+        self,
+        vlm_model,
+        max_model_len: int = 1536,
+        tensor_parallel_size=1,
+        max_num_seqs=32,
+        max_num_batched_tokens=1536,
+        temperature: float = 0.7,
+        seed: Optional[int] = None,
+        lora_path: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+    ) -> None:
+        if lora_path:
+            if cache_dir is None:
+                root_dir = torch.hub.get_dir() # default: ~/.cache/torch/hub
+
+                lora_filename = os.path.splitext(os.path.basename(lora_path))[0]
+                lora_hash = hashlib.md5(lora_path.encode()).hexdigest()[:8]
+                lora_identifier = f"{lora_filename}_{lora_hash}"
+
+                cache_dir = os.path.join(root_dir, "EditScore", f"{os.path.basename(vlm_model)}_merged_lora_{lora_identifier}")
+
+            if not os.path.exists(cache_dir):
+                print(f"Merging LORA to {vlm_model} and saving to {cache_dir}", flush=True)
+                start_time = time.time()
+                model = Qwen3VLForConditionalGeneration.from_pretrained(
+                    vlm_model, torch_dtype=torch.bfloat16, device_map="cpu"
+                )
+                model = PeftModel.from_pretrained(model, lora_path)
+                model = model.merge_and_unload()
+                model.save_pretrained(cache_dir)
+
+                processor = AutoProcessor.from_pretrained(vlm_model)
+                processor.save_pretrained(cache_dir)
+
+                print(f"Merging LORA to {vlm_model} and saving to {cache_dir} took {time.time() - start_time} seconds", flush=True)
+            else:
+                print(f"Skipping merging LORA, as merged model already exists in {cache_dir}", flush=True)
+
+            vlm_model = cache_dir
+
+        self.model = LLM(
+            model=vlm_model,
+            max_model_len=max_model_len,
+            tensor_parallel_size=tensor_parallel_size,
+            max_num_seqs=max_num_seqs,
+            max_num_batched_tokens=max_num_batched_tokens,
+            limit_mm_per_prompt={"image": 2},
+            enable_prefix_caching=True,
+        )
+
+        self.processor = AutoProcessor.from_pretrained(vlm_model)
+        self.temperature = temperature
+        self.seed = seed
+    
+    def prepare_input(self, images, text_prompt: str = ""):
+        if not isinstance(images, list):
+            images = [images]
+
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "image", "image": image} for image in images]
+                + [{"type": "text", "text": text_prompt}],
+            }
+        ]
+        # text = apply_chat_template(text_prompt, num_images=len(images))
+        text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        image_inputs, _ = process_vision_info(messages)
+
+        messages = {
+            "prompt": text,
+            "multi_modal_data": {"image": image_inputs},
+        }
+        return messages
+
+    def inference(self, messages, seed: Optional[int] = None):
+        seed = self.seed if seed is None else seed
+        sampling_params = SamplingParams(max_tokens=512, temperature=self.temperature, top_p=0.9, top_k=20, seed=seed)
+        outputs = self.model.generate(messages, sampling_params, use_tqdm=False)
+
+        responses = []
+        for output in outputs:
+            instruction = output.outputs[0].text.strip()
+            responses.append(instruction)
+
+        return responses[0]
+
+
+    def batch_inference(self, messages, seed: Optional[int] = None):
+        seed = self.seed if seed is None else seed
+        sampling_params = SamplingParams(max_tokens=512, temperature=self.temperature, top_p=0.9, top_k=20, seed=seed)
+        outputs = self.model.generate(messages, sampling_params, use_tqdm=False)
+
+        responses = []
+        for output in outputs:
+            instruction = output.outputs[0].text.strip()
+            responses.append(instruction)
+
+        return responses
diff --git a/evaluate.sh b/evaluate.sh
@@ -7,7 +7,6 @@ python evaluation.py \
 --result_dir results/EditScore-7B \
 --backbone qwen25vl \
 --model_name_or_path Qwen/Qwen2.5-VL-7B-Instruct \
---enable_lora \
 --lora_path EditScore/EditScore-7B \
 --score_range 25 \
 --max_workers 1 \
diff --git a/evaluate_72B_vllm.sh b/evaluate_72B_vllm.sh
@@ -7,7 +7,6 @@ python evaluation.py \
 --result_dir results/EditScore-72B \
 --backbone qwen25vl_vllm \
 --model_name_or_path Qwen/Qwen2.5-VL-72B-Instruct \
---enable_lora \
 --lora_path EditScore/EditScore-72B \
 --score_range 25 \
 --max_workers 1 \
diff --git a/evaluate_qwen3_vl_4B.sh b/evaluate_qwen3_vl_4B.sh
@@ -0,0 +1,23 @@
+# !/bin/bash
+SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
+cd $SHELL_FOLDER
+
+source "$(dirname $(which conda))/../etc/profile.d/conda.sh"
+conda activate editscore
+
+python evaluation.py \
+--benchmark_dir EditScore/EditReward-Bench \
+--result_dir results/EditScore-Qwen3-VL-4B \
+--backbone qwen3vl \
+--model_name_or_path /share/project/shared_models/Qwen3-VL-4B-Instruct \
+--lora_path /share/project/jiahao/LLaMA-Factory3/output/editscore_qwen3_4B_ins \
+--score_range 25 \
+--max_workers 1 \
+--max_model_len 4096 \
+--max_num_seqs 1 \
+--max_num_batched_tokens 4096 \
+--tensor_parallel_size 1 \
+--num_pass 1
+
+python calculate_statistics.py \
+--result_dir results/EditScore-Qwen3-VL-4B/qwen3vl
diff --git a/evaluate_qwen3_vl_4B_avg4.sh b/evaluate_qwen3_vl_4B_avg4.sh
@@ -0,0 +1,23 @@
+# !/bin/bash
+SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
+cd $SHELL_FOLDER
+
+source "$(dirname $(which conda))/../etc/profile.d/conda.sh"
+conda activate editscore
+
+python evaluation.py \
+--benchmark_dir EditScore/EditReward-Bench \
+--result_dir results/EditScore-Qwen3-VL-4B-avg4 \
+--backbone qwen3vl \
+--model_name_or_path /share/project/shared_models/Qwen3-VL-4B-Instruct \
+--lora_path /share/project/jiahao/LLaMA-Factory3/output/editscore_qwen3_4B_ins \
+--score_range 25 \
+--max_workers 1 \
+--max_model_len 4096 \
+--max_num_seqs 1 \
+--max_num_batched_tokens 4096 \
+--tensor_parallel_size 1 \
+--num_pass 4
+
+python calculate_statistics.py \
+--result_dir results/EditScore-Qwen3-VL-4B-avg4/qwen3vl
diff --git a/evaluate_qwen3_vl_4B_vllm.sh b/evaluate_qwen3_vl_4B_vllm.sh
@@ -0,0 +1,23 @@
+# !/bin/bash
+SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
+cd $SHELL_FOLDER
+
+source "$(dirname $(which conda))/../etc/profile.d/conda.sh"
+conda activate editscore
+
+python evaluation.py \
+--benchmark_dir EditScore/EditReward-Bench \
+--result_dir results/EditScore-Qwen3-VL-4B \
+--backbone qwen3vl_vllm \
+--model_name_or_path /share/project/shared_models/Qwen3-VL-4B-Instruct \
+--lora_path /share/project/jiahao/LLaMA-Factory3/output/editscore_qwen3_4B_ins \
+--score_range 25 \
+--max_workers 1 \
+--max_model_len 4096 \
+--max_num_seqs 1 \
+--max_num_batched_tokens 4096 \
+--tensor_parallel_size 1 \
+--num_pass 1
+
+python calculate_statistics.py \
+--result_dir results/EditScore-Qwen3-VL-4B/qwen3vl_vllm
diff --git a/evaluate_qwen3_vl_8B.sh b/evaluate_qwen3_vl_8B.sh
@@ -0,0 +1,23 @@
+# !/bin/bash
+SHELL_FOLDER=$(cd "$(dirname "$0")";pwd)
+cd $SHELL_FOLDER
+
+source "$(dirname $(which conda))/../etc/profile.d/conda.sh"
+conda activate editscore
+
+python evaluation.py \
+--benchmark_dir EditScore/EditReward-Bench \
+--result_dir results/EditScore-Qwen3-VL-8B \
+--backbone qwen3vl \
+--model_name_or_path /share/project/jiahao/models/Qwen3-VL-8B-Instruct \
+--lora_path /share/project/jiahao/LLaMA-Factory3/output/editscore_qwen3_8B_ins \
+--score_range 25 \
+--max_workers 1 \
+--max_model_len 4096 \
+--max_num_seqs 1 \
+--max_num_batched_tokens 4096 \
+--tensor_parallel_size 1 \
+--num_pass 1
+
+python calculate_statistics.py \
+--result_dir results/EditScore-Qwen3-VL-8B/qwen3vl
diff --git a/evaluate_qwen3_vl_8B_avg4.sh b/evaluate_qwen3_vl_8B_avg4.sh
diff --git a/evaluate_vllm.sh b/evaluate_vllm.sh
diff --git a/evaluation.py b/evaluation.py
