feat: add manga VQA task and related utilities; update dependencies and Python version

Author: mserra0

.python-version

@@ -0,0 +1 @@
+3.11

=0.11.0

@@ -1,4 +1,12 @@
 #!/bin/bash
+HF_CACHE="/data/users/mserrao/hf-cache"
+export CUDA_VISIBLE_DEVICES=7
+export HUGGIN
+export HF_HOME=$HF_CACHE
+export TRANSFORMERS_CACHE=$HF_CACHE
+export HF_DATASETS_CACHE=$HF_CACHE
+export HUGGINGFACE_HUB_CACHE=$HF_CACHE
+# export HF_HUB_OFFLINE=1
 
 # Qwen3-VL Evaluation Script with vLLM Backend
 # This script demonstrates how to evaluate Qwen3-VL models using vLLM for accelerated inference
@@ -23,20 +31,20 @@
 # - Qwen/Qwen3-VL-30B-A3B-Thinking
 # - Qwen/Qwen3-VL-235B-A22B-Instruct
 # - Qwen/Qwen3-VL-235B-A22B-Thinking
-MODEL="Qwen/Qwen3-VL-30B-A3B-Instruct"
+MODEL="Qwen/Qwen3-VL-2B-Instruct"
 
 # Parallelization Settings
 # Adjust based on your GPU configuration
-TENSOR_PARALLEL_SIZE=4  # Number of GPUs for tensor parallelism
+TENSOR_PARALLEL_SIZE=1 # Number of GPUs for tensor parallelism
 DATA_PARALLEL_SIZE=1     # Number of GPUs for data parallelism
 
 # Memory and Performance Settings
-GPU_MEMORY_UTILIZATION=0.85  # Fraction of GPU memory to use (0.0 - 1.0)
-BATCH_SIZE=64                # Batch size for evaluation
-
+GPU_MEMORY_UTILIZATION=0.90  # Fraction of GPU memory to use (0.0 - 1.0)
+BATCH_SIZE=16                # Batch size for evaluation
+MAX_MODEL_LEN=$((2**15))
 # Task Configuration
 # Common tasks: mmmu_val, mme, mathvista, ai2d, etc.
-TASKS="mmmu_val,mme"
+TASKS="manga_vqa_test"
 
 # Output Configuration
 OUTPUT_PATH="./logs/qwen3vl_vllm"
@@ -69,12 +77,14 @@ echo "Output Path: $OUTPUT_PATH"
 echo "=========================================="
 
 # Build the command
-CMD="uv run python -m lmms_eval \
+CMD="python -m lmms_eval \
     --model vllm \
-    --model_args model=${MODEL},tensor_parallel_size=${TENSOR_PARALLEL_SIZE},data_parallel_size=${DATA_PARALLEL_SIZE},gpu_memory_utilization=${GPU_MEMORY_UTILIZATION} \
+    --model_args model=${MODEL},tensor_parallel_size=${TENSOR_PARALLEL_SIZE},data_parallel_size=${DATA_PARALLEL_SIZE},gpu_memory_utilization=${GPU_MEMORY_UTILIZATION},max_model_len=${MAX_MODEL_LEN}\
     --tasks ${TASKS} \
     --batch_size ${BATCH_SIZE} \
-    --output_path ${OUTPUT_PATH}"
+    --output_path ${OUTPUT_PATH} \
+    --verbosity=DEBUG" 
+    
 
 # Add optional arguments
 if [ "$LOG_SAMPLES" = true ]; then
@@ -96,4 +106,4 @@ echo ""
 echo "=========================================="
 echo "Evaluation Complete!"
 echo "Results saved to: $OUTPUT_PATH"
-echo "=========================================="
+echo "=========================================="

examples/models/vllm_qwen3vl.sh

@@ -0,0 +1,46 @@
+dataset_path: hal-utokyo/MangaVQA
+# dataset_name: MangaVQA
+task: "manga_vqa_test"
+dataset_kwargs:
+  token: True
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.mangavqa_doc_to_visual
+doc_to_text: !function utils.mangavqa_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 32
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: False
+process_results: !function utils.mangavqa_test_process_results
+metric_list:
+  - metric: submission
+    aggregation: !function utils.mangavqa_test_aggregate_results
+    higher_is_better: true
+  - metric: anls
+    aggregation: mean
+    higher_is_better: true
+  - metric: f1
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer the question using a single word or phrase."
+  qwen_vl:
+    pre_prompt: ""
+    post_prompt: " Answer:"
+metadata:
+  - version: 0.0

lmms_eval/tasks/manga_vqa/manga_vqa_test.yaml

@@ -0,0 +1,46 @@
+import json
+import os
+
+from PIL import Image
+
+from loguru import logger
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+IMAGES_PATH = "/data-net/storage/users/evivoli/CoMix/data/datasets/Manga109/images"
+
+def mangavqa_doc_to_visual(doc):
+    book = doc["title"]
+    image_num = doc["image_number"]
+    
+    image = Image.open(os.path.join(IMAGES_PATH, book, f"{image_num:03d}.jpg"))
+
+    return [image.convert("RGB")]
+
+def mangavqa_doc_to_text(doc, lmms_eval_specific_kwargs):
+    question = doc["question"]
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    return f"{pre_prompt}{question}{post_prompt}"
+
+def mangavqa_test_process_results(doc, results):
+    pred = results[0]
+    questionId = doc["questionId"]
+    
+    results_dict = {
+        "anls": {"questionId": int(questionId), "answer": pred}, 
+        "f1": {"questionId": int(questionId), "answer": pred},
+        "exact_match": {"questionId": int(questionId), "answer": pred},
+        "submission": {"questionId": int(questionId), "answer": pred},
+        }
+    
+    return results_dict
+
+def mangavqa_test_aggregate_results(results, args):
+    # save results as json
+    path = generate_submission_file("docvqa_test_for_submission.json", args)
+    with open(path, "w") as f:
+        json.dump(results, f)
+    logger.info(f"Results saved to {path}")
+
+

lmms_eval/tasks/manga_vqa/utils.py

@@ -19,7 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Operating System :: OS Independent",
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "accelerate>=0.29.1",
     "black>=24.1.0",
@@ -51,7 +51,7 @@ dependencies = [
     "yt-dlp",
     "pycocoevalcap",
     "tqdm-multiprocess",
-    "transformers>=4.39.2",
+    "transformers==4.57.1",
     "transformers-stream-generator",
     "zstandard",
     "pillow",
@@ -74,6 +74,8 @@ dependencies = [
     "sentence_transformers",
     "python-dotenv",
     "qwen-vl-utils>=0.0.14",
+    "vllm>=0.11.0",
+    "decord>=0.6.0",
 ]
 
 [project.optional-dependencies]