[Benchmark] Add support for LongDocURL (#1595)

* [Benchmark] Add support for LongDocURL

Co-authored-by: Cursor <cursoragent@cursor.com>

* Fix lint errors for LongDocURL benchmark

* Trigger LongDocURL CI rerun

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Haodong Duan <dhd@pku.edu.cn>
Co-authored-by: kenny.alan <kenny.alan@bytedance.com>

Author: 3 people

scripts/build_longdocurl_tsv.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Build a VLMEvalKit TSV for LongDocURL from the public Hugging Face JSONL."""
+
+import argparse
+import json
+import os
+import os.path as osp
+
+import pandas as pd
+
+
+REPO_ID = 'dengchao/LongDocURL'
+DATA_FILE = 'LongDocURL_public_with_subtask_category.jsonl'
+
+
+def hf_download(filename, download_dir, token=None):
+    try:
+        from huggingface_hub import hf_hub_download
+    except ImportError as err:
+        raise ImportError(
+            'huggingface_hub is required to download LongDocURL. '
+            'Install it with `pip install huggingface_hub`, or pass --jsonl.'
+        ) from err
+
+    return hf_hub_download(
+        repo_id=REPO_ID,
+        filename=filename,
+        repo_type='dataset',
+        local_dir=download_dir,
+        token=token,
+    )
+
+
+def json_dumps(value):
+    return json.dumps(value, ensure_ascii=False)
+
+
+def relative_image_path(path):
+    path = str(path)
+    marker = '/pdf_pngs/'
+    if marker in path:
+        return path.split(marker, 1)[1]
+    return path.lstrip('/')
+
+
+def row_from_sample(sample, idx):
+    images = [relative_image_path(p) for p in sample.get('images', [])]
+    answer = sample.get('answer', '')
+    return {
+        'index': idx,
+        'question_id': sample.get('question_id', ''),
+        'question': sample.get('question', ''),
+        'answer': json_dumps(answer) if isinstance(answer, list) else answer,
+        'image_path': json_dumps(images),
+        'doc_no': sample.get('doc_no', ''),
+        'total_pages': sample.get('total_pages', ''),
+        'start_end_idx': json_dumps(sample.get('start_end_idx', [])),
+        'question_type': sample.get('question_type', ''),
+        'answer_format': sample.get('answer_format', ''),
+        'task_tag': sample.get('task_tag', ''),
+        'evidence_pages': json_dumps(sample.get('evidence_pages', [])),
+        'evidence_sources': json_dumps(sample.get('evidence_sources', [])),
+        'subTask': json_dumps(sample.get('subTask', [])),
+        'detailed_evidences': sample.get('detailed_evidences', ''),
+        'pdf_path': sample.get('pdf_path', ''),
+    }
+
+
+def load_jsonl(path):
+    rows = []
+    with open(path, 'r', encoding='utf-8') as f:
+        for line in f:
+            if line.strip():
+                rows.append(json.loads(line))
+    return rows
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Build VLMEvalKit TSV for LongDocURL.')
+    parser.add_argument('--jsonl', default=None, help='Optional local LongDocURL JSONL path.')
+    parser.add_argument('--download-dir', default=None, help='Directory for downloaded JSONL.')
+    parser.add_argument('--output', required=True, help='Output TSV path, e.g. LongDocURL.tsv.')
+    parser.add_argument('--token', default=os.environ.get('HF_TOKEN'), help='Optional Hugging Face token.')
+    parser.add_argument('--limit', type=int, default=None, help='Optional limit for debugging.')
+    args = parser.parse_args()
+
+    jsonl_path = args.jsonl
+    if jsonl_path is None:
+        download_dir = args.download_dir or osp.join(osp.dirname(osp.abspath(args.output)), 'downloads')
+        os.makedirs(download_dir, exist_ok=True)
+        print(f'downloading {DATA_FILE} from {REPO_ID} ...')
+        jsonl_path = hf_download(DATA_FILE, download_dir, token=args.token)
+
+    samples = load_jsonl(jsonl_path)
+    if args.limit is not None:
+        samples = samples[:args.limit]
+    data = pd.DataFrame([row_from_sample(sample, i) for i, sample in enumerate(samples)])
+    os.makedirs(osp.dirname(osp.abspath(args.output)), exist_ok=True)
+    data.to_csv(args.output, sep='\t', index=False)
+    print(f'wrote {len(data)} rows -> {args.output}')
+
+
+if __name__ == '__main__':
+    main()

scripts/prepare_longdocurl_images.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Download and extract LongDocURL PNG images into the VLMEvalKit cache."""
+
+import argparse
+import os
+
+from vlmeval.dataset.longdocurl import LongDocURL
+from vlmeval.smp import LMUDataRoot
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Prepare LongDocURL image files.')
+    parser.add_argument(
+        '--image-root',
+        default=None,
+        help='Target image root. Defaults to LMUData/images/LongDocURL/pdf_pngs.',
+    )
+    args = parser.parse_args()
+
+    if args.image_root:
+        os.environ['LONGDOCURL_IMAGE_ROOT'] = args.image_root
+    else:
+        os.environ.setdefault(
+            'LONGDOCURL_IMAGE_ROOT',
+            os.path.join(LMUDataRoot(), 'images', 'LongDocURL', 'pdf_pngs'),
+        )
+
+    dataset = LongDocURL('LongDocURL')
+    rel_paths = []
+    for _, row in dataset.data.iterrows():
+        rel_paths.extend(row['image_path'])
+    dataset._ensure_images(rel_paths)
+    print(f'LongDocURL images are prepared under {dataset._img_root()}')
+
+
+if __name__ == '__main__':
+    main()

setup.cfg

@@ -5,6 +5,8 @@ per-file-ignores =
     run.py: E402
     vlmeval/vlm/__init__.py: F401, E402
     vlmeval/dataset/video_dataset_config.py: F405, F403
+    vlmeval/dataset/utils/longdocurl.py: E501
+    vlmeval/dataset/utils/mmlongbench_judge_prompt.py: E501
     vlmeval/dataset/utils/memlens_utils.py: E501
 
 [yapf]

vlmeval/dataset/__init__.py

@@ -63,6 +63,7 @@
                         QSpatial, SeePhys, TableVQABench, TallyQA, TDBenchGrounding, VGRPBench,
                         VizWiz, VLMsAreBiased, VTCBench, WildDocBenchmark, ZEROBench)
 from .image_yorn import ImageYORNDataset
+from .longdocurl import LongDocURL
 from .longvideobench import LongVideoBench
 from .m3oralbench import M3oralBenchDataset
 from .m4bench import M4Bench
@@ -279,7 +280,7 @@ def evaluate(self, eval_file, **judge_kwargs):
 IMAGE_DATASET = [
     ImageCaptionDataset, ImageYORNDataset, ImageMCQDataset, ImageVQADataset,
     MathVision, LENS, MMMUDataset, OCRBench, MathVista, LLaVABench, LLaVABench_KO, VGRPBench, MMVet,  # noqa: E501
-    MTVQADataset, TableVQABench, MMLongBench, MemLens, MMLongBenchDoc, VCRDataset, MMDUDataset, DUDE,
+    MTVQADataset, TableVQABench, MMLongBench, MemLens, MMLongBenchDoc, VCRDataset, MMDUDataset, DUDE, LongDocURL,
     SlideVQA, MUIRDataset, CCOCRDataset, GMAIMMBenchDataset, MMERealWorld,
     HRBenchDataset, CRPE, MathVerse, NaturalBenchDataset, MIABench,
     OlympiadBench, SeePhys, WildVision, MMMath, QSpatial, Dynamath, GSM8KVDataset, MMGenBench, VizWiz,  # noqa: E501