From f1dbe3e1fe977ac134ddca79d850f1ea99a75cb6 Mon Sep 17 00:00:00 2001 From: Rakshitha Ireddi Date: Wed, 29 Apr 2026 17:24:26 +0530 Subject: [PATCH 1/2] [MVEB] Add WorldSense1MinVideoAudioCentricQA task and fix modalities --- mteb/tasks/multichoice/eng/worldsense.py | 153 +++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 mteb/tasks/multichoice/eng/worldsense.py diff --git a/mteb/tasks/multichoice/eng/worldsense.py b/mteb/tasks/multichoice/eng/worldsense.py new file mode 100644 index 0000000000..ff09b5f27d --- /dev/null +++ b/mteb/tasks/multichoice/eng/worldsense.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from datasets import Dataset, load_dataset + +from mteb.abstasks.retrieval import AbsTaskRetrieval +from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData +from mteb.abstasks.task_metadata import TaskMetadata + + +class WorldSense1MinVideoCentricQA(AbsTaskRetrieval): + metadata = TaskMetadata( + name="WorldSense1MinVideoCentricQA", + description="WorldSense_1min is a video question answering benchmark covering diverse real-world domains including sports, culture, music, and daily life. Each example pairs a ~1-minute video with a question and multiple candidate answers. The task is formulated as multiple-choice retrieval: given the (video, question) pair, retrieve the correct candidate.", + reference="https://arxiv.org/abs/2502.04326", + dataset={ + "path": "mteb/WorldSense_1min", + "revision": "10c7ce0eb32d620f1f685bfedde2724066068a1c", + }, + type="VideoCentricQA", + category="vt2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2025-02-06", "2025-02-06"), + domains=["Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + modalities=["video", "text"], + sample_creation="found", + is_beta=True, + bibtex_citation=r""" +@article{hong2025worldsense, + author = {Hong, Jack and Yan, Shilin and Cai, Jiayin and Jiang, Xiaolong and Hu, Yao and Xie, Weidi}, + journal = {arXiv preprint arXiv:2502.04326}, + title = {WorldSense: Evaluating Real-world Omnimodal Understanding for Multimodal LLMs}, + year = {2025}, +} +""", + ) + + def load_data(self, **kwargs) -> None: + if self.data_loaded: + return + self.dataset = {"default": {}} + for split in self.metadata.eval_splits: + ds = load_dataset( + self.metadata.dataset["path"], + revision=self.metadata.dataset["revision"], + split=split, + ) + ds = ds.add_column("id", [f"q{i}" for i in range(len(ds))]) + + queries = ds.select_columns(["id", "question", "video"]).rename_column( + "question", "text" + ) + + corpus_rows: list[dict] = [] + relevant_docs: dict[str, dict[str, int]] = {} + top_ranked: dict[str, list[str]] = {} + for row in ds.select_columns(["id", "candidates", "answer"]): + qid = row["id"] + answer = row["answer"] + top_ranked[qid] = [] + for j, candidate in enumerate(row["candidates"]): + doc_id = f"{qid}_c{j}" + corpus_rows.append({"id": doc_id, "text": candidate}) + top_ranked[qid].append(doc_id) + if candidate == answer: + relevant_docs[qid] = {doc_id: 1} + + corpus = Dataset.from_list(corpus_rows) + self.dataset["default"][split] = RetrievalSplitData( + queries=queries, + corpus=corpus, + relevant_docs=relevant_docs, + top_ranked=top_ranked, + ) + self.data_loaded = True + + +class WorldSense1MinVideoAudioCentricQA(AbsTaskRetrieval): + metadata = TaskMetadata( + name="WorldSense1MinVideoAudioCentricQA", + description="WorldSense_1min is a video question answering benchmark covering diverse real-world domains including sports, culture, music, and daily life. Each example pairs a ~1-minute video with audio and a question and multiple candidate answers. The task is formulated as multiple-choice retrieval: given the (video, audio, question) tuple, retrieve the correct candidate.", + reference="https://arxiv.org/abs/2502.04326", + dataset={ + "path": "mteb/WorldSense_1min", + "revision": "10c7ce0eb32d620f1f685bfedde2724066068a1c", + }, + type="VideoCentricQA", + category="vat2t", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2025-02-06", "2025-02-06"), + domains=["Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + modalities=["video", "audio", "text"], + sample_creation="found", + is_beta=True, + bibtex_citation=r""" +@article{hong2025worldsense, + author = {Hong, Jack and Yan, Shilin and Cai, Jiayin and Jiang, Xiaolong and Hu, Yao and Xie, Weidi}, + journal = {arXiv preprint arXiv:2502.04326}, + title = {WorldSense: Evaluating Real-world Omnimodal Understanding for Multimodal LLMs}, + year = {2025}, +} +""", + ) + + def load_data(self, **kwargs) -> None: + if self.data_loaded: + return + self.dataset = {"default": {}} + for split in self.metadata.eval_splits: + ds = load_dataset( + self.metadata.dataset["path"], + revision=self.metadata.dataset["revision"], + split=split, + ) + ds = ds.add_column("id", [f"q{i}" for i in range(len(ds))]) + + queries = ds.select_columns( + ["id", "question", "video", "audio"] + ).rename_column("question", "text") + + corpus_rows: list[dict] = [] + relevant_docs: dict[str, dict[str, int]] = {} + top_ranked: dict[str, list[str]] = {} + for row in ds.select_columns(["id", "candidates", "answer"]): + qid = row["id"] + answer = row["answer"] + top_ranked[qid] = [] + for j, candidate in enumerate(row["candidates"]): + doc_id = f"{qid}_c{j}" + corpus_rows.append({"id": doc_id, "text": candidate}) + top_ranked[qid].append(doc_id) + if candidate == answer: + relevant_docs[qid] = {doc_id: 1} + + corpus = Dataset.from_list(corpus_rows) + self.dataset["default"][split] = RetrievalSplitData( + queries=queries, + corpus=corpus, + relevant_docs=relevant_docs, + top_ranked=top_ranked, + ) + self.data_loaded = True From ca8176585f8a0073a9f32d0567dcab096f877600 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 29 Apr 2026 15:24:58 +0300 Subject: [PATCH 2/2] add init --- mteb/tasks/multichoice/eng/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mteb/tasks/multichoice/eng/__init__.py b/mteb/tasks/multichoice/eng/__init__.py index 3d0f8c7796..3fc51e9e6b 100644 --- a/mteb/tasks/multichoice/eng/__init__.py +++ b/mteb/tasks/multichoice/eng/__init__.py @@ -10,6 +10,7 @@ PerceptionTestVideoCentricQA, ) from .video_mme import VideoMMEShortVideoAudioCentricQA, VideoMMEShortVideoCentricQA +from .worldsense import WorldSense1MinVideoAudioCentricQA, WorldSense1MinVideoCentricQA __all__ = [ "AVMemeExamVideoAudioCentricQA", @@ -28,4 +29,6 @@ "PerceptionTestVideoCentricQA", "VideoMMEShortVideoAudioCentricQA", "VideoMMEShortVideoCentricQA", + "WorldSense1MinVideoAudioCentricQA", + "WorldSense1MinVideoCentricQA", ]