[MVEB] Add WorldSense 1min video-centric QA task

mteb/tasks/multichoice/eng/__init__.py

@@ -10,6 +10,7 @@
     PerceptionTestVideoCentricQA,
 )
 from .video_mme import VideoMMEShortVideoAudioCentricQA, VideoMMEShortVideoCentricQA
+from .worldsense import WorldSense1MinVideoAudioCentricQA, WorldSense1MinVideoCentricQA
 
 __all__ = [
     "AVMemeExamVideoAudioCentricQA",
@@ -28,4 +29,6 @@
     "PerceptionTestVideoCentricQA",
     "VideoMMEShortVideoAudioCentricQA",
     "VideoMMEShortVideoCentricQA",
+    "WorldSense1MinVideoAudioCentricQA",
+    "WorldSense1MinVideoCentricQA",
 ]

mteb/tasks/multichoice/eng/worldsense.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from datasets import Dataset, load_dataset
+
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class WorldSense1MinVideoCentricQA(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="WorldSense1MinVideoCentricQA",
+        description="WorldSense_1min is a video question answering benchmark covering diverse real-world domains including sports, culture, music, and daily life. Each example pairs a ~1-minute video with a question and multiple candidate answers. The task is formulated as multiple-choice retrieval: given the (video, question) pair, retrieve the correct candidate.",
+        reference="https://arxiv.org/abs/2502.04326",
+        dataset={
+            "path": "mteb/WorldSense_1min",
+            "revision": "10c7ce0eb32d620f1f685bfedde2724066068a1c",
+        },
+        type="VideoCentricQA",
+        category="vt2t",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2025-02-06", "2025-02-06"),
+        domains=["Web"],
+        task_subtypes=["Question answering"],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        modalities=["video", "text"],
+        sample_creation="found",
+        is_beta=True,
+        bibtex_citation=r"""
+@article{hong2025worldsense,
+  author = {Hong, Jack and Yan, Shilin and Cai, Jiayin and Jiang, Xiaolong and Hu, Yao and Xie, Weidi},
+  journal = {arXiv preprint arXiv:2502.04326},
+  title = {WorldSense: Evaluating Real-world Omnimodal Understanding for Multimodal LLMs},
+  year = {2025},
+}
+""",
+    )
+
+    def load_data(self, **kwargs) -> None:
+        if self.data_loaded:
+            return
+        self.dataset = {"default": {}}
+        for split in self.metadata.eval_splits:
+            ds = load_dataset(
+                self.metadata.dataset["path"],
+                revision=self.metadata.dataset["revision"],
+                split=split,
+            )
+            ds = ds.add_column("id", [f"q{i}" for i in range(len(ds))])
+
+            queries = ds.select_columns(["id", "question", "video"]).rename_column(
+                "question", "text"
+            )
+
+            corpus_rows: list[dict] = []
+            relevant_docs: dict[str, dict[str, int]] = {}
+            top_ranked: dict[str, list[str]] = {}
+            for row in ds.select_columns(["id", "candidates", "answer"]):
+                qid = row["id"]
+                answer = row["answer"]
+                top_ranked[qid] = []
+                for j, candidate in enumerate(row["candidates"]):
+                    doc_id = f"{qid}_c{j}"
+                    corpus_rows.append({"id": doc_id, "text": candidate})
+                    top_ranked[qid].append(doc_id)
+                    if candidate == answer:
+                        relevant_docs[qid] = {doc_id: 1}
+
+            corpus = Dataset.from_list(corpus_rows)
+            self.dataset["default"][split] = RetrievalSplitData(
+                queries=queries,
+                corpus=corpus,
+                relevant_docs=relevant_docs,
+                top_ranked=top_ranked,
+            )
+        self.data_loaded = True
+
+
+class WorldSense1MinVideoAudioCentricQA(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="WorldSense1MinVideoAudioCentricQA",
+        description="WorldSense_1min is a video question answering benchmark covering diverse real-world domains including sports, culture, music, and daily life. Each example pairs a ~1-minute video with audio and a question and multiple candidate answers. The task is formulated as multiple-choice retrieval: given the (video, audio, question) tuple, retrieve the correct candidate.",
+        reference="https://arxiv.org/abs/2502.04326",
+        dataset={
+            "path": "mteb/WorldSense_1min",
+            "revision": "10c7ce0eb32d620f1f685bfedde2724066068a1c",
+        },
+        type="VideoCentricQA",
+        category="vat2t",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2025-02-06", "2025-02-06"),
+        domains=["Web"],
+        task_subtypes=["Question answering"],
+        license="cc-by-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        modalities=["video", "audio", "text"],
+        sample_creation="found",
+        is_beta=True,
+        bibtex_citation=r"""
+@article{hong2025worldsense,
+  author = {Hong, Jack and Yan, Shilin and Cai, Jiayin and Jiang, Xiaolong and Hu, Yao and Xie, Weidi},
+  journal = {arXiv preprint arXiv:2502.04326},
+  title = {WorldSense: Evaluating Real-world Omnimodal Understanding for Multimodal LLMs},
+  year = {2025},
+}
+""",
+    )
+
+    def load_data(self, **kwargs) -> None:
+        if self.data_loaded:
+            return
+        self.dataset = {"default": {}}
+        for split in self.metadata.eval_splits:
+            ds = load_dataset(
+                self.metadata.dataset["path"],
+                revision=self.metadata.dataset["revision"],
+                split=split,
+            )
+            ds = ds.add_column("id", [f"q{i}" for i in range(len(ds))])
+
+            queries = ds.select_columns(
+                ["id", "question", "video", "audio"]
+            ).rename_column("question", "text")
+
+            corpus_rows: list[dict] = []
+            relevant_docs: dict[str, dict[str, int]] = {}
+            top_ranked: dict[str, list[str]] = {}
+            for row in ds.select_columns(["id", "candidates", "answer"]):
+                qid = row["id"]
+                answer = row["answer"]
+                top_ranked[qid] = []
+                for j, candidate in enumerate(row["candidates"]):
+                    doc_id = f"{qid}_c{j}"
+                    corpus_rows.append({"id": doc_id, "text": candidate})
+                    top_ranked[qid].append(doc_id)
+                    if candidate == answer:
+                        relevant_docs[qid] = {doc_id: 1}
+
+            corpus = Dataset.from_list(corpus_rows)
+            self.dataset["default"][split] = RetrievalSplitData(
+                queries=queries,
+                corpus=corpus,
+                relevant_docs=relevant_docs,
+                top_ranked=top_ranked,
+            )
+        self.data_loaded = True