feat: move mteb_version to individual subsets in result files

Store mteb_version per subset in score dicts so that results evaluated
with different MTEB versions can be merged without erasing existing
subset scores. The top-level mteb_version is now computed as the latest
version across all subsets.

- Add mteb_version to each subset's score dict in from_task_results()
- Remove mteb_version from default merge criteria (is_mergeable/merge)
- Compute top-level mteb_version from per-subset versions after merge
- Fix bug in aggregated_task.py where mteb_version was always
  overwritten regardless of version mismatch condition

Closes #4354

Author: fzoll

mteb/abstasks/aggregated_task.py

@@ -126,8 +126,9 @@ def combine_task_results(self, task_results: list[TaskResult]) -> TaskResult:
             msg = f"All tasks of {self.metadata.name} is not run using the same version. different versions found are: {mteb_versions}"
             logger.warning(msg)
             warnings.warn(msg)
-            task_res.mteb_version = None
-        task_res.mteb_version = task_results[0].mteb_version
+        task_res.mteb_version = TaskResult._compute_top_level_mteb_version(
+            task_res.scores
+        )
         return task_res
 
     def evaluate(

mteb/results/task_result.py

@@ -194,6 +194,7 @@ def from_task_results(
         """
         task_meta = task.metadata
         subset2langscripts = task_meta.hf_subsets_to_langscripts
+        mteb_ver = version("mteb")
         flat_scores = defaultdict(list)
         for split, hf_subset_scores in scores.items():
             for hf_subset, hf_scores in hf_subset_scores.items():
@@ -202,13 +203,14 @@ def from_task_results(
                     **hf_scores,
                     "hf_subset": hf_subset,
                     "languages": eval_langs,
+                    "mteb_version": mteb_ver,
                 }
                 flat_scores[split].append(_scores)
 
         return TaskResult(
             dataset_revision=task.metadata.revision,
             task_name=task.metadata.name,
-            mteb_version=version("mteb"),
+            mteb_version=mteb_ver,
             scores=flat_scores,
             evaluation_time=evaluation_time,
             kg_co2_emissions=kg_co2_emissions,
@@ -730,7 +732,6 @@ def is_mergeable(
         self,
         result: TaskResult | AbsTask,
         criteria: list[str] | list[Criteria] = [
-            "mteb_version",
             "dataset_revision",
         ],
         raise_error: bool = False,
@@ -739,7 +740,7 @@ def is_mergeable(
 
         Args:
             result: The TaskResult or Task object to check against.
-            criteria: Additional criteria to check for merging. Can be "mteb_version" or "dataset_revision".
+            criteria: Additional criteria to check for merging. Can be "dataset_revision" or "mteb_version" (opt-in).
                 It will always check that the task name match.
             raise_error: If True, raises an error if the objects cannot be merged. If False, returns False.
 
@@ -789,7 +790,6 @@ def merge(
         self,
         new_results: TaskResult,
         criteria: list[str] | list[Criteria] = [
-            "mteb_version",
             "dataset_revision",
         ],
     ) -> TaskResult:
@@ -831,10 +831,12 @@ def merge(
         date = self.date
         if new_results.date is not None and (date is None or new_results.date > date):
             date = new_results.date
+        mteb_ver = self._compute_top_level_mteb_version(merged_scores)
+
         merged_results = TaskResult(
             dataset_revision=new_results.dataset_revision,
             task_name=new_results.task_name,
-            mteb_version=new_results.mteb_version,
+            mteb_version=mteb_ver,
             scores=merged_scores,
             evaluation_time=merged_evaluation_time,
             kg_co2_emissions=merged_kg_co2_emissions,
@@ -843,6 +845,25 @@ def merge(
 
         return merged_results
 
+    @staticmethod
+    def _compute_top_level_mteb_version(
+        scores: dict[SplitName, list[ScoresDict]],
+    ) -> str | None:
+        """Compute the top-level mteb_version from per-subset versions.
+
+        Returns the latest version found across all subsets, or None if no
+        per-subset versions are present.
+        """
+        versions: set[str] = set()
+        for split_scores in scores.values():
+            for subset_scores in split_scores:
+                v = subset_scores.get("mteb_version")
+                if v is not None:
+                    versions.add(v)
+        if not versions:
+            return None
+        return str(max(Version(v) for v in versions))
+
     @staticmethod
     def _merge_split_scores(
         existing_scores: list[ScoresDict], new_scores: list[ScoresDict]

tests/test_results/test_task_results.py

@@ -83,10 +83,11 @@ def test_task_results_get_score(task_result: TaskResult):
 
 
 def test_task_results_to_dict(task_result: TaskResult):
+    mteb_ver = version("mteb")
     dict_repr = {
         "dataset_revision": "1.0",
         "task_name": "dummy_task",
-        "mteb_version": version("mteb"),
+        "mteb_version": mteb_ver,
         "evaluation_time": 100,
         "date": None,
         "kg_co2_emissions": None,
@@ -96,11 +97,13 @@ def test_task_results_to_dict(task_result: TaskResult):
                     "main_score": 0.5,
                     "hf_subset": "en-de",
                     "languages": ["eng-Latn", "deu-Latn"],
+                    "mteb_version": mteb_ver,
                 },
                 {
                     "main_score": 0.6,
                     "hf_subset": "en-fr",
                     "languages": ["eng-Latn", "fra-Latn"],
+                    "mteb_version": mteb_ver,
                 },
             ]
         },
@@ -153,6 +156,109 @@ def test_task_results_validate_and_filter():
     assert res3.get_score() == (0.5 + 0.3) / 2  # only en-de scores
 
 
+def test_per_subset_mteb_version(task_result: TaskResult):
+    """Test that each subset score dict includes mteb_version."""
+    mteb_ver = version("mteb")
+    for split_scores in task_result.scores.values():
+        for subset_scores in split_scores:
+            assert "mteb_version" in subset_scores
+            assert subset_scores["mteb_version"] == mteb_ver
+
+
+def test_merge_across_mteb_versions():
+    """Test that results from different MTEB versions can be merged."""
+    existing = TaskResult(
+        dataset_revision="1.0",
+        task_name="dummy_task",
+        mteb_version="2.12.4",
+        scores={
+            "train": [
+                {
+                    "main_score": 0.5,
+                    "hf_subset": "en-de",
+                    "languages": ["eng-Latn", "deu-Latn"],
+                    "mteb_version": "2.12.4",
+                },
+            ]
+        },
+        evaluation_time=50,
+    )
+
+    new = TaskResult(
+        dataset_revision="1.0",
+        task_name="dummy_task",
+        mteb_version="2.20.1",
+        scores={
+            "train": [
+                {
+                    "main_score": 0.6,
+                    "hf_subset": "en-fr",
+                    "languages": ["eng-Latn", "fra-Latn"],
+                    "mteb_version": "2.20.1",
+                },
+            ]
+        },
+        evaluation_time=60,
+    )
+
+    assert existing.is_mergeable(new)
+    merged = existing.merge(new)
+
+    # Both subsets should be present
+    assert len(merged.scores["train"]) == 2
+    subsets = {s["hf_subset"]: s for s in merged.scores["train"]}
+    assert subsets["en-de"]["mteb_version"] == "2.12.4"
+    assert subsets["en-fr"]["mteb_version"] == "2.20.1"
+
+    # Top-level version should be the latest
+    assert merged.mteb_version == "2.20.1"
+
+
+def test_merge_without_per_subset_version():
+    """Test merging old results that lack per-subset mteb_version."""
+    existing = TaskResult(
+        dataset_revision="1.0",
+        task_name="dummy_task",
+        mteb_version="2.12.4",
+        scores={
+            "train": [
+                {
+                    "main_score": 0.5,
+                    "hf_subset": "en-de",
+                    "languages": ["eng-Latn", "deu-Latn"],
+                },
+            ]
+        },
+        evaluation_time=50,
+    )
+
+    new = TaskResult(
+        dataset_revision="1.0",
+        task_name="dummy_task",
+        mteb_version="2.20.1",
+        scores={
+            "train": [
+                {
+                    "main_score": 0.6,
+                    "hf_subset": "en-fr",
+                    "languages": ["eng-Latn", "fra-Latn"],
+                    "mteb_version": "2.20.1",
+                },
+            ]
+        },
+        evaluation_time=60,
+    )
+
+    merged = existing.merge(new)
+    # Old subset has no per-subset version, new one does
+    subsets = {s["hf_subset"]: s for s in merged.scores["train"]}
+    assert "mteb_version" not in subsets["en-de"]
+    assert subsets["en-fr"]["mteb_version"] == "2.20.1"
+
+    # Top-level should be the latest found across subsets
+    assert merged.mteb_version == "2.20.1"
+
+
 @pytest.mark.parametrize(
     "path", list((tests_folder / "historic_results").glob("*.json"))
 )