fix(tasks): utils improvements across 7 benchmarks

lmms_eval/tasks/egoschema/README.md

@@ -10,24 +10,30 @@
 
 ## Evaluation
 
-### Full set: Submission
+### Full set: Submission (NOT scorable offline)
 
-EgoSchema is intended for a 0-shot evaluation benchmark, hence the entire correct answer file will not be make public. 
+EgoSchema is intended for a 0-shot evaluation benchmark, hence the entire correct answer file will not be make public. The cached `GENERATION`/`MC` configs ship every `answer` field as `None`, so the full-set tasks (`egoschema`, `egoschema_mcppl`) only emit a `submission` metric — they cannot compute a real accuracy locally.
 
 `lmms-eval` will automatically generate a submission file `inference_results_egoschema_{taskname}_{now_date_time}.json` under `logs/`. To evaluate on the entire benchmark,  please submit the generated submission file using CURL:
 
 `curl -X POST -H "Content-Type: application/json" -d @<path_to_json_file> https://validation-server.onrender.com/api/upload/`
 
-### Subset: Direct Scoring
+On a no-egress cluster this submission step (and the upstream Hub data-files lookup) raises `ConnectionError`, so the full-set tasks are unusable offline. Use a subset task instead.
 
-<a href="https://github.com/egoschema/EgoSchema">EgoSchema</a> also release the correct answers to only 500 of the EgoSchema questions provided in the subset_answers.json file intended for offline experimentation and performance tracking. Hence,`lmms-eval` will automatically generate the score for subset.
+### Subset: Direct Scoring (offline-capable, N=500)
+
+<a href="https://github.com/egoschema/EgoSchema">EgoSchema</a> also release the correct answers to only 500 of the EgoSchema questions provided in the subset_answers.json file intended for offline experimentation and performance tracking. Hence,`lmms-eval` will automatically generate the score for subset. The `Subset` config carries real `answer` indices (`0`–`4`) for all 500 questions, so `egoschema_subset` produces a genuine `score` (accuracy) metric with no network access.
+
+#### Offline cache note
+
+`datasets.load_dataset("lmms-lab/egoschema", "Subset")` resolves data files via the Hub even under `HF_HUB_OFFLINE=1`, which fails with `ConnectionError` on a no-egress node *unless the Arrow cache for the `Subset` config has already been built* under `$HF_DATASETS_CACHE` (e.g. `…/datasets/lmms-lab___egoschema/Subset/`). Build it once on a networked host (login node) before the offline run; the videos must also be unzipped under `$HF_HOME/egoschema/videos/`.
 
 # Tasks
 
-- `egoschema`: Standard MCQA for Full set. 
-- `egoschema_mc_ppl`: MCQA Perplexity task format for Full set.
-- `egoschema_subset`: Standard MCQA for Subset. 
-- `egoschema_subset_mcppl`: MCQA Perplexity task format for Subset.
+- `egoschema`: Standard MCQA for Full set. Submission-only — no offline score.
+- `egoschema_mcppl`: MCQA Perplexity task format for Full set. Submission-only — no offline score.
+- `egoschema_subset`: Standard MCQA for Subset (N=500). Scores offline. **Use this for offline eval.**
+- `egoschema_subset_mcppl`: MCQA Perplexity task format for Subset (N=500). Scores offline.
 
 ## Citation
 

lmms_eval/tasks/egoschema/_default_template_yaml

@@ -6,4 +6,10 @@ dataset_kwargs:
 lmms_eval_specific_kwargs:
   default:
     pre_prompt: ""
-    post_prompt: ""
+    post_prompt: ""
+generation_kwargs:
+  max_new_tokens: 16
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false

lmms_eval/tasks/egoschema/egoschema_subset.yaml

@@ -7,9 +7,6 @@ doc_to_text: !function utils.egoschema_doc_to_text
 doc_to_target: !function utils.egoschema_doc_to_answer
 process_results: !function utils.egoschema_process_results_generation
 metric_list:
-  - metric: submission
-    aggregation: !function utils.egoschema_aggregate_mc
-    higher_is_better: true
   - metric: score
     aggregation: !function utils.egoschema_aggregate_score
     higher_is_better: true

lmms_eval/tasks/egoschema/egoschema_subset_mcppl.yaml

@@ -8,9 +8,6 @@ doc_to_target: !function utils.egoschema_doc_to_answer
 doc_to_choice: !function utils.egoschema_doc_to_choice
 process_results: !function utils.egoschema_process_results
 metric_list:
-  - metric: submission
-    aggregation: !function utils.egoschema_aggregate_mc_ppl
-    higher_is_better: true
   - metric: score
     aggregation: !function utils.egoschema_aggregate_score
     higher_is_better: true

lmms_eval/tasks/egoschema/utils.py

@@ -2,9 +2,12 @@
 import json
 import os
 import random
+import re
 import sys
+import time
 
 import numpy as np
+import requests
 from loguru import logger as eval_logger
 
 import lmms_eval.tasks._task_utils.file_utils as file_utils
@@ -15,7 +18,7 @@
 # We will unzip all the zip files
 # To HF HOME cache dir
 # And load it here
-HF_HOME = os.environ["HF_HOME"] if "HF_HOME" in os.environ else os.path.expanduser("~/.cache/huggingface/hub")
+HF_HOME = os.environ["HF_HOME"] if "HF_HOME" in os.environ else os.path.expanduser("~/.cache/huggingface")
 cache_dir = config["dataset_kwargs"]["cache_dir"]
 cache_dir = os.path.join(HF_HOME, cache_dir)
 cache_dir = os.path.join(cache_dir, "videos")
@@ -184,19 +187,52 @@ def egoschema_aggregate_submissions(results, args, task):
     for submission_dict in results:
         combined_submission.update(submission_dict)
 
+    # Save the submission first so the predictions are never lost if scoring fails.
     with open(path, "w") as f:
         json.dump(combined_submission, f, indent=4)
-
     eval_logger.info(f"Submission file saved to {path}")
 
+    # Score by POSTing to EgoSchema's public validation server (the full test
+    # set's labels aren't public). That host is free-tier: it cold-starts and
+    # returns 5xx for ~a minute while waking, so retry with backoff + a per-
+    # request timeout — but bounded, and fail loudly on persistent failure
+    # rather than spinning forever (the old `while True`) or recording a bogus 0.
+    url = "https://validation-server.onrender.com/api/upload/"
+    headers = {"Content-Type": "application/json"}
+    pattern = r"Total result: \d+ correct, \d+ wrong, \d+ missing, \d+ invalid, accuracy: ([\d.]+)"
+    max_attempts = 8
+    last_err = None
+    for attempt in range(1, max_attempts + 1):
+        try:
+            response = requests.post(url, headers=headers, json=combined_submission, timeout=60)
+        except requests.exceptions.RequestException as e:
+            last_err = f"{type(e).__name__}: {e}"
+            eval_logger.warning(f"EgoSchema submission attempt {attempt}/{max_attempts} errored: {last_err}")
+        else:
+            if response.status_code == 200:
+                eval_logger.info(f"Submission successful: {response.text}")
+                match = re.search(pattern, response.text)
+                if match:
+                    total_accuracy = float(match.group(1)) * 100
+                    eval_logger.info(f"Total accuracy: {total_accuracy}")
+                    return total_accuracy
+                # 200 but unexpected format — retrying won't help, fail loudly.
+                raise RuntimeError(f"EgoSchema server returned HTTP 200 but no parseable accuracy. Response: {response.text[:300]}. Submission saved at {path}.")
+            last_err = f"HTTP {response.status_code}: {response.text[:200]}"
+            eval_logger.warning(f"EgoSchema submission attempt {attempt}/{max_attempts} failed: {last_err}")
+        if attempt < max_attempts:
+            time.sleep(min(30, 5 * attempt))  # linear backoff, capped at 30s
+
+    raise RuntimeError(f"EgoSchema validation-server scoring failed after {max_attempts} attempts (last: {last_err}). " f"Submission saved at {path} — score it manually.")
+
 
 # Factory into different aggregate
 def egoschema_aggregate_mc(results, args):
-    egoschema_aggregate_submissions(results, args, "MC")
+    return egoschema_aggregate_submissions(results, args, "MC")
 
 
 def egoschema_aggregate_mc_ppl(results, args):
-    egoschema_aggregate_submissions(results, args, "MC_PPL")
+    return egoschema_aggregate_submissions(results, args, "MC_PPL")
 
 
 def egoschema_aggregate_score(results, args):

lmms_eval/tasks/erqa/_default_template_yaml

@@ -3,6 +3,7 @@ test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.erqa_doc_to_visual
 doc_to_text: !function utils.erqa_doc_to_text
+doc_to_messages: !function utils.erqa_doc_to_messages
 doc_to_target: "answer"
 
 process_results: !function utils.erqa_process_results

lmms_eval/tasks/erqa/tests/test_erqa_utils.py

@@ -0,0 +1,32 @@
+from PIL import Image
+
+from lmms_eval.tasks.erqa import utils
+
+
+def test_erqa_process_results_extracts_final_choice():
+    doc = {"answer": "B", "question_id": "ERQA_1", "question_type": "Action Reasoning"}
+
+    result = utils.erqa_process_results(doc, ["I considered A, but the answer is B."])
+
+    assert result["erqa_acc"]["is_correct"] is True
+    assert result["erqa_acc"]["id"] == "ERQA_1"
+
+
+def test_erqa_doc_to_messages_interleaves_images_before_text():
+    image = Image.new("RGB", (2, 2), color="white")
+    doc = {"images": [image], "question": "Choices: A. yes B. no"}
+
+    messages = utils.erqa_doc_to_messages(doc)
+
+    assert messages[0]["role"] == "user"
+    assert messages[0]["content"][0]["type"] == "image"
+    assert messages[0]["content"][1] == {"type": "text", "text": doc["question"]}
+
+
+def test_erqa_aggregate_results_returns_accuracy():
+    results = [
+        {"sub_task": "Action Reasoning", "is_correct": True},
+        {"sub_task": "Action Reasoning", "is_correct": False},
+    ]
+
+    assert utils.erqa_aggregate_results(results) == 0.5

lmms_eval/tasks/erqa/utils.py

@@ -1,34 +1,10 @@
 import logging
-import re
 from collections import defaultdict
 
-from lmms_eval.tasks._task_utils.default_template_yaml import load_default_template_yaml
+from lmms_eval.tasks._task_utils.mcq_extract import extract_mcq_answer
 
 eval_logger = logging.getLogger("lmms-eval")
 
-config = load_default_template_yaml(__file__)
-
-
-def _extract_answer_letter(text: str) -> str:
-    """
-    Extract the answer choice letter from a string.
-
-    Examples:
-    'A answer1' -> 'A'
-    'A) answer2' -> 'A'
-    '(B) answer' -> 'B'
-    'C' -> 'C'
-    '(C)' -> 'C'
-    'A.' -> 'A'
-
-    Return an empty string if no letter is found.
-    """
-    text = text.strip()
-    match = re.match(r"[\(\s]*([A-Z])[\)\.\s]*", text, flags=re.IGNORECASE)
-    if match:
-        return match.group(1).upper()
-    return ""
-
 
 def erqa_doc_to_text(doc: dict) -> str:
     return doc["question"]
@@ -42,14 +18,20 @@ def erqa_doc_to_visual(doc: dict) -> list:
     return image_list
 
 
+def erqa_doc_to_messages(doc: dict, lmms_eval_specific_kwargs=None) -> list[dict]:
+    content = [{"type": "image", "url": image} for image in erqa_doc_to_visual(doc)]
+    content.append({"type": "text", "text": erqa_doc_to_text(doc)})
+    return [{"role": "user", "content": content}]
+
+
 def erqa_process_results(doc, results):
     key_name = "erqa_acc"
     # extract grounded answer
     grounded_output = doc["answer"]
     response = results[0]
 
     # extract predicted answer
-    pred_letter = _extract_answer_letter(response)
+    pred_letter = extract_mcq_answer(response, choices=["A", "B", "C", "D"])
     flag = pred_letter == grounded_output
 
     omnispatial_submission = {"id": doc["question_id"], "gt_content": grounded_output, "pred": response, "sub_task": doc["question_type"], "is_correct": flag}

lmms_eval/tasks/mmsi_video/mmsi_video_single_input.yaml

@@ -0,0 +1,16 @@
+# MMSI-Video-Bench: single-visual-input subset (native video).
+# Drops ref_images / Cross-Video samples via process_docs so every question has
+# exactly one video (773/1106 samples). This matches Cambrian-P's reported
+# MMSI-Video protocol (a single video input per sample). The base
+# `mmsi_video` task is left unfiltered (the full benchmark definition).
+task: "mmsi_video_single_input"
+process_docs: !function utils.process_docs_single_input_only
+doc_to_visual: !function utils.doc_to_visual_video
+doc_to_text: !function utils.doc_to_text_video
+doc_to_messages: !function utils.doc_to_messages_video
+lmms_eval_specific_kwargs:
+  default:
+    post_prompt: ""
+  gemini_api:
+    pre_prompt: ""
+include: _default_template_yaml

lmms_eval/tasks/mmsi_video/utils.py

@@ -1,13 +1,10 @@
 # MMSI-Video-Bench: A Holistic Benchmark for Video-Based Spatial Intelligence
 # https://huggingface.co/datasets/rbler/MMSI-Video-Bench
 
-import json
 import os
 import re
-from collections import defaultdict
 from pathlib import Path
 
-import datasets
 import yaml
 from loguru import logger as eval_logger
 from PIL import Image
@@ -152,6 +149,21 @@ def is_nan_or_none(value) -> bool:
     return False
 
 
+##################
+# Dataset filtering (process_docs)
+##################
+def process_docs_single_input_only(dataset):
+    """Restrict to the single-visual-input subset used by Cambrian-P's reported MMSI-Video protocol.
+
+    Drops samples that carry reference images or are "Cross-Video" type, leaving only
+    samples with a single video input.
+
+    ``ref_images`` is a (possibly empty) list/ndarray per sample; the ``or []`` guard also
+    covers a None value defensively.
+    """
+    return dataset.filter(lambda x: len(x.get("ref_images", []) or []) == 0 and "Cross-Video" not in x["type"])
+
+
 ##################
 # doc_to_visual functions
 ##################
@@ -213,7 +225,7 @@ def doc_to_visual_frames(doc, lmms_eval_specific_kwargs=None):
     # Load sampled frames as PIL Images
     pil_images = []
     eval_logger.debug(f"[doc_to_visual_frames] sampled_frames_list has {len(sampled_frames_list)} segments")
-    for seg_idx, frames in enumerate(sampled_frames_list):
+    for frames in sampled_frames_list:
         for frame_path in frames:
             full_path = os.path.join(cache_dir, "frames", frame_path)
             if not os.path.exists(full_path):