diff --git a/lmms_eval/tasks/egoschema/README.md b/lmms_eval/tasks/egoschema/README.md index 92255bcb0..ad2bf9fc6 100644 --- a/lmms_eval/tasks/egoschema/README.md +++ b/lmms_eval/tasks/egoschema/README.md @@ -10,24 +10,30 @@ ## Evaluation -### Full set: Submission +### Full set: Submission (NOT scorable offline) -EgoSchema is intended for a 0-shot evaluation benchmark, hence the entire correct answer file will not be make public. +EgoSchema is intended for a 0-shot evaluation benchmark, hence the entire correct answer file will not be make public. The cached `GENERATION`/`MC` configs ship every `answer` field as `None`, so the full-set tasks (`egoschema`, `egoschema_mcppl`) only emit a `submission` metric — they cannot compute a real accuracy locally. `lmms-eval` will automatically generate a submission file `inference_results_egoschema_{taskname}_{now_date_time}.json` under `logs/`. To evaluate on the entire benchmark, please submit the generated submission file using CURL: `curl -X POST -H "Content-Type: application/json" -d @ https://validation-server.onrender.com/api/upload/` -### Subset: Direct Scoring +On a no-egress cluster this submission step (and the upstream Hub data-files lookup) raises `ConnectionError`, so the full-set tasks are unusable offline. Use a subset task instead. -EgoSchema also release the correct answers to only 500 of the EgoSchema questions provided in the subset_answers.json file intended for offline experimentation and performance tracking. Hence,`lmms-eval` will automatically generate the score for subset. +### Subset: Direct Scoring (offline-capable, N=500) + +EgoSchema also release the correct answers to only 500 of the EgoSchema questions provided in the subset_answers.json file intended for offline experimentation and performance tracking. Hence,`lmms-eval` will automatically generate the score for subset. The `Subset` config carries real `answer` indices (`0`–`4`) for all 500 questions, so `egoschema_subset` produces a genuine `score` (accuracy) metric with no network access. + +#### Offline cache note + +`datasets.load_dataset("lmms-lab/egoschema", "Subset")` resolves data files via the Hub even under `HF_HUB_OFFLINE=1`, which fails with `ConnectionError` on a no-egress node *unless the Arrow cache for the `Subset` config has already been built* under `$HF_DATASETS_CACHE` (e.g. `…/datasets/lmms-lab___egoschema/Subset/`). Build it once on a networked host (login node) before the offline run; the videos must also be unzipped under `$HF_HOME/egoschema/videos/`. # Tasks -- `egoschema`: Standard MCQA for Full set. -- `egoschema_mc_ppl`: MCQA Perplexity task format for Full set. -- `egoschema_subset`: Standard MCQA for Subset. -- `egoschema_subset_mcppl`: MCQA Perplexity task format for Subset. +- `egoschema`: Standard MCQA for Full set. Submission-only — no offline score. +- `egoschema_mcppl`: MCQA Perplexity task format for Full set. Submission-only — no offline score. +- `egoschema_subset`: Standard MCQA for Subset (N=500). Scores offline. **Use this for offline eval.** +- `egoschema_subset_mcppl`: MCQA Perplexity task format for Subset (N=500). Scores offline. ## Citation diff --git a/lmms_eval/tasks/egoschema/_default_template_yaml b/lmms_eval/tasks/egoschema/_default_template_yaml index 29b214702..a5ece914d 100644 --- a/lmms_eval/tasks/egoschema/_default_template_yaml +++ b/lmms_eval/tasks/egoschema/_default_template_yaml @@ -6,4 +6,10 @@ dataset_kwargs: lmms_eval_specific_kwargs: default: pre_prompt: "" - post_prompt: "" \ No newline at end of file + post_prompt: "" +generation_kwargs: + max_new_tokens: 16 + temperature: 0 + top_p: 1.0 + num_beams: 1 + do_sample: false diff --git a/lmms_eval/tasks/egoschema/egoschema_subset.yaml b/lmms_eval/tasks/egoschema/egoschema_subset.yaml index 6b63c34f8..365d43783 100755 --- a/lmms_eval/tasks/egoschema/egoschema_subset.yaml +++ b/lmms_eval/tasks/egoschema/egoschema_subset.yaml @@ -7,9 +7,6 @@ doc_to_text: !function utils.egoschema_doc_to_text doc_to_target: !function utils.egoschema_doc_to_answer process_results: !function utils.egoschema_process_results_generation metric_list: - - metric: submission - aggregation: !function utils.egoschema_aggregate_mc - higher_is_better: true - metric: score aggregation: !function utils.egoschema_aggregate_score higher_is_better: true diff --git a/lmms_eval/tasks/egoschema/egoschema_subset_mcppl.yaml b/lmms_eval/tasks/egoschema/egoschema_subset_mcppl.yaml index ebe868649..3232dfeac 100755 --- a/lmms_eval/tasks/egoschema/egoschema_subset_mcppl.yaml +++ b/lmms_eval/tasks/egoschema/egoschema_subset_mcppl.yaml @@ -8,9 +8,6 @@ doc_to_target: !function utils.egoschema_doc_to_answer doc_to_choice: !function utils.egoschema_doc_to_choice process_results: !function utils.egoschema_process_results metric_list: - - metric: submission - aggregation: !function utils.egoschema_aggregate_mc_ppl - higher_is_better: true - metric: score aggregation: !function utils.egoschema_aggregate_score higher_is_better: true diff --git a/lmms_eval/tasks/egoschema/utils.py b/lmms_eval/tasks/egoschema/utils.py index c85a441d0..c229d2d73 100755 --- a/lmms_eval/tasks/egoschema/utils.py +++ b/lmms_eval/tasks/egoschema/utils.py @@ -2,9 +2,12 @@ import json import os import random +import re import sys +import time import numpy as np +import requests from loguru import logger as eval_logger import lmms_eval.tasks._task_utils.file_utils as file_utils @@ -15,7 +18,7 @@ # We will unzip all the zip files # To HF HOME cache dir # And load it here -HF_HOME = os.environ["HF_HOME"] if "HF_HOME" in os.environ else os.path.expanduser("~/.cache/huggingface/hub") +HF_HOME = os.environ["HF_HOME"] if "HF_HOME" in os.environ else os.path.expanduser("~/.cache/huggingface") cache_dir = config["dataset_kwargs"]["cache_dir"] cache_dir = os.path.join(HF_HOME, cache_dir) cache_dir = os.path.join(cache_dir, "videos") @@ -184,19 +187,52 @@ def egoschema_aggregate_submissions(results, args, task): for submission_dict in results: combined_submission.update(submission_dict) + # Save the submission first so the predictions are never lost if scoring fails. with open(path, "w") as f: json.dump(combined_submission, f, indent=4) - eval_logger.info(f"Submission file saved to {path}") + # Score by POSTing to EgoSchema's public validation server (the full test + # set's labels aren't public). That host is free-tier: it cold-starts and + # returns 5xx for ~a minute while waking, so retry with backoff + a per- + # request timeout — but bounded, and fail loudly on persistent failure + # rather than spinning forever (the old `while True`) or recording a bogus 0. + url = "https://validation-server.onrender.com/api/upload/" + headers = {"Content-Type": "application/json"} + pattern = r"Total result: \d+ correct, \d+ wrong, \d+ missing, \d+ invalid, accuracy: ([\d.]+)" + max_attempts = 8 + last_err = None + for attempt in range(1, max_attempts + 1): + try: + response = requests.post(url, headers=headers, json=combined_submission, timeout=60) + except requests.exceptions.RequestException as e: + last_err = f"{type(e).__name__}: {e}" + eval_logger.warning(f"EgoSchema submission attempt {attempt}/{max_attempts} errored: {last_err}") + else: + if response.status_code == 200: + eval_logger.info(f"Submission successful: {response.text}") + match = re.search(pattern, response.text) + if match: + total_accuracy = float(match.group(1)) * 100 + eval_logger.info(f"Total accuracy: {total_accuracy}") + return total_accuracy + # 200 but unexpected format — retrying won't help, fail loudly. + raise RuntimeError(f"EgoSchema server returned HTTP 200 but no parseable accuracy. Response: {response.text[:300]}. Submission saved at {path}.") + last_err = f"HTTP {response.status_code}: {response.text[:200]}" + eval_logger.warning(f"EgoSchema submission attempt {attempt}/{max_attempts} failed: {last_err}") + if attempt < max_attempts: + time.sleep(min(30, 5 * attempt)) # linear backoff, capped at 30s + + raise RuntimeError(f"EgoSchema validation-server scoring failed after {max_attempts} attempts (last: {last_err}). " f"Submission saved at {path} — score it manually.") + # Factory into different aggregate def egoschema_aggregate_mc(results, args): - egoschema_aggregate_submissions(results, args, "MC") + return egoschema_aggregate_submissions(results, args, "MC") def egoschema_aggregate_mc_ppl(results, args): - egoschema_aggregate_submissions(results, args, "MC_PPL") + return egoschema_aggregate_submissions(results, args, "MC_PPL") def egoschema_aggregate_score(results, args): diff --git a/lmms_eval/tasks/erqa/_default_template_yaml b/lmms_eval/tasks/erqa/_default_template_yaml index 727af9002..6e31e25cf 100644 --- a/lmms_eval/tasks/erqa/_default_template_yaml +++ b/lmms_eval/tasks/erqa/_default_template_yaml @@ -3,6 +3,7 @@ test_split: test output_type: generate_until doc_to_visual: !function utils.erqa_doc_to_visual doc_to_text: !function utils.erqa_doc_to_text +doc_to_messages: !function utils.erqa_doc_to_messages doc_to_target: "answer" process_results: !function utils.erqa_process_results diff --git a/lmms_eval/tasks/erqa/tests/test_erqa_utils.py b/lmms_eval/tasks/erqa/tests/test_erqa_utils.py new file mode 100644 index 000000000..bce5dae8a --- /dev/null +++ b/lmms_eval/tasks/erqa/tests/test_erqa_utils.py @@ -0,0 +1,32 @@ +from PIL import Image + +from lmms_eval.tasks.erqa import utils + + +def test_erqa_process_results_extracts_final_choice(): + doc = {"answer": "B", "question_id": "ERQA_1", "question_type": "Action Reasoning"} + + result = utils.erqa_process_results(doc, ["I considered A, but the answer is B."]) + + assert result["erqa_acc"]["is_correct"] is True + assert result["erqa_acc"]["id"] == "ERQA_1" + + +def test_erqa_doc_to_messages_interleaves_images_before_text(): + image = Image.new("RGB", (2, 2), color="white") + doc = {"images": [image], "question": "Choices: A. yes B. no"} + + messages = utils.erqa_doc_to_messages(doc) + + assert messages[0]["role"] == "user" + assert messages[0]["content"][0]["type"] == "image" + assert messages[0]["content"][1] == {"type": "text", "text": doc["question"]} + + +def test_erqa_aggregate_results_returns_accuracy(): + results = [ + {"sub_task": "Action Reasoning", "is_correct": True}, + {"sub_task": "Action Reasoning", "is_correct": False}, + ] + + assert utils.erqa_aggregate_results(results) == 0.5 diff --git a/lmms_eval/tasks/erqa/utils.py b/lmms_eval/tasks/erqa/utils.py index 28294c535..d384cc766 100644 --- a/lmms_eval/tasks/erqa/utils.py +++ b/lmms_eval/tasks/erqa/utils.py @@ -1,34 +1,10 @@ import logging -import re from collections import defaultdict -from lmms_eval.tasks._task_utils.default_template_yaml import load_default_template_yaml +from lmms_eval.tasks._task_utils.mcq_extract import extract_mcq_answer eval_logger = logging.getLogger("lmms-eval") -config = load_default_template_yaml(__file__) - - -def _extract_answer_letter(text: str) -> str: - """ - Extract the answer choice letter from a string. - - Examples: - 'A answer1' -> 'A' - 'A) answer2' -> 'A' - '(B) answer' -> 'B' - 'C' -> 'C' - '(C)' -> 'C' - 'A.' -> 'A' - - Return an empty string if no letter is found. - """ - text = text.strip() - match = re.match(r"[\(\s]*([A-Z])[\)\.\s]*", text, flags=re.IGNORECASE) - if match: - return match.group(1).upper() - return "" - def erqa_doc_to_text(doc: dict) -> str: return doc["question"] @@ -42,6 +18,12 @@ def erqa_doc_to_visual(doc: dict) -> list: return image_list +def erqa_doc_to_messages(doc: dict, lmms_eval_specific_kwargs=None) -> list[dict]: + content = [{"type": "image", "url": image} for image in erqa_doc_to_visual(doc)] + content.append({"type": "text", "text": erqa_doc_to_text(doc)}) + return [{"role": "user", "content": content}] + + def erqa_process_results(doc, results): key_name = "erqa_acc" # extract grounded answer @@ -49,7 +31,7 @@ def erqa_process_results(doc, results): response = results[0] # extract predicted answer - pred_letter = _extract_answer_letter(response) + pred_letter = extract_mcq_answer(response, choices=["A", "B", "C", "D"]) flag = pred_letter == grounded_output omnispatial_submission = {"id": doc["question_id"], "gt_content": grounded_output, "pred": response, "sub_task": doc["question_type"], "is_correct": flag} diff --git a/lmms_eval/tasks/mmsi_video/mmsi_video_single_input.yaml b/lmms_eval/tasks/mmsi_video/mmsi_video_single_input.yaml new file mode 100644 index 000000000..a377dbdda --- /dev/null +++ b/lmms_eval/tasks/mmsi_video/mmsi_video_single_input.yaml @@ -0,0 +1,16 @@ +# MMSI-Video-Bench: single-visual-input subset (native video). +# Drops ref_images / Cross-Video samples via process_docs so every question has +# exactly one video (773/1106 samples). This matches Cambrian-P's reported +# MMSI-Video protocol (a single video input per sample). The base +# `mmsi_video` task is left unfiltered (the full benchmark definition). +task: "mmsi_video_single_input" +process_docs: !function utils.process_docs_single_input_only +doc_to_visual: !function utils.doc_to_visual_video +doc_to_text: !function utils.doc_to_text_video +doc_to_messages: !function utils.doc_to_messages_video +lmms_eval_specific_kwargs: + default: + post_prompt: "" + gemini_api: + pre_prompt: "" +include: _default_template_yaml diff --git a/lmms_eval/tasks/mmsi_video/utils.py b/lmms_eval/tasks/mmsi_video/utils.py index c5eb39ea5..784de1052 100644 --- a/lmms_eval/tasks/mmsi_video/utils.py +++ b/lmms_eval/tasks/mmsi_video/utils.py @@ -1,13 +1,10 @@ # MMSI-Video-Bench: A Holistic Benchmark for Video-Based Spatial Intelligence # https://huggingface.co/datasets/rbler/MMSI-Video-Bench -import json import os import re -from collections import defaultdict from pathlib import Path -import datasets import yaml from loguru import logger as eval_logger from PIL import Image @@ -152,6 +149,21 @@ def is_nan_or_none(value) -> bool: return False +################## +# Dataset filtering (process_docs) +################## +def process_docs_single_input_only(dataset): + """Restrict to the single-visual-input subset used by Cambrian-P's reported MMSI-Video protocol. + + Drops samples that carry reference images or are "Cross-Video" type, leaving only + samples with a single video input. + + ``ref_images`` is a (possibly empty) list/ndarray per sample; the ``or []`` guard also + covers a None value defensively. + """ + return dataset.filter(lambda x: len(x.get("ref_images", []) or []) == 0 and "Cross-Video" not in x["type"]) + + ################## # doc_to_visual functions ################## @@ -213,7 +225,7 @@ def doc_to_visual_frames(doc, lmms_eval_specific_kwargs=None): # Load sampled frames as PIL Images pil_images = [] eval_logger.debug(f"[doc_to_visual_frames] sampled_frames_list has {len(sampled_frames_list)} segments") - for seg_idx, frames in enumerate(sampled_frames_list): + for frames in sampled_frames_list: for frame_path in frames: full_path = os.path.join(cache_dir, "frames", frame_path) if not os.path.exists(full_path): diff --git a/lmms_eval/tasks/tomato/utils.py b/lmms_eval/tasks/tomato/utils.py index 007f80660..96ba28326 100644 --- a/lmms_eval/tasks/tomato/utils.py +++ b/lmms_eval/tasks/tomato/utils.py @@ -10,7 +10,6 @@ import numpy as np import yaml from loguru import logger as eval_logger -from openai import AzureOpenAI, OpenAI random.seed(42) @@ -19,11 +18,15 @@ SYSTEM_PROMPT = "You are an expert in understanding dynamics of objects." -NUM_FRAMES = 16 MAX_ITER = 5 USE_GPT_PARSER = False # whether to use gpt parser from TOMATO's source code, else use lmms_eval parser if USE_GPT_PARSER: + from openai import ( # lazy: only needed for the (disabled) GPT parser + AzureOpenAI, + OpenAI, + ) + eval_logger.info(f"Using GPT parser for TOMATO task. The max iteration is set to {MAX_ITER}. " "If the response is not a valid answer, it will try to use GPT to parse the response.") API_TYPE = os.getenv("API_TYPE", "azure") if API_TYPE == "openai": @@ -48,7 +51,8 @@ else: raise ValueError(f"Unsupported API_TYPE: {API_TYPE}. Please set it to 'openai' or 'azure'.") -eval_logger.info(f"Using {NUM_FRAMES} frames for TOMATO task. Please set the max_num_frames=16 in model_args for the result reported in the TOMATO paper: https://arxiv.org/pdf/2410.23266.") +# Frame count is governed entirely by the model's num_frames (e.g. via +# --model_args num_frames=N); the task does not impose its own frame budget. """ @@ -69,7 +73,7 @@ with open(Path(__file__).parent / "tomato.yaml", "r") as f: raw_data = f.readlines() safe_data = [] - for i, line in enumerate(raw_data): + for line in raw_data: # remove function definition since yaml load cannot handle it if "!function" not in line: safe_data.append(line) @@ -79,12 +83,11 @@ cache_dir = os.path.join(hf_home, config["dataset_kwargs"]["cache_dir"]) -def construct_prompt(question: str, options: list, num_frames: int) -> Tuple: +def construct_prompt(question: str, options: list) -> Tuple: """ Args: question (str): question in the dataset options (list): list of options - num_frames (int): number of frames extracted from the video Returns: prompt (str): well-constructed prompt @@ -95,7 +98,7 @@ def construct_prompt(question: str, options: list, num_frames: int) -> Tuple: all_choices = [f"{chr(65 + i)}" for i in range(len(options))] index2ans = {all_choices[i]: options[i] for i in range(len(options))} - prompt = f"""You will be provided with {num_frames} separate frames uniformly sampled from a video, the frames are provided in chronological order of the video. Analyze these frames and provide the answer to the question about the video content. Answer the multiple-choice question about the video content. + prompt = f"""You will be provided with frames uniformly sampled from a video, the frames are provided in chronological order of the video. Analyze these frames and provide the answer to the question about the video content. Answer the multiple-choice question about the video content. You must use these frames to answer the multiple-choice question; do not rely on any externel knowledge or commonsense. @@ -173,7 +176,7 @@ def tomato_doc_to_text(doc, lmms_eval_specific_kwargs=None): """ Process the document to a prompt for video + audio inputs """ - prompt, all_choices, index2ans = construct_prompt(question=doc["question"], options=doc["options"], num_frames=NUM_FRAMES) + prompt, all_choices, index2ans = construct_prompt(question=doc["question"], options=doc["options"]) return prompt @@ -300,7 +303,7 @@ def tomato_process_results(doc, results): a dictionary with key: metric name (in this case av_odyssey score), value: metric value """ respone = results[0] - _, all_choices, index2ans = construct_prompt(question=doc["question"], options=doc["options"], num_frames=NUM_FRAMES) + _, all_choices, index2ans = construct_prompt(question=doc["question"], options=doc["options"]) optionized_list = [f"{chr(65 + i)}. {option}" for i, option in enumerate(doc["options"])] gt = optionized_list[doc["answer"]] if USE_GPT_PARSER: @@ -340,7 +343,6 @@ def tomato_aggregate_results(results): num_corrects = 0 num_total = 0 for result in results: - question_id = result["question_id"] score = result["score"] reason_type = result["reason_type"] demonstration_type = result["demonstration_type"] diff --git a/lmms_eval/tasks/videoevalpro/utils.py b/lmms_eval/tasks/videoevalpro/utils.py index 160b5bd2d..27c3ebfe2 100644 --- a/lmms_eval/tasks/videoevalpro/utils.py +++ b/lmms_eval/tasks/videoevalpro/utils.py @@ -122,11 +122,13 @@ def videoevalpro_aggregate_results(results): for result in results: task_type = result["task_type"] + if task_type not in category2score: + category2score[task_type] = {"correct": 0, "answered": 0} category2score[task_type]["answered"] += 1 - category2score[task_type]["correct"] += result["judge_result"] == True + category2score[task_type]["correct"] += result["judge_result"] is True # overall score category2score["overall"]["answered"] += 1 - category2score["overall"]["correct"] += result["judge_result"] == True + category2score["overall"]["correct"] += result["judge_result"] is True final_scores = {} for task_type, score in category2score.items(): diff --git a/lmms_eval/tasks/vsibench/utils.py b/lmms_eval/tasks/vsibench/utils.py index d73ca5127..330cc3b05 100644 --- a/lmms_eval/tasks/vsibench/utils.py +++ b/lmms_eval/tasks/vsibench/utils.py @@ -1,5 +1,6 @@ import os -from functools import partial +import re +from functools import partial # noqa: F401 (used inside eval()'d MRA metric strings) from pathlib import Path import datasets @@ -8,6 +9,40 @@ import yaml from loguru import logger as eval_logger +_NUMBER_WORDS = { + "zero": 0, + "one": 1, + "two": 2, + "three": 3, + "four": 4, + "five": 5, + "six": 6, + "seven": 7, + "eight": 8, + "nine": 9, + "ten": 10, + "eleven": 11, + "twelve": 12, + "thirteen": 13, + "fourteen": 14, + "fifteen": 15, + "sixteen": 16, + "seventeen": 17, + "eighteen": 18, + "nineteen": 19, + "twenty": 20, + "thirty": 30, + "forty": 40, + "fifty": 50, + "sixty": 60, + "seventy": 70, + "eighty": 80, + "ninety": 90, + "hundred": 100, + "thousand": 1000, +} +_NUM_RE = re.compile(r"-?\d+(?:\.\d+)?") + MCA_QUESTION_TYPES = [ "object_rel_direction_easy", "object_rel_direction_medium", @@ -37,7 +72,7 @@ with open(Path(__file__).parent / "vsibench.yaml", "r") as f: raw_data = f.readlines() safe_data = [] - for i, line in enumerate(raw_data): + for line in raw_data: if "!function" not in line: safe_data.append(line) cache_name = yaml.safe_load("".join(safe_data))["dataset_kwargs"]["cache_dir"] @@ -110,6 +145,65 @@ def to_float(pred): return pred +def extract_number(text): + """Pull the first numeric value out of a model response. + + Handles bare digits ("3", "3.0"), digits embedded in a sentence + ("There are 3 boxes"), and English number words ("three", "Three boxes.", + "twenty-one", "three hundred and twenty-one"). Returns None if nothing + numeric is found. + + Ordering matters: whichever number token (digit or word) appears first + in the string wins. This avoids mis-scoring responses like + "one apple, room 2" — the intended count is "one" (1), not the later + digit "2". + """ + if text is None: + return None + s = str(text).strip().lower() + if not s: + return None + + digit_match = _NUM_RE.search(s) + word_run_start = None + word_run_tokens = [] + for m in re.finditer(r"[a-z]+", s): + tok = m.group(0) + if word_run_start is None: + if tok in _NUMBER_WORDS: + word_run_start = m.start() + word_run_tokens.append(tok) + else: + if tok == "and": + continue + if tok in _NUMBER_WORDS: + word_run_tokens.append(tok) + else: + break + + digit_first = digit_match is not None and (word_run_start is None or digit_match.start() < word_run_start) + if digit_first: + try: + return float(digit_match.group(0)) + except ValueError: + pass + + if word_run_start is None: + return None + + total, current = 0, 0 + for tok in word_run_tokens: + val = _NUMBER_WORDS[tok] + if val == 100: + current = max(current, 1) * 100 + elif val == 1000: + total += max(current, 1) * 1000 + current = 0 + else: + current += val + return float(total + current) + + def vsibench_process_results(doc, results): doc["prediction"] = results[0] if doc["question_type"] in MCA_QUESTION_TYPES: @@ -118,7 +212,7 @@ def vsibench_process_results(doc, results): elif doc["question_type"] in NA_QUESTION_TYPES: for key, value in METRICS_FOR_NA.items(): try: - doc[key] = eval(value)(to_float(fuzzy_matching(doc["prediction"])), to_float(doc["ground_truth"])) + doc[key] = eval(value)(extract_number(doc["prediction"]), to_float(doc["ground_truth"])) except TypeError: doc[key] = WORST_CASE_FOR_METRICS[key] else: diff --git a/lmms_eval/tasks/vsisuper/count_streaming/vsc_streaming_utils.py b/lmms_eval/tasks/vsisuper/count_streaming/vsc_streaming_utils.py index e7d7fa038..48a6ddd4a 100644 --- a/lmms_eval/tasks/vsisuper/count_streaming/vsc_streaming_utils.py +++ b/lmms_eval/tasks/vsisuper/count_streaming/vsc_streaming_utils.py @@ -36,9 +36,13 @@ def process_docs_streaming_120mins(dataset: datasets.Dataset) -> datasets.Datase def _abs_dist_norm(pred, target): try: - return abs(pred - target) / target + pred = float(pred) + target = float(target) + if target == 0.0: + return 0.0 if pred == 0.0 else float("inf") + return abs(pred - target) / abs(target) except BaseException: - return 0.0 + return float("inf") def _mean_relative_accuracy(pred, target, start=0.5, end=0.95, interval=0.05): @@ -49,7 +53,11 @@ def _mean_relative_accuracy(pred, target, start=0.5, end=0.95, interval=0.05): def _parse_prediction_list(text): - return json.loads(text) + try: + parsed = json.loads(text) + except (TypeError, json.JSONDecodeError): + return [] + return parsed if isinstance(parsed, list) else [] def process_results(doc, results):