From 2a6543750185af72d5a8bff2e0b9fc1a30f0c3a9 Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Wed, 24 Jun 2026 23:36:52 +0300 Subject: [PATCH 1/6] feat: add PolyMath multilingual math eval (EU languages) for lm-eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the Qwen/PolyMath benchmark as a generative lm-eval-harness task, scoped to EU languages per the dataset's language configs: de, en, es, fr, it, pt. PolyMath ships one config per language and four difficulty splits per config (low/medium/high/top), so this adds one task per (language, difficulty) — 24 tasks — each tagged `polymath` and `polymath_` for aggregate reporting. Approach (mirrors lm-eval's minerva_math, but self-contained): - Prompt asks the model to reason and box its final answer in \boxed{}. - A custom process_results extracts the last \boxed{} span (falling back to the last number), normalises gold + prediction with the Minerva/Lewkowycz string normalisation, and scores exact_match with a light sympy numeric fallback (e.g. 18 == 18.0, 1/2 == 0.5). Deliberately avoids math_verify / parse_latex, which are unavailable in this venv (no math_verify; antlr4 is 4.13 not 4.11). Registry: - New `polymath-eu` task group (0-shot) with a {lang} valid_langs template, so `polymath-eu[deu_Latn]` etc. language brackets work out of the box. - task_metrics entries (exact_match) for all 24 tasks; README group list updated. Validated: lm-eval discovers all 24 tasks + tags (template not leaked), YAML include + !function resolution works, the task instantiates against the real `de` config (125 docs/split, fields id/question/answer), and process_results scores correct/incorrect generations as expected. Co-Authored-By: Claude Opus 4.8 --- README.md | 1 + .../polymath/_default_template_yaml | 21 ++ .../polymath/polymath_de_high.yaml | 7 + .../polymath/polymath_de_low.yaml | 7 + .../polymath/polymath_de_medium.yaml | 7 + .../polymath/polymath_de_top.yaml | 7 + .../polymath/polymath_en_high.yaml | 7 + .../polymath/polymath_en_low.yaml | 7 + .../polymath/polymath_en_medium.yaml | 7 + .../polymath/polymath_en_top.yaml | 7 + .../polymath/polymath_es_high.yaml | 7 + .../polymath/polymath_es_low.yaml | 7 + .../polymath/polymath_es_medium.yaml | 7 + .../polymath/polymath_es_top.yaml | 7 + .../polymath/polymath_fr_high.yaml | 7 + .../polymath/polymath_fr_low.yaml | 7 + .../polymath/polymath_fr_medium.yaml | 7 + .../polymath/polymath_fr_top.yaml | 7 + .../polymath/polymath_it_high.yaml | 7 + .../polymath/polymath_it_low.yaml | 7 + .../polymath/polymath_it_medium.yaml | 7 + .../polymath/polymath_it_top.yaml | 7 + .../polymath/polymath_pt_high.yaml | 7 + .../polymath/polymath_pt_low.yaml | 7 + .../polymath/polymath_pt_medium.yaml | 7 + .../polymath/polymath_pt_top.yaml | 7 + .../custom_lm_eval_tasks/polymath/utils.py | 235 ++++++++++++++++++ oellm/resources/task-groups.yaml | 40 +++ 28 files changed, 465 insertions(+) create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/_default_template_yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_high.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_medium.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_top.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/utils.py diff --git a/README.md b/README.md index 34f7efb..2f54c4f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ Available task groups: - `flores-200-eu-to-eng` / `flores-200-eng-to-eu` - Translation tasks - `global-mmlu-eu` - Global MMLU in EU languages - `mgsm-eu` - Multilingual GSM benchmarks +- `polymath-eu` - PolyMath multilingual math reasoning (EU languages, 0-shot, `\boxed{}` answer extraction) - `generic-multilingual` - XWinograd, XCOPA, XStoryCloze - `include` - INCLUDE benchmarks diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/_default_template_yaml b/oellm/resources/custom_lm_eval_tasks/polymath/_default_template_yaml new file mode 100644 index 0000000..f66759e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/_default_template_yaml @@ -0,0 +1,21 @@ +dataset_path: Qwen/PolyMath +output_type: generate_until +training_split: null +validation_split: null +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +process_results: !function utils.process_results +generation_kwargs: + until: + - "Problem:" + max_gen_toks: 1024 + do_sample: false + temperature: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_high.yaml new file mode 100644 index 0000000..7a8074e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_de_high +dataset_name: de +test_split: high +tag: + - polymath + - polymath_de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_low.yaml new file mode 100644 index 0000000..c105296 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_de_low +dataset_name: de +test_split: low +tag: + - polymath + - polymath_de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_medium.yaml new file mode 100644 index 0000000..2b98503 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_de_medium +dataset_name: de +test_split: medium +tag: + - polymath + - polymath_de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_top.yaml new file mode 100644 index 0000000..8f06f66 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_de_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_de_top +dataset_name: de +test_split: top +tag: + - polymath + - polymath_de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_high.yaml new file mode 100644 index 0000000..9f7fe00 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_en_high +dataset_name: en +test_split: high +tag: + - polymath + - polymath_en diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_low.yaml new file mode 100644 index 0000000..57e106b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_en_low +dataset_name: en +test_split: low +tag: + - polymath + - polymath_en diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_medium.yaml new file mode 100644 index 0000000..56646a9 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_en_medium +dataset_name: en +test_split: medium +tag: + - polymath + - polymath_en diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_top.yaml new file mode 100644 index 0000000..2bfefe0 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_en_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_en_top +dataset_name: en +test_split: top +tag: + - polymath + - polymath_en diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_high.yaml new file mode 100644 index 0000000..8434b62 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_es_high +dataset_name: es +test_split: high +tag: + - polymath + - polymath_es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_low.yaml new file mode 100644 index 0000000..d5d07c1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_es_low +dataset_name: es +test_split: low +tag: + - polymath + - polymath_es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_medium.yaml new file mode 100644 index 0000000..ad00f59 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_es_medium +dataset_name: es +test_split: medium +tag: + - polymath + - polymath_es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_top.yaml new file mode 100644 index 0000000..5d886ae --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_es_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_es_top +dataset_name: es +test_split: top +tag: + - polymath + - polymath_es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_high.yaml new file mode 100644 index 0000000..435390e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_fr_high +dataset_name: fr +test_split: high +tag: + - polymath + - polymath_fr diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_low.yaml new file mode 100644 index 0000000..956797b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_fr_low +dataset_name: fr +test_split: low +tag: + - polymath + - polymath_fr diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_medium.yaml new file mode 100644 index 0000000..7d60283 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_fr_medium +dataset_name: fr +test_split: medium +tag: + - polymath + - polymath_fr diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_top.yaml new file mode 100644 index 0000000..e200870 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_fr_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_fr_top +dataset_name: fr +test_split: top +tag: + - polymath + - polymath_fr diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_high.yaml new file mode 100644 index 0000000..b50ea06 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_it_high +dataset_name: it +test_split: high +tag: + - polymath + - polymath_it diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_low.yaml new file mode 100644 index 0000000..d287b56 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_it_low +dataset_name: it +test_split: low +tag: + - polymath + - polymath_it diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_medium.yaml new file mode 100644 index 0000000..5df3f7a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_it_medium +dataset_name: it +test_split: medium +tag: + - polymath + - polymath_it diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_top.yaml new file mode 100644 index 0000000..dd87933 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_it_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_it_top +dataset_name: it +test_split: top +tag: + - polymath + - polymath_it diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_high.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_high.yaml new file mode 100644 index 0000000..792e258 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_high.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_pt_high +dataset_name: pt +test_split: high +tag: + - polymath + - polymath_pt diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_low.yaml new file mode 100644 index 0000000..a5dd96f --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_low.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_pt_low +dataset_name: pt +test_split: low +tag: + - polymath + - polymath_pt diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_medium.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_medium.yaml new file mode 100644 index 0000000..09fe8b4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_medium.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_pt_medium +dataset_name: pt +test_split: medium +tag: + - polymath + - polymath_pt diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_top.yaml b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_top.yaml new file mode 100644 index 0000000..d23461e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_pt_top.yaml @@ -0,0 +1,7 @@ +include: _default_template_yaml +task: polymath_pt_top +dataset_name: pt +test_split: top +tag: + - polymath + - polymath_pt diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/utils.py b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py new file mode 100644 index 0000000..52d3da8 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py @@ -0,0 +1,235 @@ +"""Helpers for the multilingual PolyMath (Qwen/PolyMath) generative math task. + +The model is prompted to reason and wrap its final answer in ``\\boxed{...}``. +We extract that boxed span, normalise it (and the gold answer) with the +Minerva/Lewkowycz string normalisation, and score with exact match. A light +sympy numeric fallback catches equivalent-but-differently-written numbers +(e.g. ``18`` vs ``18.0``, ``1/2`` vs ``0.5``). + +Note: unlike ``lm_eval``'s ``minerva_math`` we deliberately avoid +``math_verify`` / ``sympy.parsing.latex.parse_latex`` here — ``math_verify`` is +not installed in this project's venv and ``parse_latex`` requires +``antlr4-python3-runtime==4.11`` (the venv ships 4.13), so importing them would +break task loading. Everything below is self-contained. +""" + +import re +import signal +from typing import Optional + + +try: + from sympy import simplify, sympify + + _HAS_SYMPY = True +except ImportError: # pragma: no cover - sympy is a transitive lm-eval dep + _HAS_SYMPY = False + + +# --- Prompt ----------------------------------------------------------------- +def doc_to_text(doc: dict) -> str: + return ( + "Solve the following math problem step by step. " + "Put your final answer inside \\boxed{}.\n\n" + "Problem:\n" + doc["question"] + "\n\nSolution:" + ) + + +def doc_to_target(doc: dict) -> str: + return doc["answer"] + + +# --- \boxed{} extraction (from lm_eval/tasks/minerva_math/utils.py) --------- +def last_boxed_only_string(string: str) -> Optional[str]: + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + return None + return string[idx : right_brace_idx + 1] + + +def remove_boxed(s: str) -> str: + if "\\boxed " in s: + left = "\\boxed " + assert s[: len(left)] == left + return s[len(left) :] + + left = "\\boxed{" + assert s[: len(left)] == left + assert s[-1] == "}" + return s[len(left) : -1] + + +def _extract_answer(text: str) -> str: + """Pull the final answer out of a model generation. + + Prefer the last ``\\boxed{...}`` span; otherwise fall back to the last + number that appears in the text; otherwise return the stripped text. + """ + boxed = last_boxed_only_string(text) + if boxed is not None: + try: + return remove_boxed(boxed) + except AssertionError: + pass + numbers = re.findall(r"-?\d[\d,]*\.?\d*", text) + if numbers: + return numbers[-1] + return text.strip() + + +# --- Normalisation (from Lewkowycz et al. 2022 appendix D, via minerva) ----- +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "ft", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """Normalise a final answer to a quantitative reasoning question.""" + final_answer = final_answer.split("=")[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract answer that is in LaTeX math, is bold, is boxed, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalise shorthand TeX (\fracab -> \frac{a}{b}, \sqrta -> \sqrt{a}). + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalise 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer.strip() + + +# --- Equivalence ------------------------------------------------------------ +class _timeout: + def __init__(self, seconds: int = 5): + self.seconds = seconds + + def handle_timeout(self, signum, frame): + raise TimeoutError + + def __enter__(self): + # SIGALRM is only settable from the main thread; degrade to no timeout + # (sympify/simplify are fast for the numeric cases we rely on) otherwise. + self._armed = False + try: + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + self._armed = True + except ValueError: + pass + + def __exit__(self, exc_type, exc_value, traceback): + if self._armed: + signal.alarm(0) + + +def _sympy_numeric_equiv(pred: str, gold: str) -> bool: + """Best-effort numeric equivalence without LaTeX parsing. + + Handles plain numbers and simple arithmetic (``1/2`` == ``0.5``). LaTeX + expressions that ``sympify`` cannot parse simply return ``False`` and the + caller falls back to normalised string equality. + """ + if not _HAS_SYMPY: + return False + try: + with _timeout(5): + a = sympify(pred.replace(",", "")) + b = sympify(gold.replace(",", "")) + return bool(simplify(a - b) == 0) + except Exception: # noqa: BLE001 - sympify/simplify raise many types incl. TimeoutError + return False + + +def process_results(doc: dict, results: list[str]) -> dict[str, int]: + candidate = results[0] + pred = normalize_final_answer(_extract_answer(candidate)) + gold = normalize_final_answer(doc["answer"]) + + correct = pred == gold or _sympy_numeric_equiv(pred, gold) + return {"exact_match": int(correct)} diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index a8a7fbc..2d89681 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -169,6 +169,30 @@ task_metrics: global_piqa_prompted_swe_latn: exact_match global_piqa_prompted_tur_latn: exact_match global_piqa_prompted_ukr_cyrl: exact_match + polymath_de_low: exact_match + polymath_de_medium: exact_match + polymath_de_high: exact_match + polymath_de_top: exact_match + polymath_en_low: exact_match + polymath_en_medium: exact_match + polymath_en_high: exact_match + polymath_en_top: exact_match + polymath_es_low: exact_match + polymath_es_medium: exact_match + polymath_es_high: exact_match + polymath_es_top: exact_match + polymath_fr_low: exact_match + polymath_fr_medium: exact_match + polymath_fr_high: exact_match + polymath_fr_top: exact_match + polymath_it_low: exact_match + polymath_it_medium: exact_match + polymath_it_high: exact_match + polymath_it_top: exact_match + polymath_pt_low: exact_match + polymath_pt_medium: exact_match + polymath_pt_high: exact_match + polymath_pt_top: exact_match task_groups: sib200-eu: @@ -508,6 +532,22 @@ task_groups: - task: "global_piqa_prompted_{lang}" subset: "{lang}" + polymath-eu: + description: "PolyMath multilingual math reasoning — EU languages (0-shot, generative; final answer extracted from \\boxed{} and scored with exact_match). One task per language per difficulty split (low/medium/high/top)." + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + valid_langs: [de, en, es, fr, it, pt] + tasks: + - task: "polymath_{lang}_low" + subset: "{lang}" + - task: "polymath_{lang}_medium" + subset: "{lang}" + - task: "polymath_{lang}_high" + subset: "{lang}" + - task: "polymath_{lang}_top" + subset: "{lang}" + super_groups: oellm-multilingual: description: "Combined Belebele EU set plus multilingual benchmarks" From 1d835e6d5b4eae7ac3b9d4e0676c266c27136f94 Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Wed, 24 Jun 2026 23:39:35 +0300 Subject: [PATCH 2/6] refactor: split polymath-eu into one task group per difficulty tier Replace the single polymath-eu group with four per-tier groups (polymath-eu-low / -medium / -high / -top) so a difficulty tier can be run in isolation. The 24 per-(language, difficulty) task YAMLs and their task_metrics entries are unchanged. 'top' is PolyMath's hardest tier (Olympiad-level). Co-Authored-By: Claude Opus 4.8 --- README.md | 2 +- oellm/resources/task-groups.yaml | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2f54c4f..5f38a56 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Available task groups: - `flores-200-eu-to-eng` / `flores-200-eng-to-eu` - Translation tasks - `global-mmlu-eu` - Global MMLU in EU languages - `mgsm-eu` - Multilingual GSM benchmarks -- `polymath-eu` - PolyMath multilingual math reasoning (EU languages, 0-shot, `\boxed{}` answer extraction) +- `polymath-eu-low` / `polymath-eu-medium` / `polymath-eu-high` / `polymath-eu-top` - PolyMath multilingual math reasoning (EU languages, 0-shot, `\boxed{}` answer extraction), one group per difficulty tier - `generic-multilingual` - XWinograd, XCOPA, XStoryCloze - `include` - INCLUDE benchmarks diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 2d89681..ef62de6 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -532,8 +532,12 @@ task_groups: - task: "global_piqa_prompted_{lang}" subset: "{lang}" - polymath-eu: - description: "PolyMath multilingual math reasoning — EU languages (0-shot, generative; final answer extracted from \\boxed{} and scored with exact_match). One task per language per difficulty split (low/medium/high/top)." + # PolyMath ships 4 expert-calibrated difficulty tiers (low/medium/high/top, + # from grade-school to Olympiad-level), each its own group so a tier can be run + # in isolation. EU languages only (de/en/es/fr/it/pt); 0-shot generative, with + # the final answer extracted from \boxed{} and scored with exact_match. + polymath-eu-low: + description: "PolyMath EU-language math reasoning — low difficulty tier (0-shot, \\boxed{} answer extraction)." suite: lm-eval-harness n_shots: [0] dataset: Qwen/PolyMath @@ -541,10 +545,34 @@ task_groups: tasks: - task: "polymath_{lang}_low" subset: "{lang}" + + polymath-eu-medium: + description: "PolyMath EU-language math reasoning — medium difficulty tier (0-shot, \\boxed{} answer extraction)." + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + valid_langs: [de, en, es, fr, it, pt] + tasks: - task: "polymath_{lang}_medium" subset: "{lang}" + + polymath-eu-high: + description: "PolyMath EU-language math reasoning — high difficulty tier (0-shot, \\boxed{} answer extraction)." + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + valid_langs: [de, en, es, fr, it, pt] + tasks: - task: "polymath_{lang}_high" subset: "{lang}" + + polymath-eu-top: + description: "PolyMath EU-language math reasoning — top (Olympiad-level) difficulty tier (0-shot, \\boxed{} answer extraction)." + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + valid_langs: [de, en, es, fr, it, pt] + tasks: - task: "polymath_{lang}_top" subset: "{lang}" From 6e81468a5579b3309038d8073cca8a28bb335984 Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Thu, 25 Jun 2026 03:59:38 +0300 Subject: [PATCH 3/6] fix: use --batch_size auto in eval sbatch Pass --batch_size auto to lm_eval so batch size is tuned per task instead of defaulting to 1. Co-Authored-By: Claude Opus 4.8 --- oellm/resources/template.sbatch | 1 + 1 file changed, 1 insertion(+) diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index b71fb17..83465c5 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -142,6 +142,7 @@ do --num_fewshot "$n_shot" \ --output_path "{evals_dir}/$(openssl rand -hex 5).json" \ --trust_remote_code \ + --batch_size auto \ ${{LM_EVAL_INCLUDE_PATH:+--include_path $LM_EVAL_INCLUDE_PATH}} \ ${{LIMIT:+--limit $LIMIT}} echo "----------------------------------------------------" From 83b3cb0ce85b9be2bcb17bee77153f2b37bc96ae Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Thu, 25 Jun 2026 05:13:53 +0300 Subject: [PATCH 4/6] fix(polymath): score with the official Qwen/PolyMath math_equal judge Replace the bespoke Minerva-normalisation + narrow sympy exact_match with the upstream PolyMath equivalence judge so our scores reproduce the benchmark instead of being a strict lower bound. - Add polymath_eval.py: the official `math_equal` (numeric isclose rel_tol=1e-4, percentage variants, LaTeX/symbolic + matrix/list/equation matching), vendored verbatim from QwenLM/PolyMath eval/scripts.py. Two justified deviations: * latex2sympy comes from latex2sympy2_extended (already in the venv via lighteval). Classic latex2sympy2 pins antlr 4.7.2 and would break lighteval; the extended fork exposes the same `latex2sympy` and only feeds the symbolic_equal fallback, so the decision logic is unchanged. * call_with_timeout runs the symbolic check in a daemon thread, not a forked process: forking corrupts latex2sympy2_extended's parser state here, which silently scored LaTeX answers wrong. A one-time warm-up at import avoids a cold-start timeout on the first comparison. Verified deterministic (50x). - utils.py: extract the answer with run_eval.py's `extract_boxed_content` (verbatim) and score with `math_equal`, mirroring upstream run_eval.py. Drops the old last-number fallback (upstream uses boxed-only). - pyproject.toml: exclude the vendored scorer from ruff to keep it verbatim. Validated end-to-end through lm-eval TaskManager: polymath_de_low loads, 125 docs, and process_results scores boxed gold/fraction/percentage/wrong correctly. Co-Authored-By: Claude Opus 4.8 --- .../polymath/polymath_eval.py | 856 ++++++++++++++++++ .../custom_lm_eval_tasks/polymath/utils.py | 258 +----- pyproject.toml | 3 + 3 files changed, 907 insertions(+), 210 deletions(-) create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py new file mode 100644 index 0000000..bf51bdf --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py @@ -0,0 +1,856 @@ +"""Vendored answer-equivalence scorer for the PolyMath benchmark. + +This is the official Qwen/PolyMath ``math_equal`` judge, copied **verbatim** +from https://github.com/QwenLM/PolyMath/blob/main/eval/scripts.py (the same +function ``eval/run_eval.py`` uses to score answers), so our exact_match +metric reproduces the upstream benchmark. + +Two deviations from upstream, both forced/justified: + 1. ``from bundled.latex2sympy2.latex2sympy2 import latex2sympy`` -> + ``from latex2sympy2_extended import latex2sympy``. The classic + ``latex2sympy2`` pins ``antlr4-python3-runtime==4.7.2``, which would + downgrade antlr and break ``latex2sympy2_extended``/``lighteval`` already + in this venv. The extended fork exposes the same ``latex2sympy`` and only + serves as the LaTeX->sympy parser in ``symbolic_equal``; the equivalence + decision logic below is unchanged. + 2. ``call_with_timeout`` runs the symbolic check in a daemon thread instead of + a forked process (forking breaks ``latex2sympy2_extended`` here; see below). + +The unused ``multi_math_equal`` / ``AdaptedEvaluatorMath`` tail (which needed +the ``symeval`` package, not on PyPI) is dropped; ``math_equal`` does not use +it. +""" +import re +from typing import Union, Optional, Any +import regex +from latex2sympy2_extended import latex2sympy +from math import isclose +from word2number import w2n + +MULTILINGUAL_ANSWER_REGEXES = [ + "Answer\s*:", + "Answer\s*:​​​​​​", # Korean invisible character + "উত্তর\s*:", + "उत्तर\s*:", + "উত্তরঃ", + "উত্তর\s*:", + "Antwort\s*:", + "답변\s*:", + "정답\s*:", + "답\s*:", + "答案\s*：", + "答案\s*:", + "答\s*：", + "答\s*:", + "答复\s*：", + "答曰\s*：", + "الإجابة:", + "الجواب:", + "إجابة:", + "الإجابة النهائية:", + "الإجابة الصحيحة:", + "الإجابة الصحيحة هي:", + "الإجابة هي:", + "Respuesta\s*:", + "Risposta\s*:", + "答え\s*:", + "答え\s*：", + "回答\s*:", + "回答\s*：", + "解答\s*:", + "Jawaban\s*:", + "Réponse\s*:", + "Resposta\s*:", + "Jibu\s*:", + "Idahun\s*:", + "Ìdáhùn\s*:", + "Idáhùn\s*:", + "Àmọ̀nà\s*:", + "Àdáhùn\s*:", + "Ànúgọ\s*:", + "Àṣàyàn\s*:", +] + +MULTILINGUAL_ANSWER_PATTERN_TEMPLATE = ( + "(?i){}\s*([A-D]|[أ-د]|[অ]|[ব]|[ড]|[ঢ]|[Ａ]|[Ｂ]|[Ｃ]|[Ｄ])" +) + +STRIP_EXCEPTIONS = ["carp_en", "minerva_math"] + +# units mainly from MathQA +unit_texts = [ + "east", + "degree", + "mph", + "kmph", + "ft", + "m sqaure", + " m east", + "sq m", + "deg", + "mile", + "q .", + "monkey", + "prime", + "ratio", + "profit of rs", + "rd", + "o", + "gm", + "p . m", + "lb", + "tile", + "per", + "dm", + "lt", + "gain", + "ab", + "way", + "west", + "a .", + "b .", + "c .", + "d .", + "e .", + "f .", + "g .", + "h .", + "t", + "a", + "h", + "no change", + "men", + "soldier", + "pie", + "bc", + "excess", + "st", + "inches", + "noon", + "percent", + "by", + "gal", + "kmh", + "c", + "acre", + "rise", + "a . m", + "th", + "π r 2", + "sq", + "mark", + "l", + "toy", + "coin", + "sq . m", + "gallon", + "° f", + "profit", + "minw", + "yr", + "women", + "feet", + "am", + "pm", + "hr", + "cu cm", + "square", + "v â € ™", + "are", + "rupee", + "rounds", + "cubic", + "cc", + "mtr", + "s", + "ohm", + "number", + "kmph", + "day", + "hour", + "minute", + "min", + "second", + "man", + "woman", + "sec", + "cube", + "mt", + "sq inch", + "mp", + "∏ cm ³", + "hectare", + "more", + "sec", + "unit", + "cu . m", + "cm 2", + "rs .", + "rs", + "kg", + "g", + "month", + "km", + "m", + "cm", + "mm", + "apple", + "liter", + "loss", + "yard", + "pure", + "year", + "increase", + "decrease", + "d", + "less", + "Surface", + "litre", + "pi sq m", + "s .", + "metre", + "meter", + "inch", +] + +unit_texts.extend([t + "s" for t in unit_texts]) + +def numeric_equal(prediction: float, reference: float): + # Note that relative tolerance has significant impact + # on the result of the synthesized GSM-Hard dataset + # if reference.is_integer(): + # return isclose(reference, round(prediction), abs_tol=1e-4) + # else: + # prediction = round(prediction, len(str(reference).split(".")[-1])) + return isclose(reference, prediction, rel_tol=1e-4) + +def parse_digits(num): + num = regex.sub(",", "", str(num)) + try: + return float(num) + except: + if num.endswith("%"): + num = num[:-1] + if num.endswith("\\"): + num = num[:-1] + try: + return float(num) / 100 + except: + pass + return None + +def is_digit(num): + # paired with parse_digits + return parse_digits(num) is not None + +def normalize_extracted_answer(extracted_answer: str) -> str: + return ( + # In arabic these are the letters used for A-D in multiple choice questions + extracted_answer.replace("أ", " A") + .replace("ب", " B") + .replace("ج", " C") + .replace("د", " D") + # In Bengali these are the letters used for A-D in multiple choice questions + .replace("অ", " A") + .replace("ব", " B") + .replace("ড", " C") + .replace("ঢ", " D") + # In Japanese these are the letters sometimes used for A-D in multiple choice questions + .replace("Ａ", " A") + .replace("Ｂ", " B") + .replace("Ｃ", " C") + .replace("Ｄ", " D") + .strip() + ) + +def convert_word_number(text: str) -> str: + try: + text = str(w2n.word_to_num(text)) + except: + pass + return text + +def _fix_sqrt(string): + _string = re.sub(r"\\sqrt(\w+)", r"\\sqrt{\1}", string) + return _string + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if len(substr) > 0 and substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + if "sqrt" not in a: + a = int(a) + if "sqrt" not in b: + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: + return string + +def strip_string(string, skip_unit=False): + string = str(string).strip() + # linebreaks + string = string.replace("\n", "") + + # right "." + string = string.rstrip(".") + + # remove inverse spaces + # replace \\ with \ + string = string.replace("\\!", "") + # string = string.replace("\\ ", "") + # string = string.replace("\\\\", "\\") + + # matrix + string = re.sub(r"\\begin\{array\}\{.*?\}", r"\\begin{pmatrix}", string) + string = re.sub(r"\\end\{array\}", r"\\end{pmatrix}", string) + string = string.replace("bmatrix", "pmatrix") + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + string = ( + string.replace("\\neq", "\\ne") + .replace("\\leq", "\\le") + .replace("\\geq", "\\ge") + ) + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + string = string.replace("\\{", "{") + string = string.replace("\\}", "}") + + # Remove unit: miles, dollars if after is not none + _string = re.sub(r"\\text{.*?}$", "", string).strip() + if _string != "" and _string != string: + # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) + string = _string + + if not skip_unit: + # Remove unit: texts + for _ in range(2): + for unit_text in unit_texts: + # use regex, the prefix should be either the start of the string or a non-alphanumeric character + # the suffix should be either the end of the string or a non-alphanumeric character + _string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string) + if _string != "": + string = _string + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + string = string.replace("$", "") + string = string.replace("\\(", "").replace("\\)", "") + + # convert word number to digit + string = convert_word_number(string) + + # replace "\\text{...}" to "..." + string = re.sub(r"\\text\{(.*?)\}", r"\1", string) + for key in ["x=", "y=", "z=", "x\\in", "y\\in", "z\\in", "x\\to", "y\\to", "z\\to"]: + string = string.replace(key, "") + string = string.replace("\\emptyset", r"{}") + string = string.replace("(-\\infty,\\infty)", "\\mathbb{R}") + + # remove percentage + string = string.replace("\\%", "") + string = string.replace("\%", "") + string = string.replace("%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + + # cdot + # string = string.replace("\\cdot", "") + if ( + string.startswith("{") + and string.endswith("}") + and string.isalnum() + or string.startswith("(") + and string.endswith(")") + and string.isalnum() + or string.startswith("[") + and string.endswith("]") + and string.isalnum() + ): + string = string[1:-1] + + # inf + string = string.replace("infinity", "\\infty") + if "\\infty" not in string: + string = string.replace("inf", "\\infty") + string = string.replace("+\\inity", "\\infty") + + # and + string = string.replace("and", "") + string = string.replace("\\mathbf", "") + + # use regex to remove \mbox{...} + string = re.sub(r"\\mbox{.*?}", "", string) + + # quote + string.replace("'", "") + string.replace('"', "") + + # i, j + if "j" in string and "i" not in string: + string = string.replace("j", "i") + + # replace a.000b where b is not number or b is end, with ab, use regex + string = re.sub(r"(\d+)\.0*([^\d])", r"\1\2", string) + string = re.sub(r"(\d+)\.0*$", r"\1", string) + + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2: + if len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + string = _fix_sqrt(string) + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string + +def str_to_pmatrix(input_str): + input_str = input_str.strip() + matrix_str = re.findall(r"\{.*,.*}", input_str) + pmatrix_list = [] + + for m in matrix_str: + m = m.strip("{}") + pmatrix = r"\begin{pmatrix}" + m.replace(",", "\\") + r"\end{pmatrix}" + pmatrix_list.append(pmatrix) + + return ", ".join(pmatrix_list) + +def choice_answer_clean(pred: str): + pred = pred.strip("\n").rstrip(".").rstrip("/").strip(" ").lstrip(":") + # Clean the answer based on the dataset + tmp = re.findall(r"\b([A-E])\b", pred.upper()) + if tmp: + pred = tmp + else: + pred = [pred.strip().strip(".")] + pred = pred[-1] + # Remove the period at the end, again! + pred = pred.rstrip(".").rstrip("/") + return pred + +def extract_answer(pred_str, data_name, use_last_number=True): + pred_str = pred_str.replace("\u043a\u0438", "") + if data_name in ["mmlu_stem", "sat_math", "aqua"]: + # TODO check multiple choice + return choice_answer_clean(pred_str) + + ######### + if "final answer is $" in pred_str and "$. I hope" in pred_str: + # minerva_math + tmp = pred_str.split("final answer is $", 1)[1] + pred = tmp.split("$. I hope", 1)[0].strip() + elif "boxed" in pred_str: + ans = pred_str.split("boxed")[-1] + if len(ans) == 0: + return "" + elif ans[0] == "{": + stack = 1 + a = "" + for c in ans[1:]: + if c == "{": + stack += 1 + a += c + elif c == "}": + stack -= 1 + if stack == 0: + break + a += c + else: + a += c + else: + a = ans.split("$")[0].strip() + pred = a + elif "he answer is" in pred_str: + pred = pred_str.split("he answer is")[-1].strip() + elif "final answer is" in pred_str: + pred = pred_str.split("final answer is")[-1].strip() + # elif extract_program_output(pred_str) != "": + # fall back to program + # pred = extract_program_output(pred_str) + else: # use the last number + if use_last_number: + pattern = "-?\d*\.?\d+" + pred = re.findall(pattern, pred_str.replace(",", "")) + if len(pred) >= 1: + pred = pred[-1] + else: + pred = "" + else: + pred = "" + + # choice answer + if ( + data_name in ["sat_math", "aqua", "arc", "gpqa", "mathqa"] + or "mmlu" in data_name + ): + tmp = re.findall(r"\b(A|B|C|D|E)\b", pred.upper()) + if tmp: + pred = tmp[-1] + else: + pred = pred.strip().strip(".") + + # multiple line + # pred = pred.split("\n")[0] + pred = re.sub(r"\n\s*", "", pred) + if pred != "" and pred[0] == ":": + pred = pred[1:] + if pred != "" and pred[-1] == ".": + pred = pred[:-1] + if pred != "" and pred[-1] == "/": + pred = pred[:-1] + pred = strip_string(pred, skip_unit=data_name in ["carp_en", "minerva_math"]) + return pred + +def symbolic_equal(a, b): + from sympy import simplify, N + from sympy.parsing.latex import parse_latex + from sympy.parsing.sympy_parser import parse_expr + + def _parse(s): + for f in [parse_latex, parse_expr, latex2sympy]: + try: + return f(s.replace("\\\\", "\\")) + except: + try: + return f(s) + except: + pass + return s + + a = _parse(a) + b = _parse(b) + + # direct equal + try: + if str(a) == str(b) or a == b: + return True + except: + pass + + # simplify equal + try: + if a.equals(b) or simplify(a - b) == 0: + return True + except: + pass + + # equation equal + try: + if (abs(a.lhs - a.rhs)).equals(abs(b.lhs - b.rhs)): + return True + except: + pass + + try: + if numeric_equal(float(N(a)), float(N(b))): + return True + except: + pass + + # matrix + try: + # if a and b are matrix + if a.shape == b.shape: + _a = a.applyfunc(lambda x: round(x, 3)) + _b = b.applyfunc(lambda x: round(x, 3)) + if _a.equals(_b): + return True + except: + pass + + return False + +def symbolic_equal_process(a, b, output_queue): + result = symbolic_equal(a, b) + output_queue.put(result) +def call_with_timeout(func, *args, timeout=1, **kwargs): + # Adaptation (vs upstream): upstream runs the symbolic check in a forked + # multiprocessing.Process. Forking breaks latex2sympy2_extended's parser + # state in this venv, so LaTeX answers (e.g. \frac{1}{2}) silently score + # wrong. A daemon thread shares memory, so the parser works; it is abandoned + # on timeout (we return False), preserving upstream's "give up after + # `timeout`s" semantics. The timeout is honoured for the pure-Python sympy + # work because the interpreter periodically yields the GIL to the joiner. + import queue as _queue + import threading + + output_queue = _queue.Queue() + thread_args = args + (output_queue,) + thread = threading.Thread( + target=func, args=thread_args, kwargs=kwargs, daemon=True + ) + thread.start() + thread.join(timeout) + + if thread.is_alive(): + return False + + try: + return output_queue.get_nowait() + except _queue.Empty: + return False + +def math_equal( + prediction: Union[bool, float, str], + reference: Union[float, str], + include_percentage: bool = True, + is_close: bool = True, + timeout: bool = True, +) -> bool: + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + # print("Judge:", prediction, reference) + if prediction is None or reference is None: + return False + if str(prediction.strip().lower()) == str(reference.strip().lower()): + return True + if ( + reference in ["A", "B", "C", "D", "E"] + and choice_answer_clean(prediction) == reference + ): + return True + + try: # 1. numerical equal + if is_digit(prediction) and is_digit(reference): + # strict_match_result = deal_strict_match(prediction, reference) + # if strict_match_result is not None: + # return strict_match_result + prediction = parse_digits(prediction) + reference = parse_digits(reference) + # number questions + if include_percentage: + gt_result = [reference / 100, reference, reference * 100] + else: + gt_result = [reference] + for item in gt_result: + try: + if is_close: + if numeric_equal(prediction, item): + return True + else: + if item == prediction: + return True + except Exception: + continue + return False + except: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + ## pmatrix (amps) + if "pmatrix" in prediction and not "pmatrix" in reference: + reference = str_to_pmatrix(reference) + + ## deal with [], (), {} + pred_str, ref_str = prediction, reference + if ( + prediction.startswith("[") + and prediction.endswith("]") + and not reference.startswith("(") + ) or ( + prediction.startswith("(") + and prediction.endswith(")") + and not reference.startswith("[") + ): + pred_str = pred_str.strip("[]()") + ref_str = ref_str.strip("[]()") + for s in ["{", "}", "(", ")"]: + ref_str = ref_str.replace(s, "") + pred_str = pred_str.replace(s, "") + if pred_str.lower() == ref_str.lower(): + return True + + ## [a, b] vs. [c, d], return a==c and b==d + if ( + regex.match(r"(\(|\[).+(\)|\])", prediction) is not None + and regex.match(r"(\(|\[).+(\)|\])", reference) is not None + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts): + if all( + [ + math_equal( + pred_parts[i], ref_parts[i], include_percentage, is_close + ) + for i in range(len(pred_parts)) + ] + ): + return True + if ( + ( + prediction.startswith("\\begin{pmatrix}") + or prediction.startswith("\\begin{bmatrix}") + ) + and ( + prediction.endswith("\\end{pmatrix}") + or prediction.endswith("\\end{bmatrix}") + ) + and ( + reference.startswith("\\begin{pmatrix}") + or reference.startswith("\\begin{bmatrix}") + ) + and ( + reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}") + ) + ): + pred_lines = [ + line.strip() + for line in prediction[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + ref_lines = [ + line.strip() + for line in reference[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + matched = True + if len(pred_lines) == len(ref_lines): + for pred_line, ref_line in zip(pred_lines, ref_lines): + pred_parts = pred_line.split("&") + ref_parts = ref_line.split("&") + if len(pred_parts) == len(ref_parts): + if not all( + [ + math_equal( + pred_parts[i], + ref_parts[i], + include_percentage, + is_close, + ) + for i in range(len(pred_parts)) + ] + ): + matched = False + break + else: + matched = False + if not matched: + break + else: + matched = False + if matched: + return True + + if prediction.count("=") == 1 and reference.count("=") == 1: + pred = prediction.split("=") + pred = f"{pred[0].strip()} - ({pred[1].strip()})" + ref = reference.split("=") + ref = f"{ref[0].strip()} - ({ref[1].strip()})" + if timeout: + if call_with_timeout(symbolic_equal_process, pred, ref) or call_with_timeout(symbolic_equal_process, f"-({pred})", ref): + return True + else: + if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref): + return True + + elif ( + prediction.count("=") == 1 + and len(prediction.split("=")[0].strip()) <= 2 + and "=" not in reference + ): + if math_equal( + prediction.split("=")[1], reference, include_percentage, is_close + ): + return True + elif ( + reference.count("=") == 1 + and len(reference.split("=")[0].strip()) <= 2 + and "=" not in prediction + ): + if math_equal( + prediction, reference.split("=")[1], include_percentage, is_close + ): + return True + + + # symbolic equal with sympy + if timeout: + if call_with_timeout(symbolic_equal_process, prediction, reference): + return True + else: + if symbolic_equal(prediction, reference): + return True + + return False + + +# One-time warm-up (vs upstream): the first symbolic comparison otherwise pays +# the sympy + latex2sympy import / parser-build cost, which can exceed the +# per-call timeout in call_with_timeout and make the first LaTeX answer of a run +# score wrong. Priming the parsers once here keeps every real comparison warm. +try: + symbolic_equal(r"\frac{1}{2}", "0.5") +except Exception: + pass diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/utils.py b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py index 52d3da8..d9840bf 100644 --- a/oellm/resources/custom_lm_eval_tasks/polymath/utils.py +++ b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py @@ -1,29 +1,31 @@ -"""Helpers for the multilingual PolyMath (Qwen/PolyMath) generative math task. +"""Task glue for the multilingual PolyMath (Qwen/PolyMath) generative math task. -The model is prompted to reason and wrap its final answer in ``\\boxed{...}``. -We extract that boxed span, normalise it (and the gold answer) with the -Minerva/Lewkowycz string normalisation, and score with exact match. A light -sympy numeric fallback catches equivalent-but-differently-written numbers -(e.g. ``18`` vs ``18.0``, ``1/2`` vs ``0.5``). +Scoring reproduces the official Qwen/PolyMath harness: + * answer extraction is ``extract_boxed_content`` copied verbatim from + https://github.com/QwenLM/PolyMath/blob/main/eval/run_eval.py (first + ``\\boxed{...}`` span, spaces stripped), and + * correctness is the upstream ``math_equal`` judge (see ``polymath_eval.py``, + vendored verbatim from ``eval/scripts.py``). -Note: unlike ``lm_eval``'s ``minerva_math`` we deliberately avoid -``math_verify`` / ``sympy.parsing.latex.parse_latex`` here — ``math_verify`` is -not installed in this project's venv and ``parse_latex`` requires -``antlr4-python3-runtime==4.11`` (the venv ships 4.13), so importing them would -break task loading. Everything below is self-contained. +``process_results`` mirrors ``run_eval.py``'s scoring loop: take the first boxed +span (``None`` if absent) and ``math_equal`` it against the gold ``answer``. """ +import importlib.util as _ilu +import os as _os import re -import signal -from typing import Optional -try: - from sympy import simplify, sympify - - _HAS_SYMPY = True -except ImportError: # pragma: no cover - sympy is a transitive lm-eval dep - _HAS_SYMPY = False +# --- Vendored upstream judge (polymath_eval.py, sibling file) --------------- +# lm-eval loads this module via spec_from_file_location, so __file__ is set but +# the task dir is not on sys.path; load the sibling scorer by absolute path. +_spec = _ilu.spec_from_file_location( + "polymath_eval", + _os.path.join(_os.path.dirname(_os.path.abspath(__file__)), "polymath_eval.py"), +) +_polymath_eval = _ilu.module_from_spec(_spec) +_spec.loader.exec_module(_polymath_eval) +math_equal = _polymath_eval.math_equal # --- Prompt ----------------------------------------------------------------- @@ -39,197 +41,33 @@ def doc_to_target(doc: dict) -> str: return doc["answer"] -# --- \boxed{} extraction (from lm_eval/tasks/minerva_math/utils.py) --------- -def last_boxed_only_string(string: str) -> Optional[str]: - idx = string.rfind("\\boxed") - if "\\boxed " in string: - return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] - if idx < 0: - idx = string.rfind("\\fbox") - if idx < 0: - return None - - i = idx - right_brace_idx = None - num_left_braces_open = 0 - while i < len(string): - if string[i] == "{": - num_left_braces_open += 1 - if string[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - i += 1 - - if right_brace_idx is None: - return None - return string[idx : right_brace_idx + 1] - - -def remove_boxed(s: str) -> str: - if "\\boxed " in s: - left = "\\boxed " - assert s[: len(left)] == left - return s[len(left) :] - - left = "\\boxed{" - assert s[: len(left)] == left - assert s[-1] == "}" - return s[len(left) : -1] - - -def _extract_answer(text: str) -> str: - """Pull the final answer out of a model generation. - - Prefer the last ``\\boxed{...}`` span; otherwise fall back to the last - number that appears in the text; otherwise return the stripped text. - """ - boxed = last_boxed_only_string(text) - if boxed is not None: - try: - return remove_boxed(boxed) - except AssertionError: - pass - numbers = re.findall(r"-?\d[\d,]*\.?\d*", text) - if numbers: - return numbers[-1] - return text.strip() - - -# --- Normalisation (from Lewkowycz et al. 2022 appendix D, via minerva) ----- -SUBSTITUTIONS = [ - ("an ", ""), - ("a ", ""), - (".$", "$"), - ("\\$", ""), - (r"\ ", ""), - (" ", ""), - ("mbox", "text"), - (",\\text{and}", ","), - ("\\text{and}", ","), - ("\\text{m}", "\\text{}"), -] -REMOVED_EXPRESSIONS = [ - "square", - "ways", - "integers", - "dollars", - "mph", - "inches", - "ft", - "hours", - "km", - "units", - "\\ldots", - "sue", - "points", - "feet", - "minutes", - "digits", - "cents", - "degrees", - "cm", - "gm", - "pounds", - "meters", - "meals", - "edges", - "students", - "childrentickets", - "multiples", - "\\text{s}", - "\\text{.}", - "\\text{\ns}", - "\\text{}^2", - "\\text{}^3", - "\\text{\n}", - "\\text{}", - r"\mathrm{th}", - r"^\circ", - r"^{\circ}", - r"\;", - r",\!", - "{,}", - '"', - "\\dots", -] - - -def normalize_final_answer(final_answer: str) -> str: - """Normalise a final answer to a quantitative reasoning question.""" - final_answer = final_answer.split("=")[-1] - - for before, after in SUBSTITUTIONS: - final_answer = final_answer.replace(before, after) - for expr in REMOVED_EXPRESSIONS: - final_answer = final_answer.replace(expr, "") - - # Extract answer that is in LaTeX math, is bold, is boxed, etc. - final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) - final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) - final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) - - # Normalise shorthand TeX (\fracab -> \frac{a}{b}, \sqrta -> \sqrt{a}). - final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) - final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) - final_answer = final_answer.replace("$", "") - - # Normalise 100,000 -> 100000 - if final_answer.replace(",", "").isdigit(): - final_answer = final_answer.replace(",", "") - - return final_answer.strip() - - -# --- Equivalence ------------------------------------------------------------ -class _timeout: - def __init__(self, seconds: int = 5): - self.seconds = seconds - - def handle_timeout(self, signum, frame): - raise TimeoutError - - def __enter__(self): - # SIGALRM is only settable from the main thread; degrade to no timeout - # (sympify/simplify are fast for the numeric cases we rely on) otherwise. - self._armed = False - try: - signal.signal(signal.SIGALRM, self.handle_timeout) - signal.alarm(self.seconds) - self._armed = True - except ValueError: - pass - - def __exit__(self, exc_type, exc_value, traceback): - if self._armed: - signal.alarm(0) - - -def _sympy_numeric_equiv(pred: str, gold: str) -> bool: - """Best-effort numeric equivalence without LaTeX parsing. - - Handles plain numbers and simple arithmetic (``1/2`` == ``0.5``). LaTeX - expressions that ``sympify`` cannot parse simply return ``False`` and the - caller falls back to normalised string equality. - """ - if not _HAS_SYMPY: - return False - try: - with _timeout(5): - a = sympify(pred.replace(",", "")) - b = sympify(gold.replace(",", "")) - return bool(simplify(a - b) == 0) - except Exception: # noqa: BLE001 - sympify/simplify raise many types incl. TimeoutError - return False +# --- Answer extraction (verbatim from PolyMath eval/run_eval.py) ------------- +def extract_boxed_content(text): + pattern = re.compile(r"boxed{") + text = text.replace(" ", "") + + matches = pattern.finditer(text) + results = [] + for match in matches: + start_pos = match.end() + brace_count = 1 + i = start_pos + while i < len(text) and brace_count > 0: + if text[i] == "{": + brace_count += 1 + elif text[i] == "}": + brace_count -= 1 + i += 1 + if brace_count == 0: + results.append(text[start_pos : i - 1]) + return results def process_results(doc: dict, results: list[str]) -> dict[str, int]: - candidate = results[0] - pred = normalize_final_answer(_extract_answer(candidate)) - gold = normalize_final_answer(doc["answer"]) - - correct = pred == gold or _sympy_numeric_equiv(pred, gold) - return {"exact_match": int(correct)} + # Mirrors run_eval.py: first boxed span (None if absent), then math_equal + # against the gold answer. + extracted = extract_boxed_content(results[0]) + pred = extracted[0] if extracted else None + gold = str(doc["answer"]) + correct = math_equal(pred, gold) + return {"exact_match": int(bool(correct))} diff --git a/pyproject.toml b/pyproject.toml index 37224b3..87532c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,9 @@ include = ["oellm/resources/*", "oellm/resources/**/*"] [tool.ruff] line-length = 90 target-version = "py312" +# Vendored verbatim from QwenLM/PolyMath (eval/scripts.py); keep it byte-for-byte +# rather than reflowing/relinting upstream code. +extend-exclude = ["oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py"] [tool.ruff.lint] select = [ From be0e568e7cad595fd7b928bb9808d5248b10e151 Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Thu, 25 Jun 2026 08:12:08 +0300 Subject: [PATCH 5/6] fix: set writable MIOpen DB path in eval sbatch (ROCm/MI250X) Point MIOPEN_USER_DB_PATH / MIOPEN_CUSTOM_CACHE_DIR at a job-unique writable dir (and create it) so the conv1d in linear-attention models stops failing on ROCm with "Cannot open database file:/tmp/gfx90a*.ukdb". Co-Authored-By: Claude Opus 4.8 --- oellm/resources/template.sbatch | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 83465c5..90e8660 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -27,6 +27,10 @@ export HF_HUB_OFFLINE={hf_hub_offline} # Path to the shared Singularity image that contains all runtime deps (container mode) export EVAL_SIF_PATH="$EVAL_BASE_DIR/$EVAL_CONTAINER_IMAGE" +export MIOPEN_USER_DB_PATH="/tmp/$USER-miopen-${{SLURM_JOB_ID:-$$}}" +export MIOPEN_CUSTOM_CACHE_DIR="$MIOPEN_USER_DB_PATH" +mkdir -p "$MIOPEN_USER_DB_PATH" + echo "Running eval on $CSV_PATH with $NUM_JOBS array jobs distributing $TOTAL_EVALS evaluations" echo "hostname: "`hostname` echo "SLURM_JOB_ID: $SLURM_JOB_ID" From b4889cb157098656b82a4c725e573580ad2c49b6 Mon Sep 17 00:00:00 2001 From: Haider Altahan Date: Wed, 1 Jul 2026 04:12:28 +0300 Subject: [PATCH 6/6] style: keep polymath task-group descriptions under 80 chars, sort utils imports --- .../custom_lm_eval_tasks/polymath/utils.py | 1 - oellm/resources/task-groups.yaml | 15 +++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/oellm/resources/custom_lm_eval_tasks/polymath/utils.py b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py index d9840bf..64fb915 100644 --- a/oellm/resources/custom_lm_eval_tasks/polymath/utils.py +++ b/oellm/resources/custom_lm_eval_tasks/polymath/utils.py @@ -15,7 +15,6 @@ import os as _os import re - # --- Vendored upstream judge (polymath_eval.py, sibling file) --------------- # lm-eval loads this module via spec_from_file_location, so __file__ is set but # the task dir is not on sys.path; load the sibling scorer by absolute path. diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index ef62de6..9a7b007 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -532,12 +532,11 @@ task_groups: - task: "global_piqa_prompted_{lang}" subset: "{lang}" - # PolyMath ships 4 expert-calibrated difficulty tiers (low/medium/high/top, - # from grade-school to Olympiad-level), each its own group so a tier can be run - # in isolation. EU languages only (de/en/es/fr/it/pt); 0-shot generative, with - # the final answer extracted from \boxed{} and scored with exact_match. + # PolyMath ships 4 difficulty tiers (low/medium/high/top); each is a + # separate group so a tier can be run in isolation. EU languages only + # (de/en/es/fr/it/pt); 0-shot generative, \boxed{} answer + exact_match. polymath-eu-low: - description: "PolyMath EU-language math reasoning — low difficulty tier (0-shot, \\boxed{} answer extraction)." + description: "PolyMath EU math, low tier (0-shot, exact_match)." suite: lm-eval-harness n_shots: [0] dataset: Qwen/PolyMath @@ -547,7 +546,7 @@ task_groups: subset: "{lang}" polymath-eu-medium: - description: "PolyMath EU-language math reasoning — medium difficulty tier (0-shot, \\boxed{} answer extraction)." + description: "PolyMath EU math, medium tier (0-shot, exact_match)." suite: lm-eval-harness n_shots: [0] dataset: Qwen/PolyMath @@ -557,7 +556,7 @@ task_groups: subset: "{lang}" polymath-eu-high: - description: "PolyMath EU-language math reasoning — high difficulty tier (0-shot, \\boxed{} answer extraction)." + description: "PolyMath EU math, high tier (0-shot, exact_match)." suite: lm-eval-harness n_shots: [0] dataset: Qwen/PolyMath @@ -567,7 +566,7 @@ task_groups: subset: "{lang}" polymath-eu-top: - description: "PolyMath EU-language math reasoning — top (Olympiad-level) difficulty tier (0-shot, \\boxed{} answer extraction)." + description: "PolyMath EU math, top/Olympiad tier (0-shot, exact_match)." suite: lm-eval-harness n_shots: [0] dataset: Qwen/PolyMath