From ec6f9be7ddab2ee8ac69890aff923743f4062285 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Wed, 29 Apr 2026 23:28:42 +0300 Subject: [PATCH 1/9] Add 8 pretrain evals in decontamination list --- eval_job.sh | 48 +++++++++ oellm/main.py | 83 +++++++++++---- oellm/resources/task-groups.yaml | 167 ++++++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 277 insertions(+), 22 deletions(-) create mode 100755 eval_job.sh diff --git a/eval_job.sh b/eval_job.sh new file mode 100755 index 0000000..7a391f4 --- /dev/null +++ b/eval_job.sh @@ -0,0 +1,48 @@ +# export HF_HUB_OFFLINE=1 # Set Hugging Face Hub to offline mode to run in interactive sessions +export BATCH_SIZE=1 + +# TASK_GROUPS="mgsm-eu,xcsqa,open-subtitles-x-to-eng,open-subtitles-eng-to-x,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x" +# TASK_GROUPS="mgsm-eu" # queued +# TASK_GROUPS="xcsqa" # queued +# TASK_GROUPS="doclevel-mt-x-to-eng,doclevel-mt-eng-to-x" # queued +# TASK_GROUPS="polymath" # queued +# TASK_GROUPS="global-piqa" # queued with qwen +TASK_GROUPS="polymath" # queued with qwen +# TASK_GROUPS="sib-200,mgsm-eu,xcsqa,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x,polymath" +# TASK_GROUPS="belebele-eu-cf,sib-200,multiblimp,global-piqa,mgsm-eu,xcsqa,open-subtitles-x-to-eng,open-subtitles-eng-to-x,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x,polymath" + +args=( + # --venv_path .venv + # --download_only true + # --slurm_template_var '{"PARTITION":"dev-g","TIME":"01:00:00","GPUS_PER_NODE":1}' # when small-g is busy + # --dry_run + # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf.csv" + # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf_9b.csv" + # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf_others.csv" + # --models "EleutherAI/pythia-14m" + --models "Qwen/Qwen2.5-0.5B" + # --models "MultiSynt/nemotron-cc-finnish-tower72b" + # # --task_groups "belebele-eu-cf" + --task_groups "${TASK_GROUPS}" + # --task_groups "sib-200" + # --task_groups "multiblimp" + # --task_groups "global-piqa" + # --task_groups "mgsm-eu" + # --task_groups "xcsqa" + # --task_groups "open-subtitles-x-to-eng" + # --task_groups "open-subtitles-eng-to-x" + # --task_groups "doclevel-mt-x-to-eng" + # --task_groups "math" + # --task_groups "doclevel-mt-eng-to-x" + # --tasks "mgsm_native_cot_bn" + # --tasks "belebele_fin_Latn_cf" + --n_shot 0 + --limit 4 +) + +uv run oellm schedule-eval "${args[@]}" + +# saves result to $EVAL_OUTPUT_DIR +# /leonardo_work/OELLM_prod2026/users/tko00000/oellm-evals/outputs +# /pfs/lustrep4/scratch/project_462000963/oellm-cli-shared-evals/tingwenk/ +# if --local is set, to ./oellm-output \ No newline at end of file diff --git a/oellm/main.py b/oellm/main.py index 6a19984..5abc6e3 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -577,6 +577,12 @@ def _first_matching_prefix( val, key = _first_matching_prefix(result_dict, metric.split(",")[0]) if val is not None: return val, key + + # Add a check for majority voting patterns (e.g., maj@4, maj@8) + for k, v in result_dict.items(): + if k.startswith("maj@") and isinstance(v, (int, float)): + return float(v), k + return None, None def _split_task_and_nshot(name: str) -> tuple[str, int | None]: @@ -589,6 +595,31 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: return base, int(after) return name, None + def _aggregate_group_from_subtasks( + group_name: str, + group_subtasks: list[str], + all_results: dict, + ) -> tuple[float | None, str | None]: + """Fallback for lm-eval groups whose aggregate row has no metric payload.""" + metric_values: list[float] = [] + metric_name: str | None = None + + for subtask_name in group_subtasks: + subtask_results = all_results.get(subtask_name) + if not isinstance(subtask_results, dict): + continue + value, resolved_metric = _resolve_metric(group_name, subtask_results) + if value is None: + continue + metric_values.append(value) + if metric_name is None: + metric_name = resolved_metric + + if not metric_values: + return None, None + + return float(sum(metric_values) / len(metric_values)), metric_name + results_path = Path(results_dir) if not results_path.exists(): raise ValueError(f"Results directory does not exist: {results_dir}") @@ -666,26 +697,35 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: for _s in _subs: group_subtask_names.add(_s) - # Prefer only the first aggregate metric from groups (simplified) + group_rows_added = 0 if groups_map: - group_name, group_results = next(iter(groups_map.items())) - # Prefer original extraction from n_shot_data and subtasks, then - # global_n_shot; only fall back to parsing the group name. - orig_group_name = group_name - n_shot = n_shot_data.get(orig_group_name, "unknown") - if n_shot == "unknown": - for subtask_name in group_subtasks_map.get(orig_group_name, []): - if subtask_name in n_shot_data: - n_shot = n_shot_data[subtask_name] - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - # Fallback: parse possible '|N' suffix from group name - group_name, parsed_n = _split_task_and_nshot(orig_group_name) - if n_shot == "unknown" and parsed_n is not None: - n_shot = parsed_n - performance, metric_name = _resolve_metric(group_name, group_results) - if performance is not None: + for orig_group_name, group_results in groups_map.items(): + # Prefer original extraction from n_shot_data and subtasks, then + # global_n_shot; only fall back to parsing the group name. + n_shot = n_shot_data.get(orig_group_name, "unknown") + if n_shot == "unknown": + for subtask_name in group_subtasks_map.get(orig_group_name, []): + if subtask_name in n_shot_data: + n_shot = n_shot_data[subtask_name] + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + + group_name, parsed_n = _split_task_and_nshot(orig_group_name) + if n_shot == "unknown" and parsed_n is not None: + n_shot = parsed_n + + performance, metric_name = _resolve_metric(group_name, group_results) + if performance is None: + performance, metric_name = _aggregate_group_from_subtasks( + group_name, + group_subtasks_map.get(orig_group_name, []), + results, + ) + + if performance is None: + continue + if check: completed_jobs.add((model_name, group_name, n_shot)) rows.append( @@ -697,7 +737,10 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: "metric_name": metric_name if metric_name is not None else "", } ) - # Skip per-task iteration when groups are present + group_rows_added += 1 + + if group_rows_added: + # Skip per-task iteration when group aggregates were extracted continue for task_name, task_results in results.items(): diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 0aa5d12..0b06bc7 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -32,10 +32,24 @@ task_metrics: winogrande: acc arc_challenge: acc_norm arc_easy: acc_norm + arc_mt: acc_norm boolq: acc commonsense_qa: acc hellaswag: acc_norm piqa: acc_norm + global_piqa_prompted: acc + sib: acc + multiblimp: acc + opensubtitles_multi40_pair: acc + xcsqa_eng_mcf: acc_norm + # math:algebra: maj@4 + # math:counting_and_probability: maj@4 + # math:geometry: maj@4 + # math:intermediate_algebra: maj@4 + # math:number_theory: maj@4 + # math:prealgebra: maj@4 + # math:precalculus: maj@4 + math: maj@4 social_iqa: acc agieval_lsat_ar: acc wsc273: acc @@ -84,6 +98,9 @@ task_groups: n_shots: [10] dataset: allenai/ai2_arc subset: ARC-Challenge + - task: arc_mt + n_shots: [10] + dataset: LumiOpen/arc_challenge_mt - task: commonsense_qa n_shots: [10] dataset: tau/commonsense_qa @@ -311,16 +328,146 @@ task_groups: description: "EU Language GSM benchmarks in Aya Expanse" suite: lm-eval-harness n_shots: [5] - dataset: jbross-ibm-research/mgsm tasks: + # Keep Greek omitted due to non-commercial license constraints. + - task: mgsm_native_cot_bn + dataset: juletxara/mgsm + subset: bn - task: mgsm_native_cot_en + dataset: juletxara/mgsm subset: en - task: mgsm_native_cot_de + dataset: juletxara/mgsm subset: de - task: mgsm_native_cot_es + dataset: juletxara/mgsm subset: es - task: mgsm_native_cot_fr + dataset: juletxara/mgsm subset: fr + - task: mgsm_native_cot_ja + dataset: juletxara/mgsm + subset: ja + - task: mgsm_native_cot_ru + dataset: juletxara/mgsm + subset: ru + - task: mgsm_native_cot_sw + dataset: juletxara/mgsm + subset: sw + - task: mgsm_native_cot_te + dataset: juletxara/mgsm + subset: te + - task: mgsm_native_cot_th + dataset: juletxara/mgsm + subset: th + - task: mgsm_native_cot_zh + dataset: juletxara/mgsm + subset: zh + + global-piqa: + description: "Global PIQA multiple-choice benchmark" + suite: lm-eval-harness + n_shots: [0] + tasks: + # Disabled for short-context models in the current eval setup. + # In the 2026-04-28-14-58-23 run this task failed with: + # "requested max tokens to generate (2048) must be less than model's + # maximum sequence length (2048)". + # [] + - task: global_piqa_prompted + + sib-200: + description: "SIB-200 sentence topic classification" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: sib + + multiblimp: + description: "MultiBLiMP pairwise sentence ranking" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: multiblimp + + xcsqa: + description: "Cross-lingual CSQA in LightEval (MCF formulation)" + suite: lighteval + n_shots: [0] + tasks: + # Explicit dataset metadata lets `--download_only` warm the same cached + # configs that lighteval will request at runtime. + - task: xcsqa_eng_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-en + - task: xcsqa_deu_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-de + - task: xcsqa_spa_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-es + - task: xcsqa_fra_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-fr + - task: xcsqa_ita_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-it + - task: xcsqa_pol_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-pl + - task: xcsqa_por_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-pt + + open-subtitles-x-to-eng: + description: "OpenSubtitles translation (x to English), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # Use lm-eval OpenSubtitles dynamic pair task format (PR #3706). + # Keep mirrored groups to preserve directional report slots. + - task: opensubtitles_multi40_pair + + open-subtitles-eng-to-x: + description: "OpenSubtitles translation (English to x), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: opensubtitles_multi40_pair + + doclevel-mt-x-to-eng: + description: "Document-level MT proxy (x to English), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # TODO: Replace with dedicated doc-level MT tasks when available in lm-eval. + - task: wmt14-fr-en + - task: wmt16-de-en + - task: wmt16-ro-en + + doclevel-mt-eng-to-x: + description: "Document-level MT proxy (English to x), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # TODO: Replace with dedicated doc-level MT tasks when available in lm-eval. + - task: wmt14-en-fr + - task: wmt16-en-de + - task: wmt16-en-ro + + polymath: + description: "MATH dataset subtasks" + suite: lighteval + n_shots: [0] + dataset: DigitalLearningGmbH/MATH-lighteval + tasks: + - task: math + # Disabled for short-context models in the current eval setup. + # `math` requests 2048 generation tokens, which fails on + # `EleutherAI/pythia-14m` with: + # "requested max tokens to generate (2048) must be less than model's + # maximum sequence length (2048)". + # [] generic-multilingual: description: "Generic multilingual benchmarks in Aya Expanse" @@ -391,6 +538,9 @@ task_groups: - task: include_base_44_ukrainian subset: Ukrainian + # NOTE: HellaSwag has translation variants with NC license constraints upstream. + # Keep using the canonical `hellaswag` task for unrestricted baseline reporting. + dclm-core-22: description: "DCLM core 22 evaluation tasks (lm-eval-harness, matching LLM Foundry task types)" suite: lm-eval-harness @@ -406,6 +556,9 @@ task_groups: n_shots: [10] dataset: allenai/ai2_arc subset: ARC-Challenge + - task: arc_mt + n_shots: [10] + dataset: LumiOpen/arc_challenge_mt - task: boolq n_shots: [10] dataset: aps/super_glue @@ -499,12 +652,22 @@ task_groups: super_groups: oellm-multilingual: - description: "Combined Belebele EU set plus multilingual benchmarks" + description: "Combined multilingual benchmark suite" task_groups: - task: flores-200-eu-to-eng - task: flores-200-eng-to-eu - task: belebele-eu-5-shot + - task: belebele-eu-cf - task: global-mmlu-eu - task: mgsm-eu + - task: global-piqa + - task: sib-200 + - task: multiblimp + - task: xcsqa + - task: polymath + - task: open-subtitles-x-to-eng + - task: open-subtitles-eng-to-x + - task: doclevel-mt-x-to-eng + - task: doclevel-mt-eng-to-x - task: generic-multilingual - task: include diff --git a/pyproject.toml b/pyproject.toml index 4d94f44..ee44267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "rich", "huggingface_hub", "pyyaml", + "transformers==4.49", ] [project.optional-dependencies] From 2a3876ccb034f78f7247cbf2123fded4c134d000 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Mon, 4 May 2026 16:32:47 +0200 Subject: [PATCH 2/9] Add tasks xcsqa, global MGSM, polymath, global-piqa --- .gitignore | 5 + README.md | 20 +- docs/TASKS.md | 12 ++ docs/VENV.md | 2 +- oellm/main.py | 18 +- .../custom_lm_eval_tasks/global_mgsm_am.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ar.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_bn.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ca.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_cs.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_cy.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_de.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_el.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_en.yaml | 35 ++++ .../custom_lm_eval_tasks/global_mgsm_es.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_eu.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_fr.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_gl.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_gu.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ha.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_hu.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ja.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_km.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_kn.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ko.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ky.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_lg.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_my.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ne.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ru.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_si.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_sn.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_sr.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_st.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_sw.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ta.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_te.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_th.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_ur.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_uz.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_vi.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_wo.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_xh.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_yo.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_zh.yaml | 3 + .../custom_lm_eval_tasks/global_mgsm_zu.yaml | 3 + .../custom_lm_eval_tasks/polymath_de_low.yaml | 3 + .../custom_lm_eval_tasks/polymath_en_low.yaml | 35 ++++ .../custom_lm_eval_tasks/polymath_es_low.yaml | 3 + .../custom_lm_eval_tasks/polymath_fr_low.yaml | 3 + oellm/resources/task-groups.yaml | 185 +++++++++++++++++- requirements-venv-evalchemy.txt | 2 +- 52 files changed, 433 insertions(+), 10 deletions(-) create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml diff --git a/.gitignore b/.gitignore index 16c2ad2..041367b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ **/task_map_cache.json *.sif results/ +oellm-output/ +.hf-cache/ +.uv-cache/ +.uv-python/ +.pre-commit-cache/ diff --git a/README.md b/README.md index dd9d43e..76849a4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A lightweight CLI for scheduling LLM evaluations across multiple HPC clusters us - **Collect results** and check for missing evaluations: `oellm collect-results` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Snellius) -- **Automatic building and deployment of containers** +- **Automatic building and deployment of containers** ## Quick Start @@ -42,16 +42,24 @@ In case you do not want to rely on the containers provided on a given cluster or ## Task Groups -Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/resources/task-groups.yaml). Each group specifies tasks, their n-shot settings, and HuggingFace dataset mappings. +Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/resources/task-groups.yaml). Each group specifies tasks, their n-shot settings, and HuggingFace dataset mappings. See [docs/TASKS.md](docs/TASKS.md) for the task-group schema and multilingual smoke-test notes. Available task groups: - `open-sci-0.01` - Standard benchmarks (COPA, MMLU, HellaSwag, ARC, etc.) - `belebele-eu-5-shot` - Belebele European language tasks -- `flores-200-eu-to-eng` / `flores-200-eng-to-eu` - Translation tasks +- `belebele-eu-cf` - Belebele European cloze-formulation tasks for lighteval +- `flores-200-eu-to-eng` - Flores 200 European languages to English translation +- `flores-200-eng-to-eu` - Flores 200 English to European languages translation - `global-mmlu-eu` - Global MMLU in EU languages - `mgsm-eu` - Multilingual GSM benchmarks +- `xcsqa` - XCSQA commonsense reasoning set +- `global-mgsm` - Global-MGSM multilingual math benchmarks +- `polymath` - PolyMath multilingual math set +- `global-piqa` - Global PIQA commonsense reasoning set - `generic-multilingual` - XWinograd, XCOPA, XStoryCloze - `include` - INCLUDE benchmarks +- `dclm-core-22` - DCLM core 22 evaluation tasks +- `reasoning` - GSM8k, IFEval, MBPP, GPQA-Diamond, MATH500, LiveCodeBench Super groups combine multiple task groups: - `oellm-multilingual` - All multilingual benchmarks combined @@ -73,7 +81,8 @@ The `--local` flag lets you run evaluations directly on your machine without a c ```bash # 1. Add eval dependencies to the project venv -uv pip install lm-eval torch transformers accelerate "datasets<4.0.0" +uv pip install "lm-eval[hf]" torch transformers accelerate "datasets<4.0.0" \ + "lighteval[multilingual]" language_data # 2. Run evaluations locally — useful for smoke-testing with a small sample oellm schedule-eval \ @@ -87,6 +96,9 @@ oellm schedule-eval \ Results are written to `./oellm-output//results/`. +See [docs/TASKS.md](docs/TASKS.md) for XCSQA, Global-MGSM, PolyMath, and +Global PIQA smoke-test commands and debugging notes. + **Air-gapped cluster nodes (no internet):** batch jobs set `HF_HUB_OFFLINE=1` and get `HF_HOME` from your cluster env. With `--local`, the CLI defaults `HF_HOME` to `~/.cache/huggingface` if unset and would otherwise allow Hub access—so on a compute node without network, export your real cache and offline flag before running, for example: ```bash diff --git a/docs/TASKS.md b/docs/TASKS.md index 6ed754c..7d8fdd2 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -62,3 +62,15 @@ oellm schedule-eval --models "model-name" --task_groups "my-benchmark" 2. **CI testing** - The test suite validates that all datasets in `task-groups.yaml` are accessible Tasks without a `dataset` field will not have their data pre-downloaded and are not covered by CI validation. + +## How to add custom lm-eval YAMLs + +There are two related pieces of configuration to be aware of: + +- `oellm/resources/task-groups.yaml` — oellm's scheduler registry. It lists the task names to run, the evaluation suite (`lm-eval-harness`, `lighteval`, or `evalchemy`), few-shot counts, and which dataset should be pre-downloaded. +- The evaluation-suite task registry (lm-eval or lighteval) — defines how a task is executed: which dataset split to load, how prompts are formatted, how answers are extracted, and which metric to compute. + +If you add a task name to `task-groups.yaml`, the evaluation suite also needs a task definition for that name. If not, place custom task YAMLs in `oellm/resources/custom_lm_eval_tasks`. Then `main.py` ... +If you add a task name to `task-groups.yaml`, the evaluation suite also needs a task definition for that name. If not, place custom task YAMLs in `oellm/resources/custom_lm_eval_tasks`. + +Then `main.py` will include that directory when generating the evaluation script: the `schedule_evals` command sets `lm_eval_include_path` (defaulting to the bundled `oellm/resources/custom_lm_eval_tasks`), and the generated SLURM script passes it to lm_eval as `--include_path`. This makes lm_eval aware of custom task YAMLs such as `polymath_en_low`. See `oellm/main.py` and `oellm/resources/template.sbatch` for the wiring. diff --git a/docs/VENV.md b/docs/VENV.md index eb04c88..d790c9a 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -76,7 +76,7 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a 3. Run with `EVALCHEMY_DIR` pointing to the cloned repo: ```bash - export HF_ALLOW_CODE_EVAL=1 # required by MBPP + export HF_ALLOW_CODE_EVAL=1 # required by MBPP EVALCHEMY_DIR=$(pwd)/evalchemy oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M \ --task_groups reasoning \ diff --git a/oellm/main.py b/oellm/main.py index 6a19984..727a1f6 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -633,6 +633,7 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: or data.get("config_general", {}).get("model_name") or data.get("config_general", {}).get("model") or data.get("config_general", {}).get("model_path") + or data.get("config_general", {}).get("model_config", {}).get("model_name") or data.get("summary_general", {}).get("model") or data.get("model") or "unknown" @@ -712,6 +713,11 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: if task_name.startswith("mmlu_") and task_name != "mmlu": continue + # lighteval writes a synthetic aggregate named "all" next to the + # concrete task; keep the concrete task rows for oellm job checks. + if task_name == "all": + continue + # Skip Global MMLU subtasks - keep only aggregates like global_mmlu_full_pt if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: continue @@ -720,9 +726,15 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: # Prefer original extraction from `n_shot_data` and `global_n_shot`, # and fall back to parsing a '|N' suffix in the task name. task_name_clean, parsed_n = _split_task_and_nshot(task_name) - n_shot = ( - n_shot_data.get(task_name_clean) or global_n_shot or parsed_n or "unknown" - ) + n_shot = n_shot_data.get(task_name_clean) + if n_shot is None: + n_shot = n_shot_data.get(task_name) + if n_shot is None: + n_shot = global_n_shot + if n_shot is None: + n_shot = parsed_n + if n_shot is None: + n_shot = "unknown" # If this is a group aggregate and n_shot is missing, derive from any subtask if task_name_clean in group_aggregate_names and n_shot == "unknown": diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml new file mode 100644 index 0000000..742fa06 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_am +dataset_name: am diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml new file mode 100644 index 0000000..69584f7 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ar +dataset_name: ar diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml new file mode 100644 index 0000000..69316d2 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_bn +dataset_name: bn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml new file mode 100644 index 0000000..4e88ef1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ca +dataset_name: ca diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml new file mode 100644 index 0000000..bbd7e54 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_cs +dataset_name: cs diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml new file mode 100644 index 0000000..f63b0f3 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_cy +dataset_name: cy diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml new file mode 100644 index 0000000..91f8d21 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_de +dataset_name: de diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml new file mode 100644 index 0000000..de5ee04 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_el +dataset_name: el diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml new file mode 100644 index 0000000..9bfa7e1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml @@ -0,0 +1,35 @@ +tag: + - global_mgsm +task: global_mgsm_en +dataset_path: CohereLabs/global-mgsm +dataset_name: en +output_type: generate_until +test_split: test +doc_to_text: "{{instruction}}\n\nQuestion: {{question}}\n{{answer_prefix}}:" +doc_to_target: "{{answer}}" +target_delimiter: " " +generation_kwargs: + until: + - "\n" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +filter_list: + - name: flexible-extract + filter: + - function: regex + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml new file mode 100644 index 0000000..585a6e4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_es +dataset_name: es diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml new file mode 100644 index 0000000..94443e8 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_eu +dataset_name: eu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml new file mode 100644 index 0000000..d6ed75f --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_fr +dataset_name: fr diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml new file mode 100644 index 0000000..02f9c11 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_gl +dataset_name: gl diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml new file mode 100644 index 0000000..f123cf5 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_gu +dataset_name: gu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml new file mode 100644 index 0000000..93d4d9a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ha +dataset_name: ha diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml new file mode 100644 index 0000000..751a78a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_hu +dataset_name: hu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml new file mode 100644 index 0000000..e2cc17a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ja +dataset_name: ja diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml new file mode 100644 index 0000000..a629afe --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_km +dataset_name: km diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml new file mode 100644 index 0000000..3677e26 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_kn +dataset_name: kn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml new file mode 100644 index 0000000..1a16bd2 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ko +dataset_name: ko diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml new file mode 100644 index 0000000..e68f141 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ky +dataset_name: ky diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml new file mode 100644 index 0000000..76a0e6e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_lg +dataset_name: lg diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml new file mode 100644 index 0000000..d9fe17b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_my +dataset_name: my diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml new file mode 100644 index 0000000..fb6f1e0 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ne +dataset_name: ne diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml new file mode 100644 index 0000000..dcfc0c4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ru +dataset_name: ru diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml new file mode 100644 index 0000000..772995c --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_si +dataset_name: si diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml new file mode 100644 index 0000000..f44d119 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sn +dataset_name: sn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml new file mode 100644 index 0000000..f44d091 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sr +dataset_name: sr diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml new file mode 100644 index 0000000..e33b2b7 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_st +dataset_name: st diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml new file mode 100644 index 0000000..a14b6f3 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sw +dataset_name: sw diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml new file mode 100644 index 0000000..1c933fc --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ta +dataset_name: ta diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml new file mode 100644 index 0000000..e326391 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_te +dataset_name: te diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml new file mode 100644 index 0000000..f33ebe1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_th +dataset_name: th diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml new file mode 100644 index 0000000..19e15e6 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ur +dataset_name: ur diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml new file mode 100644 index 0000000..01c7645 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_uz +dataset_name: uz diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml new file mode 100644 index 0000000..035cf67 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_vi +dataset_name: vi diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml new file mode 100644 index 0000000..3f4c403 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_wo +dataset_name: wo diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml new file mode 100644 index 0000000..1493a21 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_xh +dataset_name: xh diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml new file mode 100644 index 0000000..078d9fd --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_yo +dataset_name: yo diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml new file mode 100644 index 0000000..8b0024a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_zh +dataset_name: zh diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml new file mode 100644 index 0000000..2a6dd5b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_zu +dataset_name: zu diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml new file mode 100644 index 0000000..17c7c73 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_de_low +dataset_name: de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml new file mode 100644 index 0000000..b077aef --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml @@ -0,0 +1,35 @@ +tag: + - polymath +task: polymath_en_low +dataset_path: Qwen/PolyMath +dataset_name: en +output_type: generate_until +test_split: low +doc_to_text: "{{question}}\nAnswer:" +doc_to_target: "{{answer}}" +target_delimiter: " " +generation_kwargs: + until: + - "\n" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +filter_list: + - name: flexible-extract + filter: + - function: regex + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml new file mode 100644 index 0000000..83a3884 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_es_low +dataset_name: es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml new file mode 100644 index 0000000..3b985a6 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_fr_low +dataset_name: fr diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 0aa5d12..b093a48 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -47,6 +47,55 @@ task_metrics: bigbench_operators_generate_until: exact_match bigbench_repeat_copy_logic_generate_until: exact_match bigbench_cs_algorithms_generate_until: exact_match + global_mgsm_am: exact_match + global_mgsm_ar: exact_match + global_mgsm_bn: exact_match + global_mgsm_ca: exact_match + global_mgsm_cs: exact_match + global_mgsm_cy: exact_match + global_mgsm_de: exact_match + global_mgsm_el: exact_match + global_mgsm_en: exact_match + global_mgsm_es: exact_match + global_mgsm_eu: exact_match + global_mgsm_fr: exact_match + global_mgsm_gl: exact_match + global_mgsm_gu: exact_match + global_mgsm_ha: exact_match + global_mgsm_hu: exact_match + global_mgsm_ja: exact_match + global_mgsm_km: exact_match + global_mgsm_kn: exact_match + global_mgsm_ko: exact_match + global_mgsm_ky: exact_match + global_mgsm_lg: exact_match + global_mgsm_my: exact_match + global_mgsm_ne: exact_match + global_mgsm_ru: exact_match + global_mgsm_si: exact_match + global_mgsm_sn: exact_match + global_mgsm_sr: exact_match + global_mgsm_st: exact_match + global_mgsm_sw: exact_match + global_mgsm_ta: exact_match + global_mgsm_te: exact_match + global_mgsm_th: exact_match + global_mgsm_ur: exact_match + global_mgsm_uz: exact_match + global_mgsm_vi: exact_match + global_mgsm_wo: exact_match + global_mgsm_xh: exact_match + global_mgsm_yo: exact_match + global_mgsm_zh: exact_match + global_mgsm_zu: exact_match + polymath_en_low: exact_match + polymath_de_low: exact_match + polymath_es_low: exact_match + polymath_fr_low: exact_match + global_piqa_completions_eng_latn: acc_norm + global_piqa_completions_deu_latn: acc_norm + global_piqa_completions_spa_latn_spai: acc_norm + global_piqa_completions_fra_latn_fran: acc_norm task_groups: open-sci-0.01: @@ -206,7 +255,7 @@ task_groups: subset: eus_Latn - task: belebele_cat_Latn_cf subset: cat_Latn - + flores-200-eu-to-eng: description: "Flores 200 EU to English translation" suite: lighteval @@ -322,6 +371,140 @@ task_groups: - task: mgsm_native_cot_fr subset: fr + xcsqa: + description: "XCSQA commonsense reasoning set" + suite: lighteval + n_shots: [0] + dataset: INK-USC/xcsr + tasks: + - task: xcsqa_eng_mcf + subset: X-CSQA-en + - task: xcsqa_deu_mcf + subset: X-CSQA-de + - task: xcsqa_spa_mcf + subset: X-CSQA-es + - task: xcsqa_fra_mcf + subset: X-CSQA-fr + + global-mgsm: + description: "Global-MGSM multilingual math benchmarks" + suite: lm-eval-harness + n_shots: [0] + dataset: CohereLabs/global-mgsm + tasks: + - task: global_mgsm_bn + subset: bn + - task: global_mgsm_de + subset: de + - task: global_mgsm_en + subset: en + - task: global_mgsm_es + subset: es + - task: global_mgsm_fr + subset: fr + - task: global_mgsm_ja + subset: ja + - task: global_mgsm_ru + subset: ru + - task: global_mgsm_sw + subset: sw + - task: global_mgsm_te + subset: te + - task: global_mgsm_th + subset: th + - task: global_mgsm_zh + subset: zh + - task: global_mgsm_ar + subset: ar + - task: global_mgsm_ca + subset: ca + - task: global_mgsm_cs + subset: cs + - task: global_mgsm_cy + subset: cy + - task: global_mgsm_el + subset: el + - task: global_mgsm_eu + subset: eu + - task: global_mgsm_gl + subset: gl + - task: global_mgsm_hu + subset: hu + - task: global_mgsm_ko + subset: ko + - task: global_mgsm_sr + subset: sr + - task: global_mgsm_vi + subset: vi + - task: global_mgsm_am + subset: am + - task: global_mgsm_ha + subset: ha + - task: global_mgsm_lg + subset: lg + - task: global_mgsm_sn + subset: sn + - task: global_mgsm_st + subset: st + - task: global_mgsm_wo + subset: wo + - task: global_mgsm_xh + subset: xh + - task: global_mgsm_yo + subset: yo + - task: global_mgsm_zu + subset: zu + - task: global_mgsm_ur + subset: ur + - task: global_mgsm_gu + subset: gu + - task: global_mgsm_km + subset: km + - task: global_mgsm_kn + subset: kn + - task: global_mgsm_ky + subset: ky + - task: global_mgsm_my + subset: my + - task: global_mgsm_ne + subset: ne + - task: global_mgsm_si + subset: si + - task: global_mgsm_ta + subset: ta + - task: global_mgsm_uz + subset: uz + + polymath: + description: "PolyMath multilingual math set (low difficulty)" + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + tasks: + - task: polymath_en_low + subset: en + - task: polymath_de_low + subset: de + - task: polymath_es_low + subset: es + - task: polymath_fr_low + subset: fr + + global-piqa: + description: "Global PIQA commonsense reasoning set" + suite: lm-eval-harness + n_shots: [0] + dataset: mrlbenchmarks/global-piqa-nonparallel + tasks: + - task: global_piqa_completions_eng_latn + subset: eng_latn + - task: global_piqa_completions_deu_latn + subset: deu_latn + - task: global_piqa_completions_spa_latn_spai + subset: spa_latn_spai + - task: global_piqa_completions_fra_latn_fran + subset: fra_latn_fran + generic-multilingual: description: "Generic multilingual benchmarks in Aya Expanse" suite: lm-eval-harness diff --git a/requirements-venv-evalchemy.txt b/requirements-venv-evalchemy.txt index 9ec7d3d..c63c3fb 100644 --- a/requirements-venv-evalchemy.txt +++ b/requirements-venv-evalchemy.txt @@ -1,6 +1,6 @@ # Dependencies for evalchemy evaluation -# lm-eval fork used by evalchemy +# lm-eval fork used by evalchemy lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix scipy==1.17.0 From 9be08bceabc8ee1dce79a0687517298cf694cd0e Mon Sep 17 00:00:00 2001 From: kerkathy Date: Mon, 4 May 2026 18:43:52 +0200 Subject: [PATCH 3/9] Fix transformers datasets mismatch (Error of dtype missing) --- oellm/main.py | 8 ++++--- oellm/resources/template.sbatch | 3 +++ oellm/utils.py | 42 +++++++++++++++++++++++---------- tests/test_schedule_evals.py | 31 ++++++++++++++++++++++++ tests/test_utils.py | 27 +++++++++++++++++++++ 5 files changed, 95 insertions(+), 16 deletions(-) diff --git a/oellm/main.py b/oellm/main.py index 84f4487..f3668af 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -36,8 +36,9 @@ def _resolve_hf_hub_offline(local: bool) -> int: """Value embedded in the generated eval script as HF_HUB_OFFLINE. If ``HF_HUB_OFFLINE`` is set in the environment when ``oellm`` runs, that - value wins. Otherwise defaults to online Hub access for ``--local`` - (typical laptop dev) and offline for SLURM jobs (air-gapped workers). + value wins. Otherwise generated eval scripts default to offline Hub access + because model and dataset checks/downloads have already happened before the + script runs. """ raw = os.environ.get("HF_HUB_OFFLINE") if raw is not None and str(raw).strip() != "": @@ -45,7 +46,7 @@ def _resolve_hf_hub_offline(local: bool) -> int: return int(str(raw).strip()) except ValueError: logging.warning("Invalid HF_HUB_OFFLINE=%r; using default", raw) - return 0 if local else 1 + return 1 def _resolve_slurm_mem() -> str: @@ -482,6 +483,7 @@ def schedule_evals( logging.info("Local evaluation completed.") except subprocess.CalledProcessError as e: logging.error(f"Evaluation failed with exit code {e.returncode}") + raise return try: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 38ecfa2..c392195 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -1,4 +1,7 @@ #!/bin/bash +# SBATCH directives are ignored when this script is run directly with bash. +set -euo pipefail + #SBATCH --job-name=oellm-eval #SBATCH --time={time_limit} #SBATCH --gres=gpu:$GPUS_PER_NODE diff --git a/oellm/utils.py b/oellm/utils.py index 4b6830c..1195306 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -302,6 +302,7 @@ def _process_model_paths(models: Iterable[str]): def _pre_download_datasets_from_specs( specs: Iterable, trust_remote_code: bool = True ) -> None: + import datasets from datasets import get_dataset_config_names, load_dataset specs_list = list(specs) @@ -309,6 +310,31 @@ def _pre_download_datasets_from_specs( return console = get_console() + datasets_major_version = int(datasets.__version__.split(".", 1)[0]) + supports_trust_remote_code = datasets_major_version < 4 + + def _load_dataset_compat(repo_id: str, name: str | None = None): + kwargs = {"name": name} + if trust_remote_code and supports_trust_remote_code: + kwargs["trust_remote_code"] = trust_remote_code + try: + return load_dataset(repo_id, **kwargs) + except ValueError as e: + if "trust_remote_code" not in str(e): + raise + kwargs.pop("trust_remote_code", None) + return load_dataset(repo_id, **kwargs) + + def _get_dataset_config_names_compat(repo_id: str): + kwargs = {} + if trust_remote_code and supports_trust_remote_code: + kwargs["trust_remote_code"] = trust_remote_code + try: + return get_dataset_config_names(repo_id, **kwargs) + except ValueError as e: + if "trust_remote_code" not in str(e): + raise + return get_dataset_config_names(repo_id) with console.status( f"Downloading datasets… {len(specs_list)} datasets", @@ -319,16 +345,10 @@ def _pre_download_datasets_from_specs( status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") try: - load_dataset( - spec.repo_id, - name=spec.subset, - trust_remote_code=trust_remote_code, - ) + _load_dataset_compat(spec.repo_id, name=spec.subset) except ValueError as e: if "Config name is missing" in str(e) and spec.subset is None: - configs = get_dataset_config_names( - spec.repo_id, trust_remote_code=trust_remote_code - ) + configs = _get_dataset_config_names_compat(spec.repo_id) logging.info( f"Dataset '{spec.repo_id}' requires config. " f"Downloading all {len(configs)} configs." @@ -337,11 +357,7 @@ def _pre_download_datasets_from_specs( status.update( f"Downloading '{spec.repo_id}/{cfg}' ({idx}/{len(specs_list)})" ) - load_dataset( - spec.repo_id, - name=cfg, - trust_remote_code=trust_remote_code, - ) + _load_dataset_compat(spec.repo_id, name=cfg) continue if "Feature type" in str(e) and "not found" in str(e): hf_datasets_cache = os.environ.get( diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index ad6152b..6dfe107 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -64,6 +64,37 @@ def test_schedule_evals_slurm_template_var_overrides(tmp_path): assert "#SBATCH --gres=gpu:2" in sbatch_content +def test_schedule_evals_generated_script_defaults_to_offline_hub(tmp_path): + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict( + os.environ, + { + "EVAL_OUTPUT_DIR": str(tmp_path), + "PARTITION": "default_partition", + "ACCOUNT": "test_account", + }, + clear=False, + ), + ): + os.environ.pop("HF_HUB_OFFLINE", None) + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + assert "set -euo pipefail" in sbatch_content + assert "export HF_HUB_OFFLINE=1" in sbatch_content + + def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): """Verify invalid slurm_template_var raises ValueError.""" with ( diff --git a/tests/test_utils.py b/tests/test_utils.py index ebc0163..36b7d46 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,7 @@ +import sys +from types import SimpleNamespace + +from oellm.task_groups import DatasetSpec from oellm.utils import _expand_local_model_paths, _num_jobs_in_queue @@ -54,3 +58,26 @@ class Result: monkeypatch.setattr("oellm.utils.subprocess.run", lambda *a, **kw: Result()) assert _num_jobs_in_queue() == 0 + + +class TestPreDownloadDatasets: + def test_omits_trust_remote_code_for_datasets_v4(self, monkeypatch): + calls = [] + + def load_dataset(repo_id, **kwargs): + calls.append((repo_id, kwargs)) + + fake_datasets = SimpleNamespace( + __version__="4.8.5", + load_dataset=load_dataset, + get_dataset_config_names=lambda *args, **kwargs: [], + ) + monkeypatch.setitem(sys.modules, "datasets", fake_datasets) + + from oellm.utils import _pre_download_datasets_from_specs + + _pre_download_datasets_from_specs( + [DatasetSpec(repo_id="example/data", subset="en")] + ) + + assert calls == [("example/data", {"name": "en"})] From e66211c4dcc6e4fb777b39fafc063aac6a4e2a71 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Tue, 5 May 2026 10:46:11 +0200 Subject: [PATCH 4/9] Wrap lm_eval to fix dtype error --- oellm/main.py | 1 + oellm/resources/template.sbatch | 21 +++++++++++++++++---- tests/test_schedule_evals.py | 5 +++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/oellm/main.py b/oellm/main.py index f3668af..86a281f 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -438,6 +438,7 @@ def schedule_evals( hf_hub_offline=_resolve_hf_hub_offline(local), additional_model_args=_resolve_additional_model_args(local), # Batch size evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), + oellm_repo_root=str(Path(__file__).resolve().parents[1]), ) # substitute any $ENV_VAR occurrences diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index c392195..f2a2991 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -1,7 +1,4 @@ #!/bin/bash -# SBATCH directives are ignored when this script is run directly with bash. -set -euo pipefail - #SBATCH --job-name=oellm-eval #SBATCH --time={time_limit} #SBATCH --gres=gpu:$GPUS_PER_NODE @@ -12,6 +9,9 @@ set -euo pipefail #SBATCH --error={log_dir}/%x-%A-%a.err #SBATCH --array=0-{array_limit}%{max_array_len} +# SBATCH directives are ignored when this script is run directly with bash. +set -euo pipefail + CSV_PATH="{csv_path}" NUM_JOBS={num_jobs} @@ -19,6 +19,7 @@ TOTAL_EVALS={total_evals} LIMIT="{limit}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" +OELLM_REPO_ROOT="{oellm_repo_root}" # avoiding crashes due to compute nodes not having access to the internet export HF_HOME=$HF_HOME @@ -28,6 +29,14 @@ export HF_HUB_OFFLINE={hf_hub_offline} # Path to the shared Singularity image that contains all runtime deps (container mode) export EVAL_SIF_PATH="$EVAL_BASE_DIR/$EVAL_CONTAINER_IMAGE" +if [ -n "$OELLM_REPO_ROOT" ] && [ -d "$OELLM_REPO_ROOT" ]; then + if [ -n "${{PYTHONPATH:-}}" ]; then + export PYTHONPATH="$OELLM_REPO_ROOT:$PYTHONPATH" + else + export PYTHONPATH="$OELLM_REPO_ROOT" + fi +fi + echo "Running eval on $CSV_PATH with $NUM_JOBS array jobs distributing $TOTAL_EVALS evaluations" echo "SLURM_JOB_ID: $SLURM_JOB_ID" echo "SLURM_ARRAY_JOB_ID: $SLURM_ARRAY_JOB_ID" @@ -84,6 +93,10 @@ do # bind the directory that contains the model checkpoint when the path exists BIND_PATHS="$EVAL_BASE_DIR:$EVAL_BASE_DIR,$HF_HOME:$HF_HOME,$HF_DATASETS_CACHE:$HF_DATASETS_CACHE" + if [ -d "$OELLM_REPO_ROOT" ] && [[ $OELLM_REPO_ROOT != $EVAL_BASE_DIR* ]]; then + BIND_PATHS="$BIND_PATHS,$OELLM_REPO_ROOT:$OELLM_REPO_ROOT" + fi + if [ -e "$model_path" ]; then # If the model_path is a file, bind its parent directory; otherwise bind the dir itself MODEL_DIR="$model_path" @@ -136,7 +149,7 @@ do echo echo "----------------------------------------------------" echo "lm_eval Execution" - run_python -m lm_eval --model hf \ + run_python -m oellm.lm_eval_compat --model hf \ --model_args pretrained="$model_path",trust_remote_code=True \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 6dfe107..3ddae88 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -93,6 +93,11 @@ def test_schedule_evals_generated_script_defaults_to_offline_hub(tmp_path): sbatch_content = sbatch_files[0].read_text() assert "set -euo pipefail" in sbatch_content assert "export HF_HUB_OFFLINE=1" in sbatch_content + assert "python -m oellm.lm_eval_compat" in sbatch_content + assert 'OELLM_REPO_ROOT="' in sbatch_content + assert sbatch_content.index("#SBATCH --job-name=") < sbatch_content.index( + "set -euo pipefail" + ) def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): From 20dfa6ca2433d88463ad555441506265488e258e Mon Sep 17 00:00:00 2001 From: kerkathy Date: Mon, 11 May 2026 15:00:21 +0300 Subject: [PATCH 5/9] Fix lm_eval bug --- oellm/resources/template.sbatch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index f2a2991..7507074 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -149,7 +149,7 @@ do echo echo "----------------------------------------------------" echo "lm_eval Execution" - run_python -m oellm.lm_eval_compat --model hf \ + run_python -m lm_eval --model hf \ --model_args pretrained="$model_path",trust_remote_code=True \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ From 3c1bee70f44fcd56a8697366cf1492a948550cc4 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Wed, 13 May 2026 13:04:50 +0300 Subject: [PATCH 6/9] cleanup --- .gitignore | 1 - README.md | 11 ++++++----- eval_job.sh | 48 ------------------------------------------------ 3 files changed, 6 insertions(+), 54 deletions(-) delete mode 100755 eval_job.sh diff --git a/.gitignore b/.gitignore index 041367b..a3cbac7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,6 @@ **/task_map_cache.json *.sif results/ -oellm-output/ .hf-cache/ .uv-cache/ .uv-python/ diff --git a/README.md b/README.md index 76849a4..310bc56 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,9 @@ oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" The `--local` flag lets you run evaluations directly on your machine without a cluster or Singularity container. It generates the same eval script and executes it with bash, injecting fake SLURM environment variables so all tasks run sequentially in a single process. This is useful for testing that tasks and models are correctly configured before submitting to a cluster. ```bash -# 1. Add eval dependencies to the project venv -uv pip install "lm-eval[hf]" torch transformers accelerate "datasets<4.0.0" \ - "lighteval[multilingual]" language_data +# 1. Create a venv and add lm-eval dependencies +uv venv --python 3.12 .venv +uv pip install --python .venv/bin/python -r requirements-venv.txt # 2. Run evaluations locally — useful for smoke-testing with a small sample oellm schedule-eval \ @@ -96,8 +96,9 @@ oellm schedule-eval \ Results are written to `./oellm-output//results/`. -See [docs/TASKS.md](docs/TASKS.md) for XCSQA, Global-MGSM, PolyMath, and -Global PIQA smoke-test commands and debugging notes. +For lighteval tasks, install lighteval as an isolated uv tool so its +`datasets>=4.0.0` dependency does not conflict with lm-eval's `datasets<4.0.0`; +see [docs/VENV.md](docs/VENV.md). **Air-gapped cluster nodes (no internet):** batch jobs set `HF_HUB_OFFLINE=1` and get `HF_HOME` from your cluster env. With `--local`, the CLI defaults `HF_HOME` to `~/.cache/huggingface` if unset and would otherwise allow Hub access—so on a compute node without network, export your real cache and offline flag before running, for example: diff --git a/eval_job.sh b/eval_job.sh deleted file mode 100755 index 7a391f4..0000000 --- a/eval_job.sh +++ /dev/null @@ -1,48 +0,0 @@ -# export HF_HUB_OFFLINE=1 # Set Hugging Face Hub to offline mode to run in interactive sessions -export BATCH_SIZE=1 - -# TASK_GROUPS="mgsm-eu,xcsqa,open-subtitles-x-to-eng,open-subtitles-eng-to-x,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x" -# TASK_GROUPS="mgsm-eu" # queued -# TASK_GROUPS="xcsqa" # queued -# TASK_GROUPS="doclevel-mt-x-to-eng,doclevel-mt-eng-to-x" # queued -# TASK_GROUPS="polymath" # queued -# TASK_GROUPS="global-piqa" # queued with qwen -TASK_GROUPS="polymath" # queued with qwen -# TASK_GROUPS="sib-200,mgsm-eu,xcsqa,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x,polymath" -# TASK_GROUPS="belebele-eu-cf,sib-200,multiblimp,global-piqa,mgsm-eu,xcsqa,open-subtitles-x-to-eng,open-subtitles-eng-to-x,doclevel-mt-x-to-eng,doclevel-mt-eng-to-x,polymath" - -args=( - # --venv_path .venv - # --download_only true - # --slurm_template_var '{"PARTITION":"dev-g","TIME":"01:00:00","GPUS_PER_NODE":1}' # when small-g is busy - # --dry_run - # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf.csv" - # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf_9b.csv" - # --eval_csv_path "multisynt-evaluations/multisynt_evals_cf_others.csv" - # --models "EleutherAI/pythia-14m" - --models "Qwen/Qwen2.5-0.5B" - # --models "MultiSynt/nemotron-cc-finnish-tower72b" - # # --task_groups "belebele-eu-cf" - --task_groups "${TASK_GROUPS}" - # --task_groups "sib-200" - # --task_groups "multiblimp" - # --task_groups "global-piqa" - # --task_groups "mgsm-eu" - # --task_groups "xcsqa" - # --task_groups "open-subtitles-x-to-eng" - # --task_groups "open-subtitles-eng-to-x" - # --task_groups "doclevel-mt-x-to-eng" - # --task_groups "math" - # --task_groups "doclevel-mt-eng-to-x" - # --tasks "mgsm_native_cot_bn" - # --tasks "belebele_fin_Latn_cf" - --n_shot 0 - --limit 4 -) - -uv run oellm schedule-eval "${args[@]}" - -# saves result to $EVAL_OUTPUT_DIR -# /leonardo_work/OELLM_prod2026/users/tko00000/oellm-evals/outputs -# /pfs/lustrep4/scratch/project_462000963/oellm-cli-shared-evals/tingwenk/ -# if --local is set, to ./oellm-output \ No newline at end of file From abf46031dd724ce58bdd4b0f753c29cd95731fe9 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Wed, 13 May 2026 16:22:55 +0300 Subject: [PATCH 7/9] clean legacy error --- oellm/main.py | 70 +++++++++++++++++++++++++++++++++--- tests/test_schedule_evals.py | 2 +- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/oellm/main.py b/oellm/main.py index 86a281f..54c1c0b 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -32,6 +32,12 @@ ) +_SLURM_FAILURE_RE = re.compile( + r"Traceback|Exception|Error|FAILED|Killed|OOM|out of memory|No such file|No module|command not found", + re.IGNORECASE, +) + + def _resolve_hf_hub_offline(local: bool) -> int: """Value embedded in the generated eval script as HF_HUB_OFFLINE. @@ -585,7 +591,7 @@ def _first_matching_prefix( for k, v in result_dict.items(): if k.startswith("maj@") and isinstance(v, (int, float)): return float(v), k - + return None, None def _split_task_and_nshot(name: str) -> tuple[str, int | None]: @@ -652,6 +658,42 @@ def _aggregate_group_from_subtasks( jobs_df = pd.read_csv(jobs_csv_path) logging.info(f"Found {len(jobs_df)} scheduled jobs in jobs.csv") + submit_script = results_path / "submit_evals.sbatch" + slurm_logs_dir = results_path / "slurm_logs" + num_array_jobs = len(jobs_df) + total_evals = len(jobs_df) + if submit_script.exists(): + submit_text = submit_script.read_text(errors="replace") + num_jobs_match = re.search(r"^NUM_JOBS=(\d+)$", submit_text, re.M) + total_evals_match = re.search(r"^TOTAL_EVALS=(\d+)$", submit_text, re.M) + if num_jobs_match: + num_array_jobs = int(num_jobs_match.group(1)) + if total_evals_match: + total_evals = int(total_evals_match.group(1)) + evals_per_array_job = max( + 1, int(math.ceil(total_evals / max(1, num_array_jobs))) + ) + + def _slurm_status_for_job(row_index: int) -> str | None: + if not slurm_logs_dir.exists(): + return None + + array_index = row_index // evals_per_array_job + matching_logs = list(slurm_logs_dir.glob(f"*-{array_index}.out")) + list( + slurm_logs_dir.glob(f"*-{array_index}.err") + ) + if not matching_logs: + return "not_started" + + combined = "\n".join( + p.read_text(errors="replace") for p in matching_logs if p.exists() + ) + if _SLURM_FAILURE_RE.search(combined): + return "failed" + if f"Job {array_index} finished." in combined: + return "finished_without_result" + return "started_without_result" + # Collect results rows = [] completed_jobs = set() # Track (model, task, n_shot) tuples @@ -860,8 +902,9 @@ def _aggregate_group_from_subtasks( # Find missing jobs missing_jobs = [] + pending_jobs = [] - for _, job in jobs_df.iterrows(): + for row_index, job in jobs_df.iterrows(): job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) # Check if this job corresponds to one of our completed results @@ -887,14 +930,31 @@ def _aggregate_group_from_subtasks( break if not is_completed: - missing_jobs.append(job) - - completed_count = len(jobs_df) - len(missing_jobs) + slurm_status = _slurm_status_for_job(row_index) + if slurm_status == "not_started": + pending_job = job.copy() + pending_job["status"] = slurm_status + pending_jobs.append(pending_job) + else: + missing_job = job.copy() + if slurm_status: + missing_job["status"] = slurm_status + missing_jobs.append(missing_job) + + completed_count = len(jobs_df) - len(missing_jobs) - len(pending_jobs) logging.info(f"Total scheduled jobs: {len(jobs_df)}") logging.info(f"Completed jobs: {completed_count}") + if pending_jobs: + logging.info(f"Not-started jobs: {len(pending_jobs)}") logging.info(f"Missing jobs: {len(missing_jobs)}") + if len(pending_jobs) > 0: + pending_df = pd.DataFrame(pending_jobs) + pending_csv = output_csv.replace(".csv", "_pending.csv") + pending_df.to_csv(pending_csv, index=False) + logging.info(f"Not-started jobs saved to: {pending_csv}") + if len(missing_jobs) > 0: missing_df = pd.DataFrame(missing_jobs) missing_csv = output_csv.replace(".csv", "_missing.csv") diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 3ddae88..6451978 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -93,7 +93,7 @@ def test_schedule_evals_generated_script_defaults_to_offline_hub(tmp_path): sbatch_content = sbatch_files[0].read_text() assert "set -euo pipefail" in sbatch_content assert "export HF_HUB_OFFLINE=1" in sbatch_content - assert "python -m oellm.lm_eval_compat" in sbatch_content + assert "run_python -m lm_eval" in sbatch_content assert 'OELLM_REPO_ROOT="' in sbatch_content assert sbatch_content.index("#SBATCH --job-name=") < sbatch_content.index( "set -euo pipefail" From 5eb78a2d954555ff346a1a1eee0531b4394968f1 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Wed, 13 May 2026 16:32:10 +0300 Subject: [PATCH 8/9] fix lint error --- oellm/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/oellm/main.py b/oellm/main.py index 54c1c0b..b848dc9 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -31,7 +31,6 @@ capture_third_party_output_from_kwarg, ) - _SLURM_FAILURE_RE = re.compile( r"Traceback|Exception|Error|FAILED|Killed|OOM|out of memory|No such file|No module|command not found", re.IGNORECASE, From 02265befa234edfce52098cd270454c557277294 Mon Sep 17 00:00:00 2001 From: kerkathy Date: Wed, 13 May 2026 16:58:07 +0300 Subject: [PATCH 9/9] Remove duplicate task keys --- oellm/resources/task-groups.yaml | 41 -------------------------------- 1 file changed, 41 deletions(-) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 7c5a51c..63ae6de 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -413,18 +413,6 @@ task_groups: dataset: juletxara/mgsm subset: zh - global-piqa: - description: "Global PIQA multiple-choice benchmark" - suite: lm-eval-harness - n_shots: [0] - tasks: - # Disabled for short-context models in the current eval setup. - # In the 2026-04-28-14-58-23 run this task failed with: - # "requested max tokens to generate (2048) must be less than model's - # maximum sequence length (2048)". - # [] - - task: global_piqa_prompted - sib-200: description: "SIB-200 sentence topic classification" suite: lm-eval-harness @@ -504,35 +492,6 @@ task_groups: - task: wmt16-en-de - task: wmt16-en-ro - polymath: - description: "MATH dataset subtasks" - suite: lighteval - n_shots: [0] - dataset: DigitalLearningGmbH/MATH-lighteval - tasks: - - task: math - # Disabled for short-context models in the current eval setup. - # `math` requests 2048 generation tokens, which fails on - # `EleutherAI/pythia-14m` with: - # "requested max tokens to generate (2048) must be less than model's - # maximum sequence length (2048)". - # [] - - xcsqa: - description: "XCSQA commonsense reasoning set" - suite: lighteval - n_shots: [0] - dataset: INK-USC/xcsr - tasks: - - task: xcsqa_eng_mcf - subset: X-CSQA-en - - task: xcsqa_deu_mcf - subset: X-CSQA-de - - task: xcsqa_spa_mcf - subset: X-CSQA-es - - task: xcsqa_fra_mcf - subset: X-CSQA-fr - global-mgsm: description: "Global-MGSM multilingual math benchmarks" suite: lm-eval-harness