diff --git a/.gitignore b/.gitignore index 16c2ad2..a3cbac7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ **/task_map_cache.json *.sif results/ +.hf-cache/ +.uv-cache/ +.uv-python/ +.pre-commit-cache/ diff --git a/README.md b/README.md index dd9d43e..310bc56 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A lightweight CLI for scheduling LLM evaluations across multiple HPC clusters us - **Collect results** and check for missing evaluations: `oellm collect-results` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Snellius) -- **Automatic building and deployment of containers** +- **Automatic building and deployment of containers** ## Quick Start @@ -42,16 +42,24 @@ In case you do not want to rely on the containers provided on a given cluster or ## Task Groups -Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/resources/task-groups.yaml). Each group specifies tasks, their n-shot settings, and HuggingFace dataset mappings. +Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/resources/task-groups.yaml). Each group specifies tasks, their n-shot settings, and HuggingFace dataset mappings. See [docs/TASKS.md](docs/TASKS.md) for the task-group schema and multilingual smoke-test notes. Available task groups: - `open-sci-0.01` - Standard benchmarks (COPA, MMLU, HellaSwag, ARC, etc.) - `belebele-eu-5-shot` - Belebele European language tasks -- `flores-200-eu-to-eng` / `flores-200-eng-to-eu` - Translation tasks +- `belebele-eu-cf` - Belebele European cloze-formulation tasks for lighteval +- `flores-200-eu-to-eng` - Flores 200 European languages to English translation +- `flores-200-eng-to-eu` - Flores 200 English to European languages translation - `global-mmlu-eu` - Global MMLU in EU languages - `mgsm-eu` - Multilingual GSM benchmarks +- `xcsqa` - XCSQA commonsense reasoning set +- `global-mgsm` - Global-MGSM multilingual math benchmarks +- `polymath` - PolyMath multilingual math set +- `global-piqa` - Global PIQA commonsense reasoning set - `generic-multilingual` - XWinograd, XCOPA, XStoryCloze - `include` - INCLUDE benchmarks +- `dclm-core-22` - DCLM core 22 evaluation tasks +- `reasoning` - GSM8k, IFEval, MBPP, GPQA-Diamond, MATH500, LiveCodeBench Super groups combine multiple task groups: - `oellm-multilingual` - All multilingual benchmarks combined @@ -72,8 +80,9 @@ oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" The `--local` flag lets you run evaluations directly on your machine without a cluster or Singularity container. It generates the same eval script and executes it with bash, injecting fake SLURM environment variables so all tasks run sequentially in a single process. This is useful for testing that tasks and models are correctly configured before submitting to a cluster. ```bash -# 1. Add eval dependencies to the project venv -uv pip install lm-eval torch transformers accelerate "datasets<4.0.0" +# 1. Create a venv and add lm-eval dependencies +uv venv --python 3.12 .venv +uv pip install --python .venv/bin/python -r requirements-venv.txt # 2. Run evaluations locally — useful for smoke-testing with a small sample oellm schedule-eval \ @@ -87,6 +96,10 @@ oellm schedule-eval \ Results are written to `./oellm-output//results/`. +For lighteval tasks, install lighteval as an isolated uv tool so its +`datasets>=4.0.0` dependency does not conflict with lm-eval's `datasets<4.0.0`; +see [docs/VENV.md](docs/VENV.md). + **Air-gapped cluster nodes (no internet):** batch jobs set `HF_HUB_OFFLINE=1` and get `HF_HOME` from your cluster env. With `--local`, the CLI defaults `HF_HOME` to `~/.cache/huggingface` if unset and would otherwise allow Hub access—so on a compute node without network, export your real cache and offline flag before running, for example: ```bash diff --git a/docs/TASKS.md b/docs/TASKS.md index 6ed754c..7d8fdd2 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -62,3 +62,15 @@ oellm schedule-eval --models "model-name" --task_groups "my-benchmark" 2. **CI testing** - The test suite validates that all datasets in `task-groups.yaml` are accessible Tasks without a `dataset` field will not have their data pre-downloaded and are not covered by CI validation. + +## How to add custom lm-eval YAMLs + +There are two related pieces of configuration to be aware of: + +- `oellm/resources/task-groups.yaml` — oellm's scheduler registry. It lists the task names to run, the evaluation suite (`lm-eval-harness`, `lighteval`, or `evalchemy`), few-shot counts, and which dataset should be pre-downloaded. +- The evaluation-suite task registry (lm-eval or lighteval) — defines how a task is executed: which dataset split to load, how prompts are formatted, how answers are extracted, and which metric to compute. + +If you add a task name to `task-groups.yaml`, the evaluation suite also needs a task definition for that name. If not, place custom task YAMLs in `oellm/resources/custom_lm_eval_tasks`. Then `main.py` ... +If you add a task name to `task-groups.yaml`, the evaluation suite also needs a task definition for that name. If not, place custom task YAMLs in `oellm/resources/custom_lm_eval_tasks`. + +Then `main.py` will include that directory when generating the evaluation script: the `schedule_evals` command sets `lm_eval_include_path` (defaulting to the bundled `oellm/resources/custom_lm_eval_tasks`), and the generated SLURM script passes it to lm_eval as `--include_path`. This makes lm_eval aware of custom task YAMLs such as `polymath_en_low`. See `oellm/main.py` and `oellm/resources/template.sbatch` for the wiring. diff --git a/docs/VENV.md b/docs/VENV.md index eb04c88..d790c9a 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -76,7 +76,7 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a 3. Run with `EVALCHEMY_DIR` pointing to the cloned repo: ```bash - export HF_ALLOW_CODE_EVAL=1 # required by MBPP + export HF_ALLOW_CODE_EVAL=1 # required by MBPP EVALCHEMY_DIR=$(pwd)/evalchemy oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M \ --task_groups reasoning \ diff --git a/oellm/main.py b/oellm/main.py index 6a19984..b848dc9 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -31,13 +31,19 @@ capture_third_party_output_from_kwarg, ) +_SLURM_FAILURE_RE = re.compile( + r"Traceback|Exception|Error|FAILED|Killed|OOM|out of memory|No such file|No module|command not found", + re.IGNORECASE, +) + def _resolve_hf_hub_offline(local: bool) -> int: """Value embedded in the generated eval script as HF_HUB_OFFLINE. If ``HF_HUB_OFFLINE`` is set in the environment when ``oellm`` runs, that - value wins. Otherwise defaults to online Hub access for ``--local`` - (typical laptop dev) and offline for SLURM jobs (air-gapped workers). + value wins. Otherwise generated eval scripts default to offline Hub access + because model and dataset checks/downloads have already happened before the + script runs. """ raw = os.environ.get("HF_HUB_OFFLINE") if raw is not None and str(raw).strip() != "": @@ -45,7 +51,7 @@ def _resolve_hf_hub_offline(local: bool) -> int: return int(str(raw).strip()) except ValueError: logging.warning("Invalid HF_HUB_OFFLINE=%r; using default", raw) - return 0 if local else 1 + return 1 def _resolve_slurm_mem() -> str: @@ -437,6 +443,7 @@ def schedule_evals( hf_hub_offline=_resolve_hf_hub_offline(local), additional_model_args=_resolve_additional_model_args(local), # Batch size evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), + oellm_repo_root=str(Path(__file__).resolve().parents[1]), ) # substitute any $ENV_VAR occurrences @@ -482,6 +489,7 @@ def schedule_evals( logging.info("Local evaluation completed.") except subprocess.CalledProcessError as e: logging.error(f"Evaluation failed with exit code {e.returncode}") + raise return try: @@ -577,6 +585,12 @@ def _first_matching_prefix( val, key = _first_matching_prefix(result_dict, metric.split(",")[0]) if val is not None: return val, key + + # Add a check for majority voting patterns (e.g., maj@4, maj@8) + for k, v in result_dict.items(): + if k.startswith("maj@") and isinstance(v, (int, float)): + return float(v), k + return None, None def _split_task_and_nshot(name: str) -> tuple[str, int | None]: @@ -589,6 +603,31 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: return base, int(after) return name, None + def _aggregate_group_from_subtasks( + group_name: str, + group_subtasks: list[str], + all_results: dict, + ) -> tuple[float | None, str | None]: + """Fallback for lm-eval groups whose aggregate row has no metric payload.""" + metric_values: list[float] = [] + metric_name: str | None = None + + for subtask_name in group_subtasks: + subtask_results = all_results.get(subtask_name) + if not isinstance(subtask_results, dict): + continue + value, resolved_metric = _resolve_metric(group_name, subtask_results) + if value is None: + continue + metric_values.append(value) + if metric_name is None: + metric_name = resolved_metric + + if not metric_values: + return None, None + + return float(sum(metric_values) / len(metric_values)), metric_name + results_path = Path(results_dir) if not results_path.exists(): raise ValueError(f"Results directory does not exist: {results_dir}") @@ -618,6 +657,42 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: jobs_df = pd.read_csv(jobs_csv_path) logging.info(f"Found {len(jobs_df)} scheduled jobs in jobs.csv") + submit_script = results_path / "submit_evals.sbatch" + slurm_logs_dir = results_path / "slurm_logs" + num_array_jobs = len(jobs_df) + total_evals = len(jobs_df) + if submit_script.exists(): + submit_text = submit_script.read_text(errors="replace") + num_jobs_match = re.search(r"^NUM_JOBS=(\d+)$", submit_text, re.M) + total_evals_match = re.search(r"^TOTAL_EVALS=(\d+)$", submit_text, re.M) + if num_jobs_match: + num_array_jobs = int(num_jobs_match.group(1)) + if total_evals_match: + total_evals = int(total_evals_match.group(1)) + evals_per_array_job = max( + 1, int(math.ceil(total_evals / max(1, num_array_jobs))) + ) + + def _slurm_status_for_job(row_index: int) -> str | None: + if not slurm_logs_dir.exists(): + return None + + array_index = row_index // evals_per_array_job + matching_logs = list(slurm_logs_dir.glob(f"*-{array_index}.out")) + list( + slurm_logs_dir.glob(f"*-{array_index}.err") + ) + if not matching_logs: + return "not_started" + + combined = "\n".join( + p.read_text(errors="replace") for p in matching_logs if p.exists() + ) + if _SLURM_FAILURE_RE.search(combined): + return "failed" + if f"Job {array_index} finished." in combined: + return "finished_without_result" + return "started_without_result" + # Collect results rows = [] completed_jobs = set() # Track (model, task, n_shot) tuples @@ -633,6 +708,7 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: or data.get("config_general", {}).get("model_name") or data.get("config_general", {}).get("model") or data.get("config_general", {}).get("model_path") + or data.get("config_general", {}).get("model_config", {}).get("model_name") or data.get("summary_general", {}).get("model") or data.get("model") or "unknown" @@ -666,26 +742,35 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: for _s in _subs: group_subtask_names.add(_s) - # Prefer only the first aggregate metric from groups (simplified) + group_rows_added = 0 if groups_map: - group_name, group_results = next(iter(groups_map.items())) - # Prefer original extraction from n_shot_data and subtasks, then - # global_n_shot; only fall back to parsing the group name. - orig_group_name = group_name - n_shot = n_shot_data.get(orig_group_name, "unknown") - if n_shot == "unknown": - for subtask_name in group_subtasks_map.get(orig_group_name, []): - if subtask_name in n_shot_data: - n_shot = n_shot_data[subtask_name] - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - # Fallback: parse possible '|N' suffix from group name - group_name, parsed_n = _split_task_and_nshot(orig_group_name) - if n_shot == "unknown" and parsed_n is not None: - n_shot = parsed_n - performance, metric_name = _resolve_metric(group_name, group_results) - if performance is not None: + for orig_group_name, group_results in groups_map.items(): + # Prefer original extraction from n_shot_data and subtasks, then + # global_n_shot; only fall back to parsing the group name. + n_shot = n_shot_data.get(orig_group_name, "unknown") + if n_shot == "unknown": + for subtask_name in group_subtasks_map.get(orig_group_name, []): + if subtask_name in n_shot_data: + n_shot = n_shot_data[subtask_name] + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + + group_name, parsed_n = _split_task_and_nshot(orig_group_name) + if n_shot == "unknown" and parsed_n is not None: + n_shot = parsed_n + + performance, metric_name = _resolve_metric(group_name, group_results) + if performance is None: + performance, metric_name = _aggregate_group_from_subtasks( + group_name, + group_subtasks_map.get(orig_group_name, []), + results, + ) + + if performance is None: + continue + if check: completed_jobs.add((model_name, group_name, n_shot)) rows.append( @@ -697,7 +782,10 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: "metric_name": metric_name if metric_name is not None else "", } ) - # Skip per-task iteration when groups are present + group_rows_added += 1 + + if group_rows_added: + # Skip per-task iteration when group aggregates were extracted continue for task_name, task_results in results.items(): @@ -712,6 +800,11 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: if task_name.startswith("mmlu_") and task_name != "mmlu": continue + # lighteval writes a synthetic aggregate named "all" next to the + # concrete task; keep the concrete task rows for oellm job checks. + if task_name == "all": + continue + # Skip Global MMLU subtasks - keep only aggregates like global_mmlu_full_pt if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: continue @@ -720,9 +813,15 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: # Prefer original extraction from `n_shot_data` and `global_n_shot`, # and fall back to parsing a '|N' suffix in the task name. task_name_clean, parsed_n = _split_task_and_nshot(task_name) - n_shot = ( - n_shot_data.get(task_name_clean) or global_n_shot or parsed_n or "unknown" - ) + n_shot = n_shot_data.get(task_name_clean) + if n_shot is None: + n_shot = n_shot_data.get(task_name) + if n_shot is None: + n_shot = global_n_shot + if n_shot is None: + n_shot = parsed_n + if n_shot is None: + n_shot = "unknown" # If this is a group aggregate and n_shot is missing, derive from any subtask if task_name_clean in group_aggregate_names and n_shot == "unknown": @@ -802,8 +901,9 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: # Find missing jobs missing_jobs = [] + pending_jobs = [] - for _, job in jobs_df.iterrows(): + for row_index, job in jobs_df.iterrows(): job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) # Check if this job corresponds to one of our completed results @@ -829,14 +929,31 @@ def _split_task_and_nshot(name: str) -> tuple[str, int | None]: break if not is_completed: - missing_jobs.append(job) - - completed_count = len(jobs_df) - len(missing_jobs) + slurm_status = _slurm_status_for_job(row_index) + if slurm_status == "not_started": + pending_job = job.copy() + pending_job["status"] = slurm_status + pending_jobs.append(pending_job) + else: + missing_job = job.copy() + if slurm_status: + missing_job["status"] = slurm_status + missing_jobs.append(missing_job) + + completed_count = len(jobs_df) - len(missing_jobs) - len(pending_jobs) logging.info(f"Total scheduled jobs: {len(jobs_df)}") logging.info(f"Completed jobs: {completed_count}") + if pending_jobs: + logging.info(f"Not-started jobs: {len(pending_jobs)}") logging.info(f"Missing jobs: {len(missing_jobs)}") + if len(pending_jobs) > 0: + pending_df = pd.DataFrame(pending_jobs) + pending_csv = output_csv.replace(".csv", "_pending.csv") + pending_df.to_csv(pending_csv, index=False) + logging.info(f"Not-started jobs saved to: {pending_csv}") + if len(missing_jobs) > 0: missing_df = pd.DataFrame(missing_jobs) missing_csv = output_csv.replace(".csv", "_missing.csv") diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml new file mode 100644 index 0000000..742fa06 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_am.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_am +dataset_name: am diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml new file mode 100644 index 0000000..69584f7 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ar.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ar +dataset_name: ar diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml new file mode 100644 index 0000000..69316d2 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_bn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_bn +dataset_name: bn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml new file mode 100644 index 0000000..4e88ef1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ca.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ca +dataset_name: ca diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml new file mode 100644 index 0000000..bbd7e54 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cs.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_cs +dataset_name: cs diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml new file mode 100644 index 0000000..f63b0f3 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_cy.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_cy +dataset_name: cy diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml new file mode 100644 index 0000000..91f8d21 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_de.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_de +dataset_name: de diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml new file mode 100644 index 0000000..de5ee04 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_el.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_el +dataset_name: el diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml new file mode 100644 index 0000000..9bfa7e1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_en.yaml @@ -0,0 +1,35 @@ +tag: + - global_mgsm +task: global_mgsm_en +dataset_path: CohereLabs/global-mgsm +dataset_name: en +output_type: generate_until +test_split: test +doc_to_text: "{{instruction}}\n\nQuestion: {{question}}\n{{answer_prefix}}:" +doc_to_target: "{{answer}}" +target_delimiter: " " +generation_kwargs: + until: + - "\n" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +filter_list: + - name: flexible-extract + filter: + - function: regex + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml new file mode 100644 index 0000000..585a6e4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_es.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_es +dataset_name: es diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml new file mode 100644 index 0000000..94443e8 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_eu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_eu +dataset_name: eu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml new file mode 100644 index 0000000..d6ed75f --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_fr.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_fr +dataset_name: fr diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml new file mode 100644 index 0000000..02f9c11 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gl.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_gl +dataset_name: gl diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml new file mode 100644 index 0000000..f123cf5 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_gu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_gu +dataset_name: gu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml new file mode 100644 index 0000000..93d4d9a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ha.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ha +dataset_name: ha diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml new file mode 100644 index 0000000..751a78a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_hu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_hu +dataset_name: hu diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml new file mode 100644 index 0000000..e2cc17a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ja.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ja +dataset_name: ja diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml new file mode 100644 index 0000000..a629afe --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_km.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_km +dataset_name: km diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml new file mode 100644 index 0000000..3677e26 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_kn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_kn +dataset_name: kn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml new file mode 100644 index 0000000..1a16bd2 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ko.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ko +dataset_name: ko diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml new file mode 100644 index 0000000..e68f141 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ky.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ky +dataset_name: ky diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml new file mode 100644 index 0000000..76a0e6e --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_lg.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_lg +dataset_name: lg diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml new file mode 100644 index 0000000..d9fe17b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_my.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_my +dataset_name: my diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml new file mode 100644 index 0000000..fb6f1e0 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ne.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ne +dataset_name: ne diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml new file mode 100644 index 0000000..dcfc0c4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ru.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ru +dataset_name: ru diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml new file mode 100644 index 0000000..772995c --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_si.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_si +dataset_name: si diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml new file mode 100644 index 0000000..f44d119 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sn.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sn +dataset_name: sn diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml new file mode 100644 index 0000000..f44d091 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sr.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sr +dataset_name: sr diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml new file mode 100644 index 0000000..e33b2b7 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_st.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_st +dataset_name: st diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml new file mode 100644 index 0000000..a14b6f3 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_sw.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_sw +dataset_name: sw diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml new file mode 100644 index 0000000..1c933fc --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ta.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ta +dataset_name: ta diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml new file mode 100644 index 0000000..e326391 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_te.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_te +dataset_name: te diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml new file mode 100644 index 0000000..f33ebe1 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_th.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_th +dataset_name: th diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml new file mode 100644 index 0000000..19e15e6 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_ur.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_ur +dataset_name: ur diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml new file mode 100644 index 0000000..01c7645 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_uz.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_uz +dataset_name: uz diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml new file mode 100644 index 0000000..035cf67 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_vi.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_vi +dataset_name: vi diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml new file mode 100644 index 0000000..3f4c403 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_wo.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_wo +dataset_name: wo diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml new file mode 100644 index 0000000..1493a21 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_xh.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_xh +dataset_name: xh diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml new file mode 100644 index 0000000..078d9fd --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_yo.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_yo +dataset_name: yo diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml new file mode 100644 index 0000000..8b0024a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zh.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_zh +dataset_name: zh diff --git a/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml new file mode 100644 index 0000000..2a6dd5b --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/global_mgsm_zu.yaml @@ -0,0 +1,3 @@ +include: global_mgsm_en.yaml +task: global_mgsm_zu +dataset_name: zu diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml new file mode 100644 index 0000000..17c7c73 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_de_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_de_low +dataset_name: de diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml new file mode 100644 index 0000000..b077aef --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_en_low.yaml @@ -0,0 +1,35 @@ +tag: + - polymath +task: polymath_en_low +dataset_path: Qwen/PolyMath +dataset_name: en +output_type: generate_until +test_split: low +doc_to_text: "{{question}}\nAnswer:" +doc_to_target: "{{answer}}" +target_delimiter: " " +generation_kwargs: + until: + - "\n" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +filter_list: + - name: flexible-extract + filter: + - function: regex + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml new file mode 100644 index 0000000..83a3884 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_es_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_es_low +dataset_name: es diff --git a/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml b/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml new file mode 100644 index 0000000..3b985a6 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/polymath_fr_low.yaml @@ -0,0 +1,3 @@ +include: polymath_en_low.yaml +task: polymath_fr_low +dataset_name: fr diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 0aa5d12..63ae6de 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -32,10 +32,24 @@ task_metrics: winogrande: acc arc_challenge: acc_norm arc_easy: acc_norm + arc_mt: acc_norm boolq: acc commonsense_qa: acc hellaswag: acc_norm piqa: acc_norm + global_piqa_prompted: acc + sib: acc + multiblimp: acc + opensubtitles_multi40_pair: acc + xcsqa_eng_mcf: acc_norm + # math:algebra: maj@4 + # math:counting_and_probability: maj@4 + # math:geometry: maj@4 + # math:intermediate_algebra: maj@4 + # math:number_theory: maj@4 + # math:prealgebra: maj@4 + # math:precalculus: maj@4 + math: maj@4 social_iqa: acc agieval_lsat_ar: acc wsc273: acc @@ -47,6 +61,55 @@ task_metrics: bigbench_operators_generate_until: exact_match bigbench_repeat_copy_logic_generate_until: exact_match bigbench_cs_algorithms_generate_until: exact_match + global_mgsm_am: exact_match + global_mgsm_ar: exact_match + global_mgsm_bn: exact_match + global_mgsm_ca: exact_match + global_mgsm_cs: exact_match + global_mgsm_cy: exact_match + global_mgsm_de: exact_match + global_mgsm_el: exact_match + global_mgsm_en: exact_match + global_mgsm_es: exact_match + global_mgsm_eu: exact_match + global_mgsm_fr: exact_match + global_mgsm_gl: exact_match + global_mgsm_gu: exact_match + global_mgsm_ha: exact_match + global_mgsm_hu: exact_match + global_mgsm_ja: exact_match + global_mgsm_km: exact_match + global_mgsm_kn: exact_match + global_mgsm_ko: exact_match + global_mgsm_ky: exact_match + global_mgsm_lg: exact_match + global_mgsm_my: exact_match + global_mgsm_ne: exact_match + global_mgsm_ru: exact_match + global_mgsm_si: exact_match + global_mgsm_sn: exact_match + global_mgsm_sr: exact_match + global_mgsm_st: exact_match + global_mgsm_sw: exact_match + global_mgsm_ta: exact_match + global_mgsm_te: exact_match + global_mgsm_th: exact_match + global_mgsm_ur: exact_match + global_mgsm_uz: exact_match + global_mgsm_vi: exact_match + global_mgsm_wo: exact_match + global_mgsm_xh: exact_match + global_mgsm_yo: exact_match + global_mgsm_zh: exact_match + global_mgsm_zu: exact_match + polymath_en_low: exact_match + polymath_de_low: exact_match + polymath_es_low: exact_match + polymath_fr_low: exact_match + global_piqa_completions_eng_latn: acc_norm + global_piqa_completions_deu_latn: acc_norm + global_piqa_completions_spa_latn_spai: acc_norm + global_piqa_completions_fra_latn_fran: acc_norm task_groups: open-sci-0.01: @@ -84,6 +147,9 @@ task_groups: n_shots: [10] dataset: allenai/ai2_arc subset: ARC-Challenge + - task: arc_mt + n_shots: [10] + dataset: LumiOpen/arc_challenge_mt - task: commonsense_qa n_shots: [10] dataset: tau/commonsense_qa @@ -206,7 +272,7 @@ task_groups: subset: eus_Latn - task: belebele_cat_Latn_cf subset: cat_Latn - + flores-200-eu-to-eng: description: "Flores 200 EU to English translation" suite: lighteval @@ -311,16 +377,239 @@ task_groups: description: "EU Language GSM benchmarks in Aya Expanse" suite: lm-eval-harness n_shots: [5] - dataset: jbross-ibm-research/mgsm tasks: + # Keep Greek omitted due to non-commercial license constraints. + - task: mgsm_native_cot_bn + dataset: juletxara/mgsm + subset: bn - task: mgsm_native_cot_en + dataset: juletxara/mgsm subset: en - task: mgsm_native_cot_de + dataset: juletxara/mgsm subset: de - task: mgsm_native_cot_es + dataset: juletxara/mgsm subset: es - task: mgsm_native_cot_fr + dataset: juletxara/mgsm subset: fr + - task: mgsm_native_cot_ja + dataset: juletxara/mgsm + subset: ja + - task: mgsm_native_cot_ru + dataset: juletxara/mgsm + subset: ru + - task: mgsm_native_cot_sw + dataset: juletxara/mgsm + subset: sw + - task: mgsm_native_cot_te + dataset: juletxara/mgsm + subset: te + - task: mgsm_native_cot_th + dataset: juletxara/mgsm + subset: th + - task: mgsm_native_cot_zh + dataset: juletxara/mgsm + subset: zh + + sib-200: + description: "SIB-200 sentence topic classification" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: sib + + multiblimp: + description: "MultiBLiMP pairwise sentence ranking" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: multiblimp + + xcsqa: + description: "Cross-lingual CSQA in LightEval (MCF formulation)" + suite: lighteval + n_shots: [0] + tasks: + # Explicit dataset metadata lets `--download_only` warm the same cached + # configs that lighteval will request at runtime. + - task: xcsqa_eng_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-en + - task: xcsqa_deu_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-de + - task: xcsqa_spa_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-es + - task: xcsqa_fra_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-fr + - task: xcsqa_ita_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-it + - task: xcsqa_pol_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-pl + - task: xcsqa_por_mcf + dataset: INK-USC/xcsr + subset: X-CSQA-pt + + open-subtitles-x-to-eng: + description: "OpenSubtitles translation (x to English), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # Use lm-eval OpenSubtitles dynamic pair task format (PR #3706). + # Keep mirrored groups to preserve directional report slots. + - task: opensubtitles_multi40_pair + + open-subtitles-eng-to-x: + description: "OpenSubtitles translation (English to x), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: opensubtitles_multi40_pair + + doclevel-mt-x-to-eng: + description: "Document-level MT proxy (x to English), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # TODO: Replace with dedicated doc-level MT tasks when available in lm-eval. + - task: wmt14-fr-en + - task: wmt16-de-en + - task: wmt16-ro-en + + doclevel-mt-eng-to-x: + description: "Document-level MT proxy (English to x), directional reporting group" + suite: lm-eval-harness + n_shots: [0] + tasks: + # TODO: Replace with dedicated doc-level MT tasks when available in lm-eval. + - task: wmt14-en-fr + - task: wmt16-en-de + - task: wmt16-en-ro + + global-mgsm: + description: "Global-MGSM multilingual math benchmarks" + suite: lm-eval-harness + n_shots: [0] + dataset: CohereLabs/global-mgsm + tasks: + - task: global_mgsm_bn + subset: bn + - task: global_mgsm_de + subset: de + - task: global_mgsm_en + subset: en + - task: global_mgsm_es + subset: es + - task: global_mgsm_fr + subset: fr + - task: global_mgsm_ja + subset: ja + - task: global_mgsm_ru + subset: ru + - task: global_mgsm_sw + subset: sw + - task: global_mgsm_te + subset: te + - task: global_mgsm_th + subset: th + - task: global_mgsm_zh + subset: zh + - task: global_mgsm_ar + subset: ar + - task: global_mgsm_ca + subset: ca + - task: global_mgsm_cs + subset: cs + - task: global_mgsm_cy + subset: cy + - task: global_mgsm_el + subset: el + - task: global_mgsm_eu + subset: eu + - task: global_mgsm_gl + subset: gl + - task: global_mgsm_hu + subset: hu + - task: global_mgsm_ko + subset: ko + - task: global_mgsm_sr + subset: sr + - task: global_mgsm_vi + subset: vi + - task: global_mgsm_am + subset: am + - task: global_mgsm_ha + subset: ha + - task: global_mgsm_lg + subset: lg + - task: global_mgsm_sn + subset: sn + - task: global_mgsm_st + subset: st + - task: global_mgsm_wo + subset: wo + - task: global_mgsm_xh + subset: xh + - task: global_mgsm_yo + subset: yo + - task: global_mgsm_zu + subset: zu + - task: global_mgsm_ur + subset: ur + - task: global_mgsm_gu + subset: gu + - task: global_mgsm_km + subset: km + - task: global_mgsm_kn + subset: kn + - task: global_mgsm_ky + subset: ky + - task: global_mgsm_my + subset: my + - task: global_mgsm_ne + subset: ne + - task: global_mgsm_si + subset: si + - task: global_mgsm_ta + subset: ta + - task: global_mgsm_uz + subset: uz + + polymath: + description: "PolyMath multilingual math set (low difficulty)" + suite: lm-eval-harness + n_shots: [0] + dataset: Qwen/PolyMath + tasks: + - task: polymath_en_low + subset: en + - task: polymath_de_low + subset: de + - task: polymath_es_low + subset: es + - task: polymath_fr_low + subset: fr + + global-piqa: + description: "Global PIQA commonsense reasoning set" + suite: lm-eval-harness + n_shots: [0] + dataset: mrlbenchmarks/global-piqa-nonparallel + tasks: + - task: global_piqa_completions_eng_latn + subset: eng_latn + - task: global_piqa_completions_deu_latn + subset: deu_latn + - task: global_piqa_completions_spa_latn_spai + subset: spa_latn_spai + - task: global_piqa_completions_fra_latn_fran + subset: fra_latn_fran generic-multilingual: description: "Generic multilingual benchmarks in Aya Expanse" @@ -391,6 +680,9 @@ task_groups: - task: include_base_44_ukrainian subset: Ukrainian + # NOTE: HellaSwag has translation variants with NC license constraints upstream. + # Keep using the canonical `hellaswag` task for unrestricted baseline reporting. + dclm-core-22: description: "DCLM core 22 evaluation tasks (lm-eval-harness, matching LLM Foundry task types)" suite: lm-eval-harness @@ -406,6 +698,9 @@ task_groups: n_shots: [10] dataset: allenai/ai2_arc subset: ARC-Challenge + - task: arc_mt + n_shots: [10] + dataset: LumiOpen/arc_challenge_mt - task: boolq n_shots: [10] dataset: aps/super_glue @@ -499,12 +794,22 @@ task_groups: super_groups: oellm-multilingual: - description: "Combined Belebele EU set plus multilingual benchmarks" + description: "Combined multilingual benchmark suite" task_groups: - task: flores-200-eu-to-eng - task: flores-200-eng-to-eu - task: belebele-eu-5-shot + - task: belebele-eu-cf - task: global-mmlu-eu - task: mgsm-eu + - task: global-piqa + - task: sib-200 + - task: multiblimp + - task: xcsqa + - task: polymath + - task: open-subtitles-x-to-eng + - task: open-subtitles-eng-to-x + - task: doclevel-mt-x-to-eng + - task: doclevel-mt-eng-to-x - task: generic-multilingual - task: include diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 38ecfa2..7507074 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -9,6 +9,9 @@ #SBATCH --error={log_dir}/%x-%A-%a.err #SBATCH --array=0-{array_limit}%{max_array_len} +# SBATCH directives are ignored when this script is run directly with bash. +set -euo pipefail + CSV_PATH="{csv_path}" NUM_JOBS={num_jobs} @@ -16,6 +19,7 @@ TOTAL_EVALS={total_evals} LIMIT="{limit}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" +OELLM_REPO_ROOT="{oellm_repo_root}" # avoiding crashes due to compute nodes not having access to the internet export HF_HOME=$HF_HOME @@ -25,6 +29,14 @@ export HF_HUB_OFFLINE={hf_hub_offline} # Path to the shared Singularity image that contains all runtime deps (container mode) export EVAL_SIF_PATH="$EVAL_BASE_DIR/$EVAL_CONTAINER_IMAGE" +if [ -n "$OELLM_REPO_ROOT" ] && [ -d "$OELLM_REPO_ROOT" ]; then + if [ -n "${{PYTHONPATH:-}}" ]; then + export PYTHONPATH="$OELLM_REPO_ROOT:$PYTHONPATH" + else + export PYTHONPATH="$OELLM_REPO_ROOT" + fi +fi + echo "Running eval on $CSV_PATH with $NUM_JOBS array jobs distributing $TOTAL_EVALS evaluations" echo "SLURM_JOB_ID: $SLURM_JOB_ID" echo "SLURM_ARRAY_JOB_ID: $SLURM_ARRAY_JOB_ID" @@ -81,6 +93,10 @@ do # bind the directory that contains the model checkpoint when the path exists BIND_PATHS="$EVAL_BASE_DIR:$EVAL_BASE_DIR,$HF_HOME:$HF_HOME,$HF_DATASETS_CACHE:$HF_DATASETS_CACHE" + if [ -d "$OELLM_REPO_ROOT" ] && [[ $OELLM_REPO_ROOT != $EVAL_BASE_DIR* ]]; then + BIND_PATHS="$BIND_PATHS,$OELLM_REPO_ROOT:$OELLM_REPO_ROOT" + fi + if [ -e "$model_path" ]; then # If the model_path is a file, bind its parent directory; otherwise bind the dir itself MODEL_DIR="$model_path" diff --git a/oellm/utils.py b/oellm/utils.py index 4b6830c..1195306 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -302,6 +302,7 @@ def _process_model_paths(models: Iterable[str]): def _pre_download_datasets_from_specs( specs: Iterable, trust_remote_code: bool = True ) -> None: + import datasets from datasets import get_dataset_config_names, load_dataset specs_list = list(specs) @@ -309,6 +310,31 @@ def _pre_download_datasets_from_specs( return console = get_console() + datasets_major_version = int(datasets.__version__.split(".", 1)[0]) + supports_trust_remote_code = datasets_major_version < 4 + + def _load_dataset_compat(repo_id: str, name: str | None = None): + kwargs = {"name": name} + if trust_remote_code and supports_trust_remote_code: + kwargs["trust_remote_code"] = trust_remote_code + try: + return load_dataset(repo_id, **kwargs) + except ValueError as e: + if "trust_remote_code" not in str(e): + raise + kwargs.pop("trust_remote_code", None) + return load_dataset(repo_id, **kwargs) + + def _get_dataset_config_names_compat(repo_id: str): + kwargs = {} + if trust_remote_code and supports_trust_remote_code: + kwargs["trust_remote_code"] = trust_remote_code + try: + return get_dataset_config_names(repo_id, **kwargs) + except ValueError as e: + if "trust_remote_code" not in str(e): + raise + return get_dataset_config_names(repo_id) with console.status( f"Downloading datasets… {len(specs_list)} datasets", @@ -319,16 +345,10 @@ def _pre_download_datasets_from_specs( status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") try: - load_dataset( - spec.repo_id, - name=spec.subset, - trust_remote_code=trust_remote_code, - ) + _load_dataset_compat(spec.repo_id, name=spec.subset) except ValueError as e: if "Config name is missing" in str(e) and spec.subset is None: - configs = get_dataset_config_names( - spec.repo_id, trust_remote_code=trust_remote_code - ) + configs = _get_dataset_config_names_compat(spec.repo_id) logging.info( f"Dataset '{spec.repo_id}' requires config. " f"Downloading all {len(configs)} configs." @@ -337,11 +357,7 @@ def _pre_download_datasets_from_specs( status.update( f"Downloading '{spec.repo_id}/{cfg}' ({idx}/{len(specs_list)})" ) - load_dataset( - spec.repo_id, - name=cfg, - trust_remote_code=trust_remote_code, - ) + _load_dataset_compat(spec.repo_id, name=cfg) continue if "Feature type" in str(e) and "not found" in str(e): hf_datasets_cache = os.environ.get( diff --git a/pyproject.toml b/pyproject.toml index 4d94f44..ee44267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "rich", "huggingface_hub", "pyyaml", + "transformers==4.49", ] [project.optional-dependencies] diff --git a/requirements-venv-evalchemy.txt b/requirements-venv-evalchemy.txt index 9ec7d3d..c63c3fb 100644 --- a/requirements-venv-evalchemy.txt +++ b/requirements-venv-evalchemy.txt @@ -1,6 +1,6 @@ # Dependencies for evalchemy evaluation -# lm-eval fork used by evalchemy +# lm-eval fork used by evalchemy lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix scipy==1.17.0 diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index ad6152b..6451978 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -64,6 +64,42 @@ def test_schedule_evals_slurm_template_var_overrides(tmp_path): assert "#SBATCH --gres=gpu:2" in sbatch_content +def test_schedule_evals_generated_script_defaults_to_offline_hub(tmp_path): + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict( + os.environ, + { + "EVAL_OUTPUT_DIR": str(tmp_path), + "PARTITION": "default_partition", + "ACCOUNT": "test_account", + }, + clear=False, + ), + ): + os.environ.pop("HF_HUB_OFFLINE", None) + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + assert "set -euo pipefail" in sbatch_content + assert "export HF_HUB_OFFLINE=1" in sbatch_content + assert "run_python -m lm_eval" in sbatch_content + assert 'OELLM_REPO_ROOT="' in sbatch_content + assert sbatch_content.index("#SBATCH --job-name=") < sbatch_content.index( + "set -euo pipefail" + ) + + def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): """Verify invalid slurm_template_var raises ValueError.""" with ( diff --git a/tests/test_utils.py b/tests/test_utils.py index ebc0163..36b7d46 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,7 @@ +import sys +from types import SimpleNamespace + +from oellm.task_groups import DatasetSpec from oellm.utils import _expand_local_model_paths, _num_jobs_in_queue @@ -54,3 +58,26 @@ class Result: monkeypatch.setattr("oellm.utils.subprocess.run", lambda *a, **kw: Result()) assert _num_jobs_in_queue() == 0 + + +class TestPreDownloadDatasets: + def test_omits_trust_remote_code_for_datasets_v4(self, monkeypatch): + calls = [] + + def load_dataset(repo_id, **kwargs): + calls.append((repo_id, kwargs)) + + fake_datasets = SimpleNamespace( + __version__="4.8.5", + load_dataset=load_dataset, + get_dataset_config_names=lambda *args, **kwargs: [], + ) + monkeypatch.setitem(sys.modules, "datasets", fake_datasets) + + from oellm.utils import _pre_download_datasets_from_specs + + _pre_download_datasets_from_specs( + [DatasetSpec(repo_id="example/data", subset="en")] + ) + + assert calls == [("example/data", {"name": "en"})]