From 303ee64e2a7305cf091ac48f1cd4e55d04926ba8 Mon Sep 17 00:00:00 2001 From: jrausch Date: Tue, 14 Apr 2026 07:24:15 -0700 Subject: [PATCH 1/2] fix lm-eval version conflict: pin puzzletron requirements to 0.4.8; add version compat warning to lm_eval_hf.py Signed-off-by: jrausch --- examples/llm_eval/lm_eval_hf.py | 7 +++++++ examples/puzzletron/requirements.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/llm_eval/lm_eval_hf.py b/examples/llm_eval/lm_eval_hf.py index 405e8590a5..330bc31d1e 100755 --- a/examples/llm_eval/lm_eval_hf.py +++ b/examples/llm_eval/lm_eval_hf.py @@ -38,8 +38,15 @@ # limitations under the License. import warnings +import lm_eval from lm_eval import utils from lm_eval.__main__ import cli_evaluate, parse_eval_args, setup_parser + +if not lm_eval.__version__.startswith("0.4.8"): + warnings.warn( + f"lm_eval_hf.py is tested with lm-eval 0.4.8; found {lm_eval.__version__}. " + "Later versions may have incompatible API changes." + ) from lm_eval.api.model import T from lm_eval.models.huggingface import HFLM from quantization_utils import quantize_model diff --git a/examples/puzzletron/requirements.txt b/examples/puzzletron/requirements.txt index 0511fb473b..5ee167275d 100644 --- a/examples/puzzletron/requirements.txt +++ b/examples/puzzletron/requirements.txt @@ -1,3 +1,3 @@ -lm-eval==0.4.10 +lm-eval==0.4.8 math-verify ray From 746f6697177183da78585ea3b8f286872328f4f1 Mon Sep 17 00:00:00 2001 From: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com> Date: Tue, 14 Apr 2026 20:44:34 +0530 Subject: [PATCH 2/2] Fix code quality Signed-off-by: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com> --- examples/llm_eval/lm_eval_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_eval/lm_eval_hf.py b/examples/llm_eval/lm_eval_hf.py index 756003d00b..dbdf22d868 100755 --- a/examples/llm_eval/lm_eval_hf.py +++ b/examples/llm_eval/lm_eval_hf.py @@ -39,8 +39,8 @@ import contextlib import warnings -import lm_eval import datasets +import lm_eval from lm_eval import utils from lm_eval.__main__ import cli_evaluate, parse_eval_args, setup_parser