[Create PR]:

- Remove --use_lmeval_batching arg

Author: arubique

examples/mmlu_benchmark/mmlu_benchmark.py

@@ -161,11 +161,6 @@ def parse_args():
         default=None,
         help="Pad predictions to this size with -inf (default: no padding, disco-public uses 31)",
     )
-    parser.add_argument(
-        "--use_lmeval_batching",
-        action="store_true",
-        help="Use lm-evaluation-harness batching for exact numerical match. This batches ALL requests together before computing logprobs.",
-    )
 
     return parser.parse_args()
 
@@ -564,7 +559,7 @@ def _resolve_data_path(data_path: str) -> str:
 
 
 def _apply_eval_config_from_repo(repo_path: Path, args: "argparse.Namespace") -> None:
-    """Load eval_config from repo; forbid passing --pca/--pad_to_size/--use_lmeval_batching, then set args from eval_config."""
+    """Load eval_config from repo; forbid passing --pca/--pad_to_size, then set args from eval_config."""
     config_path = repo_path / DISCO_CONFIG_FILENAME
     if not config_path.exists():
         return
@@ -579,8 +574,6 @@ def _apply_eval_config_from_repo(repo_path: Path, args: "argparse.Namespace") ->
         errors.append(f"do not pass --pca (model uses pca={eval_config['pca']})")
     if "pad_to_size" in eval_config and args.pad_to_size is not None:
         errors.append(f"do not pass --pad_to_size (model uses pad_to_size={eval_config['pad_to_size']})")
-    if "use_lmeval_batching" in eval_config and args.use_lmeval_batching:
-        errors.append("do not pass --use_lmeval_batching (model uses use_lmeval_batching=True)")
     if errors:
         raise ValueError("When using a DISCO model from the Hub, " + "; ".join(errors) + ". Omit these flags to use the model's eval_config.")
     # Require data_path to match model config (use_full_prompt is always True)
@@ -592,8 +585,6 @@ def _apply_eval_config_from_repo(repo_path: Path, args: "argparse.Namespace") ->
         args.pca = eval_config["pca"]
     if "pad_to_size" in eval_config:
         args.pad_to_size = eval_config["pad_to_size"]
-    if "use_lmeval_batching" in eval_config:
-        args.use_lmeval_batching = eval_config["use_lmeval_batching"]
 
 
 def _resolve_hf_disco_repo(
@@ -700,12 +691,10 @@ def main():
         num_workers=args.num_workers,
     )
 
-    # Optionally precompute logprobs using lm-eval batching for exact match
-    if args.use_lmeval_batching:
-        print("\nPrecomputing logprobs using lm-eval batching ...")
-        # Get task list for precomputation
-        task_list = list(tasks._anchor_tasks if hasattr(tasks, "_anchor_tasks") else tasks._tasks)
-        benchmark.precompute_all_logprobs_lmeval(task_list)
+    # Precompute logprobs using lm-eval batching for exact numerical match
+    print("\nPrecomputing logprobs using lm-eval batching ...")
+    task_list = list(tasks._anchor_tasks if hasattr(tasks, "_anchor_tasks") else tasks._tasks)
+    benchmark.precompute_all_logprobs_lmeval(task_list)
 
     # Run evaluation
     print("\nRunning evaluation...")