refactor(tasks): migrate 10 benchmarks from llm_judge to the verifiers pipeline (#1388)

* refactor(tasks): migrate 10 benchmarks from llm_judge to the verifiers pipeline

* fix(tasks): restore judge-failure fallbacks and judge system prompts in verifier migrations

Author: Luodian

lmms_eval/tasks/ami/utils.py

@@ -1,20 +1,16 @@
 import os
 import re
 import string
+import threading
 
 import numpy as np
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
-API_TYPE = os.getenv("API_TYPE", "openai")
 JUDGE_MODEL_VERSION = os.getenv("JUDGE_MODEL_VERSION", "gpt-4o-mini")
 
-server_config = ServerConfig(
-    model_name=JUDGE_MODEL_VERSION,
-)
-server = get_server(server_name=API_TYPE, config=server_config)
-
 
 def get_column_value(doc, candidates):
     """Helper function to get value from document with multiple possible column names"""
@@ -259,16 +255,7 @@ def calculate_wer(reference, hypothesis):
     return wer
 
 
-def ami_process_results_llm_judge(doc, results):
-    """
-    Process results using LLM as judge for open-ended evaluation.
-    Similar to voicebench's open-ended evaluation.
-    """
-    parsed_preds = []
-    scores = []
-
-    # Evaluation prompt template
-    meta_prompt = """I need your help to evaluate the quality of an automatic speech recognition (ASR) transcription.
+AMI_LLM_JUDGE_PROMPT = """I need your help to evaluate the quality of an automatic speech recognition (ASR) transcription.
 
 You will be provided with:
 1. [Reference]: The ground truth transcription
@@ -288,6 +275,51 @@ def ami_process_results_llm_judge(doc, results):
 After evaluating, please output the score only without anything else.
 You don't need to provide any explanations."""
 
+
+# Previously sent as a system message; now included as a prefix in the user prompt.
+_AMI_SYSTEM_INSTRUCTION = "You are a helpful assistant who evaluates speech recognition quality."
+
+
+def _ami_judge_prompt_fn(question, prediction, ground_truth, **kwargs):
+    prompt = AMI_LLM_JUDGE_PROMPT.format(reference=ground_truth, hypothesis=prediction)
+    return f"{_AMI_SYSTEM_INSTRUCTION}\n\n{prompt}"
+
+
+def _parse_raw_score(text):
+    """Parse raw 1-5 score without normalization."""
+    numbers = re.findall(r"\d+(?:\.\d+)?", text.strip())
+    score = float(numbers[0]) if numbers else 0.0
+    return VerifyResult(score=score, is_correct=score >= 3.0, raw_output=text)
+
+
+_ami_judge_pipeline = None
+_ami_judge_pipeline_lock = threading.Lock()
+
+
+def _get_ami_judge_pipeline():
+    global _ami_judge_pipeline
+    if _ami_judge_pipeline is None:
+        with _ami_judge_pipeline_lock:
+            if _ami_judge_pipeline is None:  # double-check
+                _ami_judge_pipeline = VerificationPipeline(
+                    verifier=OpenAIVerifier(
+                        model=JUDGE_MODEL_VERSION,
+                        custom_prompt=_ami_judge_prompt_fn,
+                        response_parser=_parse_raw_score,
+                        temperature=0.5,
+                        max_tokens=10,
+                    ),
+                )
+    return _ami_judge_pipeline
+
+
+def ami_process_results_llm_judge(doc, results):
+    """
+    Process results using LLM as judge for open-ended evaluation.
+    Similar to voicebench's open-ended evaluation.
+    """
+    scores = []
+
     for pred in results:
         prediction = pred.strip() if isinstance(pred, str) else str(pred)
 
@@ -304,34 +336,9 @@ def ami_process_results_llm_judge(doc, results):
         # Get reference text
         reference_text = get_column_value(doc, ["text", "transcript", "transcription"])
 
-        formatted_prompt = meta_prompt.format(reference=reference_text, hypothesis=prediction)
-
-        try:
-            from lmms_eval.llm_judge.protocol import Request, ServerConfig
-
-            custom_config = ServerConfig(model_name=JUDGE_MODEL_VERSION, temperature=0.5, max_tokens=10)
-
-            request = Request(messages=[{"role": "system", "content": "You are a helpful assistant who evaluates speech recognition quality."}, {"role": "user", "content": formatted_prompt}], config=custom_config)
-
-            response = server.evaluate(request)
-
-            if response.success:
-                judge_response = response.content.strip()
-                try:
-                    judge_score = int(judge_response)
-                except ValueError:
-                    eval_logger.error(f"Failed to parse score: {judge_response}")
-                    judge_score = 0.0
-            else:
-                eval_logger.error(f"Judge evaluation failed: {response.content}")
-                judge_score = 0.0
-
-        except Exception as e:
-            eval_logger.error(f"Error getting judge response: {e}")
-            judge_score = 0.0
-
-        scores.append(judge_score)
-        parsed_preds.append(prediction)
+        pipeline = _get_ami_judge_pipeline()
+        result = pipeline(question="", prediction=prediction, ground_truth=reference_text)
+        scores.append(result.score)
 
     avg_score = sum(scores) / len(scores) if scores else 0.0
     return {"llm_as_judge_eval": avg_score}

lmms_eval/tasks/browsecomp/utils.py

@@ -2,6 +2,7 @@
 import hashlib
 import os
 import re
+import threading
 from collections import defaultdict
 from pathlib import Path
 
@@ -22,23 +23,43 @@ def _load_yaml_stripped(path: Path) -> dict:
 _config = _load_yaml_stripped(_default_template_path)
 _config.update(_load_yaml_stripped(_browsecomp_config_path))
 
-_judge_server = None
-_judge_server_config = None
-if _config.get("metadata", {}).get("use_lmms_judge"):
-    try:
-        from lmms_eval.llm_judge import get_server
-        from lmms_eval.llm_judge.protocol import ServerConfig
-
-        API_TYPE = os.getenv("API_TYPE", "openai").lower()
-        DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
-
-        _judge_server_config = ServerConfig(model_name=DEPLOYMENT_NAME)
-        _judge_server = get_server(server_name=API_TYPE, config=_judge_server_config)
-        eval_logger.info("Using LMMS judge server for BrowseComp task.")
-    except Exception as err:
-        eval_logger.warning("Failed to initialize LMMS judge for BrowseComp: {}", err)
-        _judge_server = None
-        _judge_server_config = None
+_use_lmms_judge = _config.get("metadata", {}).get("use_lmms_judge", False)
+
+# Lazy pipeline singleton for LLM judge
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline():
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                from lmms_eval.verifiers import VerificationPipeline
+                from lmms_eval.verifiers.extractors import StripReasoningExtractor
+                from lmms_eval.verifiers.openai import OpenAIVerifier
+
+                API_TYPE = os.getenv("API_TYPE", "openai").lower()
+                DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
+
+                def _build_prompt(question, prediction, ground_truth, **kwargs):
+                    return BROWSECOMP_JUDGE_PROMPT.format(
+                        question=question,
+                        response=prediction,
+                        correct_answer=ground_truth,
+                    )
+
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=DEPLOYMENT_NAME,
+                        api_type=API_TYPE,
+                        custom_prompt=_build_prompt,
+                        response_format="binary",
+                    ),
+                )
+                eval_logger.info("Using LMMS judge server for BrowseComp task.")
+    return _pipeline
 
 
 BROWSECOMP_JUDGE_PROMPT = """Judge whether the following [response] to [question] is correct based on the [correct_answer].
@@ -128,25 +149,18 @@ def _normalize_for_exact_match(text: str) -> str:
 
 
 def _judge_is_correct(question: str, response: str, correct_answer: str):
-    if _judge_server is None or _judge_server_config is None:
+    if not _use_lmms_judge:
         return None
 
     try:
-        from lmms_eval.llm_judge.protocol import Request
-
-        submit_prompt = BROWSECOMP_JUDGE_PROMPT.format(
-            question=question,
-            response=response,
-            correct_answer=correct_answer,
-        )
-        request = Request(messages=[{"role": "user", "content": submit_prompt}], config=_judge_server_config)
-        judge_response_obj = _judge_server.evaluate(request)
-        judge_result = str(judge_response_obj.content).strip().lower()
-
-        if "incorrect" in judge_result:
-            return False
-        if "correct" in judge_result:
-            return True
+        pipeline = _get_pipeline()
+        result = pipeline(question=question, prediction=response, ground_truth=correct_answer)
+        # OpenAIVerifier swallows judge-call errors and flags them via
+        # judge_failed metadata; return None so the caller falls back to exact
+        # match instead of scoring an infra failure as incorrect.
+        if result.metadata.get("judge_failed"):
+            return None
+        return result.is_correct
     except Exception as err:
         eval_logger.debug("BrowseComp LLM judge failed, fallback to exact match: {}", err)
 

lmms_eval/tasks/corecognition/utils.py

@@ -7,6 +7,7 @@
 import logging
 import os
 import re
+import threading
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, Dict, Optional
@@ -30,24 +31,43 @@ def _load_yaml_stripped(path: Path) -> dict:
 _corecognition_config = _load_yaml_stripped(_default_template_path)
 _corecognition_config.update(_load_yaml_stripped(_corecognition_config_path))
 
-# Initialize LLM judge server when use_lmms_judge is True (reference: stare/utils.py)
-_judge_server = None
-_judge_server_config = None
-if _corecognition_config.get("metadata", {}).get("use_lmms_judge"):
-    try:
-        from lmms_eval.llm_judge import get_server
-        from lmms_eval.llm_judge.protocol import ServerConfig
-
-        eval_logger.info("Using LMMS judge server for CoreCognition task.")
-        API_TYPE = os.getenv("API_TYPE", "openai").lower()
-        DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
-
-        _judge_server_config = ServerConfig(model_name=DEPLOYMENT_NAME)
-        _judge_server = get_server(server_name=API_TYPE, config=_judge_server_config)
-    except Exception as e:
-        eval_logger.warning("Failed to initialize LMMS judge for CoreCognition: %s", e)
-        _judge_server = None
-        _judge_server_config = None
+_use_lmms_judge = _corecognition_config.get("metadata", {}).get("use_lmms_judge", False)
+
+# Lazy pipeline singleton for LLM judge
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline():
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                from lmms_eval.verifiers import VerificationPipeline
+                from lmms_eval.verifiers.extractors import StripReasoningExtractor
+                from lmms_eval.verifiers.openai import OpenAIVerifier
+
+                eval_logger.info("Using LMMS judge server for CoreCognition task.")
+                API_TYPE = os.getenv("API_TYPE", "openai").lower()
+                DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
+
+                def _build_prompt(question, prediction, ground_truth, **kwargs):
+                    return CORECOGNITION_JUDGE_PROMPT.format(
+                        response=prediction,
+                        answer=ground_truth,
+                    )
+
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=DEPLOYMENT_NAME,
+                        api_type=API_TYPE,
+                        custom_prompt=_build_prompt,
+                        response_format="binary",
+                    ),
+                )
+    return _pipeline
+
 
 # Judge prompt for binary correct/incorrect (same style as stare create_test_prompt)
 CORECOGNITION_JUDGE_PROMPT = """You are judging whether a model's response matches the correct answer for a single-choice or yes/no question.
@@ -59,12 +79,6 @@ def _load_yaml_stripped(path: Path) -> dict:
 Correct_or_not:"""
 
 
-def _create_judge_prompt(doc: dict[str, Any], pred: str) -> str:
-    """Build judge prompt: response + ground truth answer."""
-    answer = str(doc.get("answer", "")).strip()
-    return CORECOGNITION_JUDGE_PROMPT.format(response=pred, answer=answer)
-
-
 # Answer options for template matching
 OPTIONS_MCQ = ["A", "B", "C", "D", "E", "F"]
 OPTIONS_YORN = ["YES", "NO"]
@@ -199,19 +213,21 @@ def corecognition_process_results(doc: dict[str, Any], results: list[str]) -> di
         is_correct = matched == gt_normalized
     else:
         # Template match failed: try LLM judge if enabled, else direct comparison
-        if _judge_server is not None and _judge_server_config is not None:
+        if _use_lmms_judge:
             try:
-                from lmms_eval.llm_judge.protocol import Request
-
-                submit_prompt = _create_judge_prompt(doc, pred)
-                request = Request(
-                    messages=[{"role": "user", "content": submit_prompt}],
-                    config=_judge_server_config,
-                )
-                judge_response_obj = _judge_server.evaluate(request)
-                judge_result = judge_response_obj.content.strip().lower()
-                is_correct = "correct" in judge_result and "incorrect" not in judge_result
+                pipeline = _get_pipeline()
+                result = pipeline(question=pred, prediction=pred, ground_truth=ground_truth)
+                if result.metadata.get("judge_failed"):
+                    # OpenAIVerifier swallows judge-call errors and flags them
+                    # here; fall back to direct comparison rather than counting
+                    # an infra failure as a wrong answer.
+                    pred_normalized = _rm_model_special(pred).upper().strip()
+                    gt_normalized = ground_truth.upper().strip()
+                    is_correct = pred_normalized == gt_normalized
+                else:
+                    is_correct = result.is_correct
             except Exception as e:
+                # Pipeline construction (import/config) failed.
                 eval_logger.debug("CoreCognition LLM judge failed, falling back to direct comparison: %s", e)
                 pred_normalized = _rm_model_special(pred).upper().strip()
                 gt_normalized = ground_truth.upper().strip()