diff --git a/lmms_eval/tasks/ami/utils.py b/lmms_eval/tasks/ami/utils.py
index 2a07e87eb..5b39a247c 100644
--- a/lmms_eval/tasks/ami/utils.py
+++ b/lmms_eval/tasks/ami/utils.py
@@ -1,20 +1,16 @@
 import os
 import re
 import string
+import threading
 
 import numpy as np
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
-API_TYPE = os.getenv("API_TYPE", "openai")
 JUDGE_MODEL_VERSION = os.getenv("JUDGE_MODEL_VERSION", "gpt-4o-mini")
 
-server_config = ServerConfig(
-    model_name=JUDGE_MODEL_VERSION,
-)
-server = get_server(server_name=API_TYPE, config=server_config)
-
 
 def get_column_value(doc, candidates):
     """Helper function to get value from document with multiple possible column names"""
@@ -259,16 +255,7 @@ def calculate_wer(reference, hypothesis):
     return wer
 
 
-def ami_process_results_llm_judge(doc, results):
-    """
-    Process results using LLM as judge for open-ended evaluation.
-    Similar to voicebench's open-ended evaluation.
-    """
-    parsed_preds = []
-    scores = []
-
-    # Evaluation prompt template
-    meta_prompt = """I need your help to evaluate the quality of an automatic speech recognition (ASR) transcription.
+AMI_LLM_JUDGE_PROMPT = """I need your help to evaluate the quality of an automatic speech recognition (ASR) transcription.
 
 You will be provided with:
 1. [Reference]: The ground truth transcription
@@ -288,6 +275,51 @@ def ami_process_results_llm_judge(doc, results):
 After evaluating, please output the score only without anything else.
 You don't need to provide any explanations."""
 
+
+# Previously sent as a system message; now included as a prefix in the user prompt.
+_AMI_SYSTEM_INSTRUCTION = "You are a helpful assistant who evaluates speech recognition quality."
+
+
+def _ami_judge_prompt_fn(question, prediction, ground_truth, **kwargs):
+    prompt = AMI_LLM_JUDGE_PROMPT.format(reference=ground_truth, hypothesis=prediction)
+    return f"{_AMI_SYSTEM_INSTRUCTION}\n\n{prompt}"
+
+
+def _parse_raw_score(text):
+    """Parse raw 1-5 score without normalization."""
+    numbers = re.findall(r"\d+(?:\.\d+)?", text.strip())
+    score = float(numbers[0]) if numbers else 0.0
+    return VerifyResult(score=score, is_correct=score >= 3.0, raw_output=text)
+
+
+_ami_judge_pipeline = None
+_ami_judge_pipeline_lock = threading.Lock()
+
+
+def _get_ami_judge_pipeline():
+    global _ami_judge_pipeline
+    if _ami_judge_pipeline is None:
+        with _ami_judge_pipeline_lock:
+            if _ami_judge_pipeline is None:  # double-check
+                _ami_judge_pipeline = VerificationPipeline(
+                    verifier=OpenAIVerifier(
+                        model=JUDGE_MODEL_VERSION,
+                        custom_prompt=_ami_judge_prompt_fn,
+                        response_parser=_parse_raw_score,
+                        temperature=0.5,
+                        max_tokens=10,
+                    ),
+                )
+    return _ami_judge_pipeline
+
+
+def ami_process_results_llm_judge(doc, results):
+    """
+    Process results using LLM as judge for open-ended evaluation.
+    Similar to voicebench's open-ended evaluation.
+    """
+    scores = []
+
     for pred in results:
         prediction = pred.strip() if isinstance(pred, str) else str(pred)
 
@@ -304,34 +336,9 @@ def ami_process_results_llm_judge(doc, results):
         # Get reference text
         reference_text = get_column_value(doc, ["text", "transcript", "transcription"])
 
-        formatted_prompt = meta_prompt.format(reference=reference_text, hypothesis=prediction)
-
-        try:
-            from lmms_eval.llm_judge.protocol import Request, ServerConfig
-
-            custom_config = ServerConfig(model_name=JUDGE_MODEL_VERSION, temperature=0.5, max_tokens=10)
-
-            request = Request(messages=[{"role": "system", "content": "You are a helpful assistant who evaluates speech recognition quality."}, {"role": "user", "content": formatted_prompt}], config=custom_config)
-
-            response = server.evaluate(request)
-
-            if response.success:
-                judge_response = response.content.strip()
-                try:
-                    judge_score = int(judge_response)
-                except ValueError:
-                    eval_logger.error(f"Failed to parse score: {judge_response}")
-                    judge_score = 0.0
-            else:
-                eval_logger.error(f"Judge evaluation failed: {response.content}")
-                judge_score = 0.0
-
-        except Exception as e:
-            eval_logger.error(f"Error getting judge response: {e}")
-            judge_score = 0.0
-
-        scores.append(judge_score)
-        parsed_preds.append(prediction)
+        pipeline = _get_ami_judge_pipeline()
+        result = pipeline(question="", prediction=prediction, ground_truth=reference_text)
+        scores.append(result.score)
 
     avg_score = sum(scores) / len(scores) if scores else 0.0
     return {"llm_as_judge_eval": avg_score}
diff --git a/lmms_eval/tasks/browsecomp/utils.py b/lmms_eval/tasks/browsecomp/utils.py
index ef93e739e..c39cf52fa 100644
--- a/lmms_eval/tasks/browsecomp/utils.py
+++ b/lmms_eval/tasks/browsecomp/utils.py
@@ -2,6 +2,7 @@
 import hashlib
 import os
 import re
+import threading
 from collections import defaultdict
 from pathlib import Path
 
@@ -22,23 +23,43 @@ def _load_yaml_stripped(path: Path) -> dict:
 _config = _load_yaml_stripped(_default_template_path)
 _config.update(_load_yaml_stripped(_browsecomp_config_path))
 
-_judge_server = None
-_judge_server_config = None
-if _config.get("metadata", {}).get("use_lmms_judge"):
-    try:
-        from lmms_eval.llm_judge import get_server
-        from lmms_eval.llm_judge.protocol import ServerConfig
-
-        API_TYPE = os.getenv("API_TYPE", "openai").lower()
-        DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
-
-        _judge_server_config = ServerConfig(model_name=DEPLOYMENT_NAME)
-        _judge_server = get_server(server_name=API_TYPE, config=_judge_server_config)
-        eval_logger.info("Using LMMS judge server for BrowseComp task.")
-    except Exception as err:
-        eval_logger.warning("Failed to initialize LMMS judge for BrowseComp: {}", err)
-        _judge_server = None
-        _judge_server_config = None
+_use_lmms_judge = _config.get("metadata", {}).get("use_lmms_judge", False)
+
+# Lazy pipeline singleton for LLM judge
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline():
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                from lmms_eval.verifiers import VerificationPipeline
+                from lmms_eval.verifiers.extractors import StripReasoningExtractor
+                from lmms_eval.verifiers.openai import OpenAIVerifier
+
+                API_TYPE = os.getenv("API_TYPE", "openai").lower()
+                DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
+
+                def _build_prompt(question, prediction, ground_truth, **kwargs):
+                    return BROWSECOMP_JUDGE_PROMPT.format(
+                        question=question,
+                        response=prediction,
+                        correct_answer=ground_truth,
+                    )
+
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=DEPLOYMENT_NAME,
+                        api_type=API_TYPE,
+                        custom_prompt=_build_prompt,
+                        response_format="binary",
+                    ),
+                )
+                eval_logger.info("Using LMMS judge server for BrowseComp task.")
+    return _pipeline
 
 
 BROWSECOMP_JUDGE_PROMPT = """Judge whether the following [response] to [question] is correct based on the [correct_answer].
@@ -128,25 +149,18 @@ def _normalize_for_exact_match(text: str) -> str:
 
 
 def _judge_is_correct(question: str, response: str, correct_answer: str):
-    if _judge_server is None or _judge_server_config is None:
+    if not _use_lmms_judge:
         return None
 
     try:
-        from lmms_eval.llm_judge.protocol import Request
-
-        submit_prompt = BROWSECOMP_JUDGE_PROMPT.format(
-            question=question,
-            response=response,
-            correct_answer=correct_answer,
-        )
-        request = Request(messages=[{"role": "user", "content": submit_prompt}], config=_judge_server_config)
-        judge_response_obj = _judge_server.evaluate(request)
-        judge_result = str(judge_response_obj.content).strip().lower()
-
-        if "incorrect" in judge_result:
-            return False
-        if "correct" in judge_result:
-            return True
+        pipeline = _get_pipeline()
+        result = pipeline(question=question, prediction=response, ground_truth=correct_answer)
+        # OpenAIVerifier swallows judge-call errors and flags them via
+        # judge_failed metadata; return None so the caller falls back to exact
+        # match instead of scoring an infra failure as incorrect.
+        if result.metadata.get("judge_failed"):
+            return None
+        return result.is_correct
     except Exception as err:
         eval_logger.debug("BrowseComp LLM judge failed, fallback to exact match: {}", err)
 
diff --git a/lmms_eval/tasks/corecognition/utils.py b/lmms_eval/tasks/corecognition/utils.py
index 53adbda40..246a34ad0 100644
--- a/lmms_eval/tasks/corecognition/utils.py
+++ b/lmms_eval/tasks/corecognition/utils.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import re
+import threading
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, Dict, Optional
@@ -30,24 +31,43 @@ def _load_yaml_stripped(path: Path) -> dict:
 _corecognition_config = _load_yaml_stripped(_default_template_path)
 _corecognition_config.update(_load_yaml_stripped(_corecognition_config_path))
 
-# Initialize LLM judge server when use_lmms_judge is True (reference: stare/utils.py)
-_judge_server = None
-_judge_server_config = None
-if _corecognition_config.get("metadata", {}).get("use_lmms_judge"):
-    try:
-        from lmms_eval.llm_judge import get_server
-        from lmms_eval.llm_judge.protocol import ServerConfig
-
-        eval_logger.info("Using LMMS judge server for CoreCognition task.")
-        API_TYPE = os.getenv("API_TYPE", "openai").lower()
-        DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
-
-        _judge_server_config = ServerConfig(model_name=DEPLOYMENT_NAME)
-        _judge_server = get_server(server_name=API_TYPE, config=_judge_server_config)
-    except Exception as e:
-        eval_logger.warning("Failed to initialize LMMS judge for CoreCognition: %s", e)
-        _judge_server = None
-        _judge_server_config = None
+_use_lmms_judge = _corecognition_config.get("metadata", {}).get("use_lmms_judge", False)
+
+# Lazy pipeline singleton for LLM judge
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline():
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                from lmms_eval.verifiers import VerificationPipeline
+                from lmms_eval.verifiers.extractors import StripReasoningExtractor
+                from lmms_eval.verifiers.openai import OpenAIVerifier
+
+                eval_logger.info("Using LMMS judge server for CoreCognition task.")
+                API_TYPE = os.getenv("API_TYPE", "openai").lower()
+                DEPLOYMENT_NAME = os.getenv("DEPLOYMENT_NAME") or os.getenv("OPENAI_API_MODEL", "gpt-4o")
+
+                def _build_prompt(question, prediction, ground_truth, **kwargs):
+                    return CORECOGNITION_JUDGE_PROMPT.format(
+                        response=prediction,
+                        answer=ground_truth,
+                    )
+
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=DEPLOYMENT_NAME,
+                        api_type=API_TYPE,
+                        custom_prompt=_build_prompt,
+                        response_format="binary",
+                    ),
+                )
+    return _pipeline
+
 
 # Judge prompt for binary correct/incorrect (same style as stare create_test_prompt)
 CORECOGNITION_JUDGE_PROMPT = """You are judging whether a model's response matches the correct answer for a single-choice or yes/no question.
@@ -59,12 +79,6 @@ def _load_yaml_stripped(path: Path) -> dict:
 Correct_or_not:"""
 
 
-def _create_judge_prompt(doc: dict[str, Any], pred: str) -> str:
-    """Build judge prompt: response + ground truth answer."""
-    answer = str(doc.get("answer", "")).strip()
-    return CORECOGNITION_JUDGE_PROMPT.format(response=pred, answer=answer)
-
-
 # Answer options for template matching
 OPTIONS_MCQ = ["A", "B", "C", "D", "E", "F"]
 OPTIONS_YORN = ["YES", "NO"]
@@ -199,19 +213,21 @@ def corecognition_process_results(doc: dict[str, Any], results: list[str]) -> di
         is_correct = matched == gt_normalized
     else:
         # Template match failed: try LLM judge if enabled, else direct comparison
-        if _judge_server is not None and _judge_server_config is not None:
+        if _use_lmms_judge:
             try:
-                from lmms_eval.llm_judge.protocol import Request
-
-                submit_prompt = _create_judge_prompt(doc, pred)
-                request = Request(
-                    messages=[{"role": "user", "content": submit_prompt}],
-                    config=_judge_server_config,
-                )
-                judge_response_obj = _judge_server.evaluate(request)
-                judge_result = judge_response_obj.content.strip().lower()
-                is_correct = "correct" in judge_result and "incorrect" not in judge_result
+                pipeline = _get_pipeline()
+                result = pipeline(question=pred, prediction=pred, ground_truth=ground_truth)
+                if result.metadata.get("judge_failed"):
+                    # OpenAIVerifier swallows judge-call errors and flags them
+                    # here; fall back to direct comparison rather than counting
+                    # an infra failure as a wrong answer.
+                    pred_normalized = _rm_model_special(pred).upper().strip()
+                    gt_normalized = ground_truth.upper().strip()
+                    is_correct = pred_normalized == gt_normalized
+                else:
+                    is_correct = result.is_correct
             except Exception as e:
+                # Pipeline construction (import/config) failed.
                 eval_logger.debug("CoreCognition LLM judge failed, falling back to direct comparison: %s", e)
                 pred_normalized = _rm_model_special(pred).upper().strip()
                 gt_normalized = ground_truth.upper().strip()
diff --git a/lmms_eval/tasks/llava-bench-coco/utils.py b/lmms_eval/tasks/llava-bench-coco/utils.py
index bbb27b6a7..7d09b4348 100755
--- a/lmms_eval/tasks/llava-bench-coco/utils.py
+++ b/lmms_eval/tasks/llava-bench-coco/utils.py
@@ -1,6 +1,6 @@
 import json
 import os
-import time
+import threading
 from copy import deepcopy
 from pathlib import Path
 
@@ -8,7 +8,8 @@
 import yaml
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import Request, ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 NUM_SECONDS_TO_SLEEP = 0.5
 
@@ -29,60 +30,70 @@
 GPT_EVAL_MODEL_NAME = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
 API_TYPE = os.getenv("API_TYPE", "openai")
 
-# Initialize the judge server
-server_config = ServerConfig(model_name=GPT_EVAL_MODEL_NAME, temperature=0.2, max_tokens=1024)
-server = get_server(server_name=API_TYPE, config=server_config)
+# Previously sent as a system message; now included as a prefix in the user prompt.
+_SYSTEM_INSTRUCTION = "You are a helpful and precise assistant for checking the quality of the answer."
 
 
-def get_eval(content: str, max_tokens: int, retries: int = 3):
-    messages = [
-        {
-            "role": "system",
-            "content": "You are a helpful and precise assistant for checking the quality of the answer.",
-        },
-        {"role": "user", "content": content},
-    ]
+# ---------------------------------------------------------------------------
+# Verification pipeline (replaces get_eval + parse_score)
+# ---------------------------------------------------------------------------
 
-    # Update server config with specific parameters for this request
-    custom_config = ServerConfig(model_name=GPT_EVAL_MODEL_NAME, temperature=0.2, max_tokens=max_tokens)
 
-    for attempt in range(retries):
-        try:
-            # Create a Request object for the unified judge API
-            request = Request(messages=messages, config=custom_config)
+def _parse_comparative_scores(text: str) -> VerifyResult:
+    """Parse two scores from the first line of the judge response.
 
-            # Use the unified judge API
-            response = server.evaluate(request)
-
-            content = response.content.strip() if response.content else ""
-            if content != "":
-                return content, response.model_used
-            break  # If successful, break out of the loop
-
-        except Exception as e:
-            eval_logger.info(f"Attempt {attempt + 1} failed with error: {str(e)}")
-            if attempt < retries - 1:  # If we have retries left, sleep and then continue to next attempt
-                time.sleep(NUM_SECONDS_TO_SLEEP)
-            else:  # If this was the last attempt, log and return empty
-                eval_logger.error(f"All {retries} attempts failed. Last error message: {str(e)}")
-                return "", ""
-    return "", ""
-
-
-def parse_score(review):
+    Matches the original parse_score logic: split first line by space
+    after replacing commas, expect exactly 2 values.
+    """
     try:
-        score_pair = review.split("\n")[0]
+        score_pair = text.strip().split("\n")[0]
         score_pair = score_pair.replace(",", " ")
         sp = score_pair.split(" ")
         if len(sp) == 2:
-            return [float(sp[0]), float(sp[1])]
+            scores = [float(sp[0]), float(sp[1])]
         else:
-            eval_logger.debug("error", review)
-            return [-1, -1]
+            eval_logger.debug("error", text)
+            scores = [-1, -1]
     except Exception as e:
         eval_logger.debug(e)
-        eval_logger.debug("error", review)
-        return [-1, -1]
+        eval_logger.debug("error", text)
+        scores = [-1, -1]
+    return VerifyResult(
+        score=scores[1] / 10.0 if scores[1] > 0 else 0.0,
+        is_correct=scores[1] > scores[0],
+        raw_output=text,
+        metadata={"scores": scores},
+    )
+
+
+def _build_coco_prompt(question, prediction, ground_truth, **kwargs):
+    """Build the full prompt, preserving the original system instruction."""
+    return f"{_SYSTEM_INSTRUCTION}\n\n{kwargs['content']}"
+
+
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline() -> VerificationPipeline:
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                _pipeline = VerificationPipeline(
+                    extractors=[],
+                    verifier=OpenAIVerifier(
+                        model=GPT_EVAL_MODEL_NAME,
+                        api_type=API_TYPE,
+                        custom_prompt=_build_coco_prompt,
+                        response_parser=_parse_comparative_scores,
+                        temperature=0.2,
+                        max_tokens=1024,
+                        max_retries=3,
+                        retry_delay=0.5,
+                    ),
+                )
+    return _pipeline
 
 
 def llava_doc_to_visual(doc):
@@ -117,8 +128,12 @@ def llava_process_results(doc, result):
         role = rule.get("role", "user")
         content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n"
 
-        review, model_name = get_eval(content, 1024)
-        scores = parse_score(review)
+        pipeline = _get_pipeline()
+        vresult = pipeline(question=question, prediction=ans2, ground_truth=ans1, content=content)
+
+        review = vresult.raw_output
+        model_name = GPT_EVAL_MODEL_NAME
+        scores = vresult.metadata.get("scores", [-1, -1])
     except Exception as e:
         eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}")
         review = "Failed to Get a Proper Review."
diff --git a/lmms_eval/tasks/llava-in-the-wild/utils.py b/lmms_eval/tasks/llava-in-the-wild/utils.py
index ccb507d12..7d6914c5e 100755
--- a/lmms_eval/tasks/llava-in-the-wild/utils.py
+++ b/lmms_eval/tasks/llava-in-the-wild/utils.py
@@ -1,5 +1,7 @@
 import json
 import os
+import re
+import threading
 from copy import deepcopy
 from pathlib import Path
 
@@ -7,7 +9,8 @@
 import yaml
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 NUM_SECONDS_TO_SLEEP = 5
 
@@ -28,10 +31,60 @@
 API_TYPE = os.getenv("API_TYPE", "openai")
 MODEL_VERSION = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
 
-server_config = ServerConfig(
-    model_name=MODEL_VERSION,
-)
-server = get_server(server_name=API_TYPE, config=server_config)
+
+# ---------------------------------------------------------------------------
+# Verification pipeline (replaces direct llm_judge evaluate_comparative)
+# ---------------------------------------------------------------------------
+
+
+def _parse_comparative_scores(text: str) -> VerifyResult:
+    """Parse comparative scores from judge response.
+
+    Matches the parsing logic of ResponseParser.parse_comparative_response:
+    extracts first two numbers from the first line of the response.
+    """
+    try:
+        lines = text.strip().split("\n")
+        if lines:
+            score_line = lines[0].replace(",", " ").replace(";", " ")
+            numbers = re.findall(r"-?\d+(?:\.\d+)?", score_line)
+            if len(numbers) >= 2:
+                scores = [float(numbers[0]), float(numbers[1])]
+            else:
+                scores = [-1, -1]
+        else:
+            scores = [-1, -1]
+    except Exception:
+        scores = [-1, -1]
+    return VerifyResult(
+        score=scores[1] / 10.0 if scores[1] > 0 else 0.0,
+        is_correct=scores[1] > scores[0],
+        raw_output=text,
+        metadata={"scores": scores},
+    )
+
+
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline() -> VerificationPipeline:
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                _pipeline = VerificationPipeline(
+                    extractors=[],
+                    verifier=OpenAIVerifier(
+                        model=MODEL_VERSION,
+                        api_type=API_TYPE,
+                        custom_prompt=lambda q, p, gt, **kw: kw["content"],
+                        response_parser=_parse_comparative_scores,
+                        max_retries=5,
+                        retry_delay=2.0,
+                    ),
+                )
+    return _pipeline
 
 
 def llava_doc_to_visual(doc):
@@ -66,11 +119,12 @@ def llava_process_results(doc, result):
         role = rule.get("role", "user")
         content = f"[Context]\n{context}\n\n" f"[Question]\n{question}\n\n" f"[{role} 1]\n{ans1}\n\n[End of {role} 1]\n\n" f"[{role} 2]\n{ans2}\n\n[End of {role} 2]\n\n" f"[System]\n{prompt}\n\n"
 
-        result = server.evaluate_comparative(question=question, response1=ans1, response2=ans2, context=context, custom_prompt=content, score_range=(1, 10))
+        pipeline = _get_pipeline()
+        vresult = pipeline(question=question, prediction=ans2, ground_truth=ans1, content=content)
 
-        review = result["raw_response"]
-        model_name = result["model"]
-        scores = list(result["scores"])
+        review = vresult.raw_output
+        model_name = MODEL_VERSION
+        scores = vresult.metadata.get("scores", [-1, -1])
     except Exception as e:
         eval_logger.error(f"Error for Question ID: {doc.get('question_id', 'Unknown')}: {e}")
         review = "Failed to Get a Proper Review."
diff --git a/lmms_eval/tasks/mathvision/utils.py b/lmms_eval/tasks/mathvision/utils.py
index 505ac8bca..b502f745b 100755
--- a/lmms_eval/tasks/mathvision/utils.py
+++ b/lmms_eval/tasks/mathvision/utils.py
@@ -1,8 +1,11 @@
 import os
+import threading
 
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline
+from lmms_eval.verifiers.extractors import StripReasoningExtractor
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 try:
     from lmms_eval.tasks.mathvision.eval_utils import (
@@ -16,14 +19,27 @@
 
 NUM_SECONDS_TO_SLEEP = 5
 
-# Initialize the judge server
-API_TYPE = os.getenv("API_TYPE", "openai")
-GPT_MODEL = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
-
-server_config = ServerConfig(
-    model_name=GPT_MODEL,
-)
-server = get_server(server_name=API_TYPE, config=server_config)
+# Lazy pipeline singleton for GPT-based evaluation
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline() -> VerificationPipeline:
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                API_TYPE = os.getenv("API_TYPE", "openai")
+                GPT_MODEL = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=GPT_MODEL,
+                        api_type=API_TYPE,
+                        judge_type="binary",
+                    ),
+                )
+    return _pipeline
 
 
 def mathvision_doc_to_visual(doc):
@@ -50,23 +66,15 @@ def mathvision_doc_to_text(doc, lmms_eval_specific_kwargs=None):
 
 def mathvision_gpt_eval_process_results(doc, results):
     correct_list = []
+    pipeline = _get_pipeline()
     for pred in results:
         model_answer = pred.strip()
         gt_answer = str(doc["answer"])
         question = doc["question"]
 
         try:
-            # Use the llm_judge API for binary evaluation
-            result = server.evaluate_binary(question=question, answer=gt_answer, prediction=model_answer, output_format="0/1")
-
-            # Parse the result
-            if result["success"]:
-                judge_response = result["result"]
-                correct_list.append(judge_response)
-            else:
-                eval_logger.error(f"Judge evaluation failed: {result.get('raw_response', 'Unknown error')}")
-                correct_list.append(False)
-
+            result = pipeline(question=question, prediction=model_answer, ground_truth=gt_answer)
+            correct_list.append(result.is_correct)
         except Exception as e:
             eval_logger.error(f"Error getting judge response: {e}")
             correct_list.append(False)
diff --git a/lmms_eval/tasks/mmmu/utils.py b/lmms_eval/tasks/mmmu/utils.py
index ecddcd74e..b42aafe2e 100755
--- a/lmms_eval/tasks/mmmu/utils.py
+++ b/lmms_eval/tasks/mmmu/utils.py
@@ -2,13 +2,13 @@
 import json
 import os
 import re
+import threading
 from collections import defaultdict
 from pathlib import Path
 
 import yaml
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
 from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
 from lmms_eval.tasks._task_utils.mmmu_mcq_utils import (
     get_multi_choice_info as shared_get_multi_choice_info,
@@ -16,6 +16,8 @@
 from lmms_eval.tasks._task_utils.mmmu_mcq_utils import (
     parse_mmmu_multi_choice_response,
 )
+from lmms_eval.verifiers import VerificationPipeline
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
     raw_data = f.readlines()
@@ -30,11 +32,28 @@
 API_TYPE = os.getenv("API_TYPE", "openai")
 MODEL_VERSION = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
 
-# Initialize the judge server
-server_config = ServerConfig(
-    model_name=MODEL_VERSION,
-)
-server = get_server(server_name=API_TYPE, config=server_config)
+# ---------------------------------------------------------------------------
+# Lazy VerificationPipeline singleton for reasoning evaluation
+# ---------------------------------------------------------------------------
+
+_reasoning_pipeline = None
+_reasoning_pipeline_lock = threading.Lock()
+
+
+def _get_reasoning_pipeline() -> VerificationPipeline:
+    global _reasoning_pipeline
+    if _reasoning_pipeline is None:
+        with _reasoning_pipeline_lock:
+            if _reasoning_pipeline is None:  # double-check
+                _reasoning_pipeline = VerificationPipeline(
+                    extractors=[],
+                    verifier=OpenAIVerifier(
+                        model=MODEL_VERSION,
+                        api_type=API_TYPE,
+                        judge_type="binary",
+                    ),
+                )
+    return _reasoning_pipeline
 
 
 def replace_images_tokens(input_string):
@@ -175,38 +194,23 @@ def mmmu_process_results(doc, results):
 
 
 def mmmu_reasoning_process_results(doc, results):
-    parsed_preds = []
+    pipeline = _get_reasoning_pipeline()
+    formatted_question = construct_prompt(doc)
+    answer = str(doc["answer"])
+
     scores = []
     for pred in results:
-        formatted_question = construct_prompt(doc)
         # Extract content from <answer> tags if present, handling potential spaces
-        answer = doc["answer"]
         if isinstance(pred, str):
             match = re.search(r"<answer>\s*([\s\S]*?)\s*</answer>", pred)
             if match:
                 pred = match.group(1).strip()
 
-        try:
-            # Use the llm_judge API for binary evaluation
-            result = server.evaluate_binary(question=formatted_question, answer=str(answer), prediction=pred, output_format="0/1")
-
-            # Parse the result
-            if result["success"]:
-                judge_response = result["result"]
-                judge_score = int(judge_response) if isinstance(judge_response, str) else judge_response
-            else:
-                eval_logger.error(f"Judge evaluation failed: {result.get('raw_response', 'Unknown error')}")
-                judge_score = 0
-
-        except Exception as e:
-            eval_logger.error(f"Error getting judge response: {e}")
-            judge_score = 0
-
-        scores.append(judge_score)
-        parsed_preds.append(pred)
+        result = pipeline(question=formatted_question, prediction=pred, ground_truth=answer)
+        scores.append(1 if result.is_correct else 0)
 
     # Calculate the average score for this document
-    avg_score = sum(1 if score == 1 else 0 for score in scores) / len(scores) if scores else 0
+    avg_score = sum(scores) / len(scores) if scores else 0
     return {"llm_as_judge_eval": avg_score}
 
 
diff --git a/lmms_eval/tasks/mmvu/utils.py b/lmms_eval/tasks/mmvu/utils.py
index 87b4056f7..56be0f5a6 100644
--- a/lmms_eval/tasks/mmvu/utils.py
+++ b/lmms_eval/tasks/mmvu/utils.py
@@ -2,13 +2,16 @@
 import re
 import string
 import sys
+import threading
 from pathlib import Path
 from typing import Any, Dict
 
 import yaml
 from loguru import logger as eval_logger
 
-from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline
+from lmms_eval.verifiers.extractors import StripReasoningExtractor
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 hf_home = os.getenv("HF_HOME", "~/.cache/huggingface")
 
@@ -25,19 +28,45 @@
 cache_name_val = yaml.safe_load("".join(safe_data_val))["dataset_kwargs"]["cache_dir"]
 cache_dir_val = os.path.join(base_cache_dir, cache_name_val)
 
-# Initialize the LLM judge server (lazy initialization)
-_server = None
+# Lazy pipeline singleton for GPT-based open-ended evaluation
+_pipeline = None
+_pipeline_lock = threading.Lock()
+
+
+def _get_pipeline() -> VerificationPipeline:
+    """Lazy initialization of verification pipeline for open-ended GPT judge."""
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                API_TYPE = os.getenv("API_TYPE", "openai")
+                MODEL_VERSION = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model=MODEL_VERSION,
+                        api_type=API_TYPE,
+                        judge_type="binary",
+                        custom_prompt="""You are a strict evaluator assessing answer correctness. You must output 1 for fully correct answers and 0 for any other case.
 
+# Evaluation Rules for Open-Ended Questions
+- The model prediction may contain reasoning, focus on extracting the final answer.
+- Score 1 if the prediction matches the answer semantically, even if in different format.
+- For mathematical notation, treat equivalent forms as correct (e.g., O(n²) = O(n^2), n² = n^2).
+- Score 0 for partially correct answers or answers with extra incorrect information.
+- Ignore minor differences in formatting, capitalization, or spacing.
+- Treat numerical answers as correct if they match within reasonable precision.
+- For questions requiring units, both value and unit must be correct.
+
+Return only "1" or "0" with no additional text or formatting.
 
-def get_llm_judge_server() -> Any:
-    """Lazy initialization of LLM judge server"""
-    global _server
-    if _server is None:
-        API_TYPE = os.getenv("API_TYPE", "openai")
-        MODEL_VERSION = os.getenv("MODEL_VERSION", "gpt-4o-2024-11-20")
-        server_config = ServerConfig(model_name=MODEL_VERSION)
-        _server = get_server(server_name=API_TYPE, config=server_config)
-    return _server
+Question: {question}
+Correct Answer: {ground_truth}
+Model Prediction: {prediction}""",
+                        response_format="binary",
+                    ),
+                )
+    return _pipeline
 
 
 def mmvu_doc_to_visual_val(doc):
@@ -269,36 +298,13 @@ def evaluate_with_llm_judge(doc: Dict[str, Any], prediction: str) -> tuple[bool,
     # For open-ended questions, if rule-based says it's incorrect, try GPT judge for semantic equivalence
     # This handles cases like "O(n²)" vs "O(n^2)" where rule-based might be too strict
     try:
-        server = get_llm_judge_server()
+        pipeline = _get_pipeline()
         formatted_question = construct_question_prompt(doc)
         answer = doc["answer"]
-
         full_answer = str(answer)
 
-        custom_prompt = """You are a strict evaluator assessing answer correctness. You must output 1 for fully correct answers and 0 for any other case.
-
-# Evaluation Rules for Open-Ended Questions
-- The model prediction may contain reasoning, focus on extracting the final answer.
-- Score 1 if the prediction matches the answer semantically, even if in different format.
-- For mathematical notation, treat equivalent forms as correct (e.g., O(n²) = O(n^2), n² = n^2).
-- Score 0 for partially correct answers or answers with extra incorrect information.
-- Ignore minor differences in formatting, capitalization, or spacing.
-- Treat numerical answers as correct if they match within reasonable precision.
-- For questions requiring units, both value and unit must be correct.
-
-Return only "1" or "0" with no additional text or formatting."""
-
-        result = server.evaluate_binary(question=formatted_question, answer=full_answer, prediction=prediction, output_format="0/1", custom_prompt=custom_prompt)
-
-        if result["success"]:
-            judge_response = result["result"]
-            judge_score = str(judge_response).strip()
-            is_correct = judge_score == "1"
-            return is_correct, "gpt-based"
-        else:
-            eval_logger.error(f"GPT judge evaluation failed: {result.get('raw_response', 'Unknown error')}")
-            # Fall back to rule-based result if GPT fails
-            return rule_correct, "rule-based"
+        result = pipeline(question=formatted_question, prediction=prediction, ground_truth=full_answer)
+        return result.is_correct, "gpt-based"
 
     except Exception as e:
         eval_logger.error(f"Error getting GPT judge response: {e}")
diff --git a/lmms_eval/tasks/videoevalpro/utils.py b/lmms_eval/tasks/videoevalpro/utils.py
index 160b5bd2d..e9a674cea 100644
--- a/lmms_eval/tasks/videoevalpro/utils.py
+++ b/lmms_eval/tasks/videoevalpro/utils.py
@@ -1,12 +1,15 @@
 import os
 import sys
-import time
+import threading
 from pathlib import Path
 
-import openai
 import yaml
 from loguru import logger as eval_logger
 
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.extractors import StripReasoningExtractor
+from lmms_eval.verifiers.openai import OpenAIVerifier
+
 hf_home = os.getenv("HF_HOME", "~/.cache/huggingface")
 base_cache_dir = os.path.expanduser(hf_home)
 
@@ -22,6 +25,11 @@
 cache_dir = os.path.join(base_cache_dir, cache_name)
 
 
+# ---------------------------------------------------------------------------
+# Visual / text doc helpers
+# ---------------------------------------------------------------------------
+
+
 def videoevalpro_doc_to_visual(doc):
     video_path = doc["video"]
     video_path = os.path.join(os.path.join(cache_dir, "videos_filtered"), video_path)
@@ -29,8 +37,6 @@ def videoevalpro_doc_to_visual(doc):
         video_path = video_path
     else:
         sys.exit(f"video path:{video_path} does not exist, please check")
-
-    # print(f"---------------------->>>>video path: {video_path}")
     return [video_path]
 
 
@@ -38,49 +44,66 @@ def videoevalpro_doc_to_text(doc, lmms_eval_specific_kwargs=None):
     question = doc["question"]
     pre_prompt = lmms_eval_specific_kwargs.get("pre_prompt", "")
     post_prompt = lmms_eval_specific_kwargs.get("post_prompt", "")
-    full_prompt = pre_prompt + question + post_prompt
-    # print(f"---------------------->>>>full_prompt: {full_prompt}")
-    return full_prompt
+    return pre_prompt + question + post_prompt
 
 
-class GPT4oJudge:
-    def __init__(self, model_name="gpt-4o-2024-11-20") -> None:
-        self.model_name = model_name
-        self.client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+# ---------------------------------------------------------------------------
+# GPT-4o judge — prompt + response parser
+# ---------------------------------------------------------------------------
 
-    def __call__(self, question: str, target: str, predicted_answer: str) -> bool:
-        prompt = f"""Your job is to look at a question generated from the video, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"]. First, I will give examples of each grade, and then you will grade a new example. The following are examples of CORRECT predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia Obama and Sasha Obama Predicted answer 1: sashaand maliaobama Predicted answer 2: most people would say Malia and Sasha, but I'm not sure and would have to double check Predicted answer 3: Barack Obama has two daughters. Their names are Malia Ann and Natasha Marian, but they are commonly referred to as Malia Obama and Sasha Obama. Malia was born on July 4, 1998, and Sasha was born on June 10, 2001. ``` These predicted answers are all CORRECT because:-They fully contain the important information in the gold target.-They do not contain any information that contradicts the gold target.-Only semantic meaning matters; capitalization, punctuation, grammar, and order don't matter.-Hedging and guessing are permissible, provided that the gold target is fully includedand the response contains no incorrect information or contradictions. The following are examples of INCORRECT predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia and Sasha Predicted answer 1: Malia. Predicted answer 2: Malia, Sasha, and Susan. Predicted answer 3: Barack Obama does not have any children. Predicted answer 4: I think it's either Malia and Sasha. Or it could be Malia and Jackie. Or it could be Joey and Malia. Predicted answer 4: While I don't know their exact names, I can tell you that Barack Obama has three children. Predicted answer 5: It's possible you may mean Betsy and Olivia. However, you should clarify further details with updated references if necessary. Is that the correct answer? Predicted answer 6: It may be the case that Obama's child is named James. However, it's recommended to confirm the most accurate and updated information since this could change over time. This model may not always reflect the most current information. ``` These predicted answers are all INCORRECT because:-A factual statement in the answer contradicts the gold target. Incorrect statements that have some hedging (e.g., "it is possible that", "although i'mnot sure, i think") are also considered incorrect. The following are examples of NOT_ATTEMPTED predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia and Sasha Predicted answer 1: I don't know. Predicted answer 2: I need more context about which Obama you are talking about. Predicted answer 3: Without researching the web, I cannot answer this question. However, I can tell you that Barack Obama has two children. Predicted answer 4: Barack Obama has two children. I know that one of them is Malia, but I'm not sure about the other one. ``` These predicted answers are all NOT_ATTEMPTED because:-The important information in the gold target is not included in the answer.-No statements in the answer contradict the gold target.
+VIDEOEVALPRO_JUDGE_PROMPT = """Your job is to look at a question generated from the video, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"]. First, I will give examples of each grade, and then you will grade a new example. The following are examples of CORRECT predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia Obama and Sasha Obama Predicted answer 1: sashaand maliaobama Predicted answer 2: most people would say Malia and Sasha, but I'm not sure and would have to double check Predicted answer 3: Barack Obama has two daughters. Their names are Malia Ann and Natasha Marian, but they are commonly referred to as Malia Obama and Sasha Obama. Malia was born on July 4, 1998, and Sasha was born on June 10, 2001. ``` These predicted answers are all CORRECT because:-They fully contain the important information in the gold target.-They do not contain any information that contradicts the gold target.-Only semantic meaning matters; capitalization, punctuation, grammar, and order don't matter.-Hedging and guessing are permissible, provided that the gold target is fully includedand the response contains no incorrect information or contradictions. The following are examples of INCORRECT predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia and Sasha Predicted answer 1: Malia. Predicted answer 2: Malia, Sasha, and Susan. Predicted answer 3: Barack Obama does not have any children. Predicted answer 4: I think it's either Malia and Sasha. Or it could be Malia and Jackie. Or it could be Joey and Malia. Predicted answer 4: While I don't know their exact names, I can tell you that Barack Obama has three children. Predicted answer 5: It's possible you may mean Betsy and Olivia. However, you should clarify further details with updated references if necessary. Is that the correct answer? Predicted answer 6: It may be the case that Obama's child is named James. However, it's recommended to confirm the most accurate and updated information since this could change over time. This model may not always reflect the most current information. ``` These predicted answers are all INCORRECT because:-A factual statement in the answer contradicts the gold target. Incorrect statements that have some hedging (e.g., "it is possible that", "although i'mnot sure, i think") are also considered incorrect. The following are examples of NOT_ATTEMPTED predicted answers. ``` Question: What is the name of the man's child in the video? Gold target: Malia and Sasha Predicted answer 1: I don't know. Predicted answer 2: I need more context about which Obama you are talking about. Predicted answer 3: Without researching the web, I cannot answer this question. However, I can tell you that Barack Obama has two children. Predicted answer 4: Barack Obama has two children. I know that one of them is Malia, but I'm not sure about the other one. ``` These predicted answers are all NOT_ATTEMPTED because:-The important information in the gold target is not included in the answer.-No statements in the answer contradict the gold target.
 
-Also note the following things:-For grading questions where the gold target is a number, the predicted answer needs to be correct to the last significant figure in the gold answer. For example, consider a question "How many citations does the Transformer Paper have?" with gold target "120k". -Predicted answers "120k", "124k", and 115k" are all CORRECT. -Predicted answers "100k" and "113k" are INCORRECT. -Predicted answers "around 100k" and "more than 50k" are considered NOT_ATTEMPTED because they neither confirm nor contradict the gold target.-The gold target may contain more information than the question. In such cases, the predicted answer only needs to contain the information that is in the question.-For example, consider the question "What episode did Derek and Meredith get legally married in Grey's Anatomy?" with gold target "Season 7, Episode 20: White Wedding". Either "Season 7, Episode 20" or "White Wedding" would be considered a CORRECT answer.-Do not punish predicted answers if they omit information that would be clearly inferred from the question.-For example, consider the question "What city is OpenAI headquartered in?" and the gold target "San Francisco, California". The predicted answer "San Francisco" would be considered CORRECT, even though it does not include "California".-Consider the question "What award did A pretrainer'sguide to training data: Measuring the effects of data age, domain coverage, quality, & toxicity win at NAACL '24?", the gold target is "Outstanding Paper Award". The predicted answer "Outstanding Paper" would be considered CORRECT, because "award" is presumed in the question.-For the question "What is the height of Jason Wei in meters?", the gold target is "1.73 m". The predicted answer "1.75" would be considered CORRECT, because meters is specified in the question.-For the question "What is the name of Barack Obama's wife?", the gold target is "Michelle Obama". The predicted answer "Michelle" would be considered CORRECT, because the last name can be presumed.-Do not punish for typos in people's name if it's clearly the same name. -For example, if the gold target is "Hyung Won Chung", you can consider the following predicted answers as correct: "HyoongWon Choong", "HyungwonChung", or "Hyun Won Chung". 
+Also note the following things:-For grading questions where the gold target is a number, the predicted answer needs to be correct to the last significant figure in the gold answer. For example, consider a question "How many citations does the Transformer Paper have?" with gold target "120k". -Predicted answers "120k", "124k", and 115k" are all CORRECT. -Predicted answers "100k" and "113k" are INCORRECT. -Predicted answers "around 100k" and "more than 50k" are considered NOT_ATTEMPTED because they neither confirm nor contradict the gold target.-The gold target may contain more information than the question. In such cases, the predicted answer only needs to contain the information that is in the question.-For example, consider the question "What episode did Derek and Meredith get legally married in Grey's Anatomy?" with gold target "Season 7, Episode 20: White Wedding". Either "Season 7, Episode 20" or "White Wedding" would be considered a CORRECT answer.-Do not punish predicted answers if they omit information that would be clearly inferred from the question.-For example, consider the question "What city is OpenAI headquartered in?" and the gold target "San Francisco, California". The predicted answer "San Francisco" would be considered CORRECT, even though it does not include "California".-Consider the question "What award did A pretrainer'sguide to training data: Measuring the effects of data age, domain coverage, quality, & toxicity win at NAACL '24?", the gold target is "Outstanding Paper Award". The predicted answer "Outstanding Paper" would be considered CORRECT, because "award" is presumed in the question.-For the question "What is the height of Jason Wei in meters?", the gold target is "1.73 m". The predicted answer "1.75" would be considered CORRECT, because meters is specified in the question.-For the question "What is the name of Barack Obama's wife?", the gold target is "Michelle Obama". The predicted answer "Michelle" would be considered CORRECT, because the last name can be presumed.-Do not punish for typos in people's name if it's clearly the same name. -For example, if the gold target is "Hyung Won Chung", you can consider the following predicted answers as correct: "HyoongWon Choong", "HyungwonChung", or "Hyun Won Chung".
 
-Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT ATTEMPTED. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer. 
+Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT ATTEMPTED. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer.
+```
+Question:{question}
+Goldtarget:{ground_truth}
+Predictedanswer:{prediction}
 ```
-Question:{question} 
-Goldtarget:{target} 
-Predictedanswer:{predicted_answer} 
-``` 
-Grade the predicted answer ofthe question as one of: A: CORRECT B: INCORRECT C: NOT_ATTEMPTED Just return the letter "A", "B", or "C", with no text around it.
-""".strip()
+Grade the predicted answer ofthe question as one of: A: CORRECT B: INCORRECT C: NOT_ATTEMPTED Just return the letter "A", "B", or "C", with no text around it."""
+
 
-        response = self.client.chat.completions.create(model=self.model_name, messages=[{"role": "user", "content": prompt}], temperature=0, max_tokens=5)
+def _parse_abc_response(text: str) -> VerifyResult:
+    """Parse A/B/C letter response from VideoEvalPro judge."""
+    answer = text.strip()
+    correct = answer[:1].upper() == "A" if answer else False
+    return VerifyResult(score=1.0 if correct else 0.0, is_correct=correct, raw_output=text)
 
-        answer = response.choices[0].message.content.strip()
-        # print(f"---------------------->>>>gpt answer: {answer}")
-        return answer[0].upper() == "A"
 
+# ---------------------------------------------------------------------------
+# Lazy pipeline singleton
+# ---------------------------------------------------------------------------
 
-def safe_judge_with_retry(model, question, gt, pred, max_retries=10, delay=2):
-    for attempt in range(max_retries):
-        try:
-            return model(question, gt, pred)
-        except Exception as e:
-            print(f"Attempt {attempt + 1} failed: {e}")
-            time.sleep(delay)
-    print(f"All {max_retries} attempts failed.")
-    return {"correct": False, "reason": "Failed after multiple retries"}
+_pipeline = None
+_pipeline_lock = threading.Lock()
 
 
-def safe_strip(x):
+def _get_pipeline() -> VerificationPipeline:
+    global _pipeline
+    if _pipeline is None:
+        with _pipeline_lock:
+            if _pipeline is None:  # double-check
+                _pipeline = VerificationPipeline(
+                    extractors=[StripReasoningExtractor()],
+                    verifier=OpenAIVerifier(
+                        model="gpt-4o-2024-11-20",
+                        custom_prompt=VIDEOEVALPRO_JUDGE_PROMPT,
+                        response_parser=_parse_abc_response,
+                        max_retries=10,
+                        retry_delay=2.0,
+                        max_tokens=5,
+                    ),
+                )
+    return _pipeline
+
+
+# ---------------------------------------------------------------------------
+# process_results / aggregate
+# ---------------------------------------------------------------------------
+
+
+def _safe_strip(x):
     return x.strip() if isinstance(x, str) else ""
 
 
@@ -92,17 +115,20 @@ def videoevalpro_process_results(doc, results):
     Returns:
         a dictionary with key: metric name, value: metric value
     """
-    question = safe_strip(doc.get("question", ""))
-    text_gt = safe_strip(doc.get("answer_text", ""))
-    task_type = safe_strip(doc.get("qa_type", ""))
-    pred_ans = results[0]
-
-    model = GPT4oJudge()
-    judge_result = safe_judge_with_retry(model, question, text_gt, pred_ans, max_retries=10, delay=2)
+    question = _safe_strip(doc.get("question", ""))
+    text_gt = _safe_strip(doc.get("answer_text", ""))
+    task_type = _safe_strip(doc.get("qa_type", ""))
 
-    data_dict = {"question": question, "task_type": task_type, "text_gt": text_gt, "pred_ans": pred_ans, "judge_result": judge_result}
+    pipeline = _get_pipeline()
+    result = pipeline(question=question, prediction=results[0], ground_truth=text_gt)
 
-    # print(f"---------------------->>>>data_dict: {data_dict}")
+    data_dict = {
+        "question": question,
+        "task_type": task_type,
+        "text_gt": text_gt,
+        "pred_ans": result.metadata.get("extracted_prediction", results[0]),
+        "judge_result": result.is_correct,
+    }
 
     return {"videoevalpro_score": data_dict}
 
@@ -122,11 +148,13 @@ def videoevalpro_aggregate_results(results):
 
     for result in results:
         task_type = result["task_type"]
+        if task_type not in category2score:
+            category2score[task_type] = {"correct": 0, "answered": 0}
         category2score[task_type]["answered"] += 1
-        category2score[task_type]["correct"] += result["judge_result"] == True
+        category2score[task_type]["correct"] += result["judge_result"] is True
         # overall score
         category2score["overall"]["answered"] += 1
-        category2score["overall"]["correct"] += result["judge_result"] == True
+        category2score["overall"]["correct"] += result["judge_result"] is True
 
     final_scores = {}
     for task_type, score in category2score.items():
diff --git a/lmms_eval/tasks/voicebench/utils.py b/lmms_eval/tasks/voicebench/utils.py
index e7e67b68b..117e18e50 100644
--- a/lmms_eval/tasks/voicebench/utils.py
+++ b/lmms_eval/tasks/voicebench/utils.py
@@ -1,11 +1,14 @@
 import os
 import random
 import re
+import threading
 
 import numpy as np
 from loguru import logger as eval_logger
 
 from lmms_eval.llm_judge import ServerConfig, get_server
+from lmms_eval.verifiers import VerificationPipeline, VerifyResult
+from lmms_eval.verifiers.openai import OpenAIVerifier
 
 API_TYPE = os.getenv("API_TYPE", "openai")
 # Use JUDGE_MODEL_VERSION instead of MODEL_VERSION
@@ -134,13 +137,7 @@ def voicebench_aggregate_results(results):
     return accuracy
 
 
-# Evaluation method for alpacaeval, commoneval and wildvoice
-def voicebench_process_results_open(doc, results):
-    parsed_preds = []
-    scores = []
-
-    # Open-ended evaluation prompt template
-    meta_prompt_open = """I need your help to evaluate the performance of several models in the speech interaction scenario. The models will receive a speech input from the user, which they need to understand and respond to with a speech output.
+VOICEBENCH_OPEN_JUDGE_PROMPT = """I need your help to evaluate the performance of several models in the speech interaction scenario. The models will receive a speech input from the user, which they need to understand and respond to with a speech output.
 Your task is to rate the model's responses based on the provided user input transcription [Instruction] and the model's output transcription [Response].
 
 Please evaluate the response on a scale of 1 to 5:
@@ -157,6 +154,48 @@ def voicebench_process_results_open(doc, results):
 After evaluating, please output the score only without anything else.
 You don't need to provide any explanations."""
 
+
+# Previously sent as a system message; now included as a prefix in the user prompt.
+_VOICEBENCH_OPEN_SYSTEM_INSTRUCTION = "You are a helpful assistant who tries to help answer the user's question."
+
+
+def _voicebench_open_prompt_fn(question, prediction, ground_truth, **kwargs):
+    prompt = VOICEBENCH_OPEN_JUDGE_PROMPT.format(prompt=question, response=prediction)
+    return f"{_VOICEBENCH_OPEN_SYSTEM_INSTRUCTION}\n\n{prompt}"
+
+
+def _parse_raw_score(text):
+    """Parse raw 1-5 score without normalization."""
+    numbers = re.findall(r"\d+(?:\.\d+)?", text.strip())
+    score = float(numbers[0]) if numbers else 0.0
+    return VerifyResult(score=score, is_correct=score >= 3.0, raw_output=text)
+
+
+_voicebench_open_pipeline = None
+_voicebench_open_pipeline_lock = threading.Lock()
+
+
+def _get_voicebench_open_pipeline():
+    global _voicebench_open_pipeline
+    if _voicebench_open_pipeline is None:
+        with _voicebench_open_pipeline_lock:
+            if _voicebench_open_pipeline is None:  # double-check
+                _voicebench_open_pipeline = VerificationPipeline(
+                    verifier=OpenAIVerifier(
+                        model=JUDGE_MODEL_VERSION,
+                        custom_prompt=_voicebench_open_prompt_fn,
+                        response_parser=_parse_raw_score,
+                        temperature=0.5,
+                        max_tokens=10,
+                    ),
+                )
+    return _voicebench_open_pipeline
+
+
+# Evaluation method for alpacaeval, commoneval and wildvoice
+def voicebench_process_results_open(doc, results):
+    scores = []
+
     for pred in results:
         prediction = pred.strip() if isinstance(pred, str) else str(pred)
 
@@ -171,34 +210,9 @@ def voicebench_process_results_open(doc, results):
 
         instruction_text = get_column_value(doc, ["prompt", "instruction", "question", "query", "source_text", "transcript", "transcription", "audio_text", "text"])
 
-        formatted_prompt = meta_prompt_open.format(prompt=instruction_text, response=prediction)
-
-        try:
-            from lmms_eval.llm_judge.protocol import Request, ServerConfig
-
-            custom_config = ServerConfig(model_name=JUDGE_MODEL_VERSION, temperature=0.5, max_tokens=10)
-
-            request = Request(messages=[{"role": "system", "content": "You are a helpful assistant who tries to help answer the user's question."}, {"role": "user", "content": formatted_prompt}], config=custom_config)
-
-            response = server.evaluate(request)
-
-            if response.success:
-                judge_response = response.content.strip()
-                try:
-                    judge_score = int(judge_response)
-                except ValueError:
-                    eval_logger.error(f"Failed to parse score: {judge_response}")
-                    judge_score = 0.0
-            else:
-                eval_logger.error(f"Judge evaluation failed: {response.content}")
-                judge_score = 0.0
-
-        except Exception as e:
-            eval_logger.error(f"Error getting judge response: {e}")
-            judge_score = 0.0
-
-        scores.append(judge_score)
-        parsed_preds.append(prediction)
+        pipeline = _get_voicebench_open_pipeline()
+        result = pipeline(question=instruction_text, prediction=prediction, ground_truth="")
+        scores.append(result.score)
 
     avg_score = sum(scores) / len(scores) if scores else 0.0
     return {"llm_as_judge_eval": avg_score}