fix(llm_judge): OpenAI base URL, max_completion_tokens for reasoning models, PIL image support

lmms_eval/llm_judge/protocol.py

@@ -35,7 +35,7 @@ class Request:
     """Standard request format for judge evaluation"""
 
     messages: List[Dict[str, Any]]
-    images: Optional[List[Union[str, bytes]]] = None  # Image paths or base64 encoded
+    images: Optional[List[Union[str, bytes, "Image.Image"]]] = None  # Image paths, base64 bytes, or PIL images
     config: Optional[ServerConfig] = None
 
     # Structured input for specific judge types

lmms_eval/llm_judge/providers/async_openai.py

@@ -4,12 +4,13 @@
 
 import aiohttp
 from loguru import logger as eval_logger
+from PIL import Image
 
 from lmms_eval.models.model_utils.usage_metrics import log_usage
 
 from ..base import AsyncServerInterface
 from ..protocol import Request, Response, ServerConfig
-from .openai import OpenAIProvider
+from .openai import OpenAIProvider, build_sampling_kwargs
 
 
 class AsyncOpenAIProvider(AsyncServerInterface):
@@ -18,14 +19,15 @@ class AsyncOpenAIProvider(AsyncServerInterface):
     def __init__(self, config: Optional[ServerConfig] = None):
         super().__init__(config)
         self.api_key = os.getenv("OPENAI_API_KEY", "")
-        self.api_url = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions")
+        base_url = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1")
+        self.api_url = base_url.rstrip("/") + "/chat/completions"
 
         # Try to use async OpenAI client if available
         self.use_async_client = False
         try:
             from openai import AsyncOpenAI
 
-            self.async_client = AsyncOpenAI(api_key=self.api_key)
+            self.async_client = AsyncOpenAI(api_key=self.api_key, base_url=base_url)
             self.use_async_client = True
         except ImportError:
             eval_logger.warning("AsyncOpenAI client not available, using aiohttp")
@@ -49,8 +51,7 @@ async def evaluate_async(self, request: Request) -> Response:
         payload = {
             "model": config.model_name,
             "messages": messages,
-            "temperature": config.temperature,
-            "max_tokens": config.max_tokens,
+            **build_sampling_kwargs(config),
         }
 
         if config.top_p is not None:
@@ -118,7 +119,7 @@ async def _make_async_request(self, payload: Dict, timeout: int) -> Dict:
                 response.raise_for_status()
                 return await response.json()
 
-    def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes]]) -> List[Dict]:
+    def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes, Image.Image]]) -> List[Dict]:
         """Add images to messages - reuse from base implementation"""
         return OpenAIProvider._add_images_to_messages(self, messages, images)
 

lmms_eval/llm_judge/providers/openai.py

@@ -1,9 +1,10 @@
 import os
 import time
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 import requests
 from loguru import logger as eval_logger
+from PIL import Image
 
 from lmms_eval.models.model_utils.media_encoder import encode_image_to_base64
 from lmms_eval.models.model_utils.usage_metrics import log_usage
@@ -12,19 +13,35 @@
 from ..protocol import Request, Response, ServerConfig
 
 
+def _is_reasoning_model(model_name: str) -> bool:
+    """Reasoning models (gpt-5*, o1/o3/o4*) require max_completion_tokens and reject an explicit temperature."""
+    return model_name.startswith(("gpt-5", "o1", "o3", "o4"))
+
+
+def build_sampling_kwargs(config: ServerConfig) -> Dict[str, Any]:
+    """Token-limit and temperature kwargs for a chat.completions payload.
+
+    Reasoning models use max_completion_tokens and reject a non-default temperature
+    (the API returns 400), so temperature is omitted for them.
+    """
+    if _is_reasoning_model(config.model_name):
+        return {"max_completion_tokens": config.max_tokens}
+    return {"max_tokens": config.max_tokens, "temperature": config.temperature}
+
+
 class OpenAIProvider(ServerInterface):
     """OpenAI API implementation of the Judge interface"""
 
     def __init__(self, config: Optional[ServerConfig] = None):
         super().__init__(config)
         self.api_key = os.getenv("OPENAI_API_KEY", "")
-        self.api_url = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1/chat/completions/v1")
+        self.base_url = os.getenv("OPENAI_API_URL", "https://api.openai.com/v1")
 
         # Initialize OpenAI client
         try:
             from openai import OpenAI
 
-            self.client = OpenAI(api_key=self.api_key, base_url=self.api_url)
+            self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
             self.use_client = True
         except ImportError:
             eval_logger.warning("OpenAI client not available, falling back to requests")
@@ -49,8 +66,7 @@ def evaluate(self, request: Request) -> Response:
         payload = {
             "model": config.model_name,
             "messages": messages,
-            "temperature": config.temperature,
-            "max_tokens": config.max_tokens,
+            **build_sampling_kwargs(config),
         }
 
         if config.top_p is not None:
@@ -112,11 +128,12 @@ def _make_request(self, payload: Dict, timeout: int) -> Dict:
             "Content-Type": "application/json",
         }
 
-        response = requests.post(self.api_url, headers=headers, json=payload, timeout=timeout)
+        url = self.base_url.rstrip("/") + "/chat/completions"
+        response = requests.post(url, headers=headers, json=payload, timeout=timeout)
         response.raise_for_status()
         return response.json()
 
-    def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes]]) -> List[Dict]:
+    def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str, bytes, Image.Image]]) -> List[Dict]:
         """Add images to the last user message"""
         # Find the last user message
         for i in range(len(messages) - 1, -1, -1):
@@ -127,7 +144,11 @@ def _add_images_to_messages(self, messages: List[Dict], images: List[Union[str,
 
                 # Add images
                 for image in images:
-                    if isinstance(image, str):
+                    if isinstance(image, Image.Image):
+                        # PIL Image object
+                        base64_image = encode_image_to_base64(image, image_format="JPEG", convert_rgb=True, quality=85)
+                        messages[i]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}})
+                    elif isinstance(image, str):
                         # File path
                         base64_image = self._encode_image(image)
                         messages[i]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}})