feat: add MiniMax as LLM judge provider

lmms_eval/llm_judge/factory.py

@@ -5,17 +5,19 @@
 from .protocol import ServerConfig
 from .providers import (
     AsyncAzureOpenAIProvider,
+    AsyncMiniMaxProvider,
     AsyncOpenAIProvider,
     AzureOpenAIProvider,
     DummyProvider,
+    MiniMaxProvider,
     OpenAIProvider,
 )
 
 
 class ProviderFactory:
     """Factory for creating judge instances based on configuration"""
 
-    _provider_classes = {"openai": OpenAIProvider, "azure": AzureOpenAIProvider, "async_openai": AsyncOpenAIProvider, "async_azure": AsyncAzureOpenAIProvider, "dummy": DummyProvider}
+    _provider_classes = {"openai": OpenAIProvider, "azure": AzureOpenAIProvider, "async_openai": AsyncOpenAIProvider, "async_azure": AsyncAzureOpenAIProvider, "minimax": MiniMaxProvider, "async_minimax": AsyncMiniMaxProvider, "dummy": DummyProvider}
 
     # TODO
     # This should actually be a decorator that registers the class

lmms_eval/llm_judge/providers/__init__.py

@@ -1,13 +1,17 @@
 from .async_azure_openai import AsyncAzureOpenAIProvider
+from .async_minimax import AsyncMiniMaxProvider
 from .async_openai import AsyncOpenAIProvider
 from .azure_openai import AzureOpenAIProvider
 from .dummy import DummyProvider
+from .minimax import MiniMaxProvider
 from .openai import OpenAIProvider
 
 __all__ = [
     "OpenAIProvider",
     "AzureOpenAIProvider",
     "AsyncOpenAIProvider",
     "AsyncAzureOpenAIProvider",
+    "MiniMaxProvider",
+    "AsyncMiniMaxProvider",
     "DummyProvider",
 ]

lmms_eval/llm_judge/providers/async_minimax.py

@@ -0,0 +1,170 @@
+import asyncio
+import os
+from typing import Dict, List, Optional, Union
+
+import aiohttp
+from loguru import logger as eval_logger
+
+from lmms_eval.models.model_utils.usage_metrics import log_usage
+
+from ..base import AsyncServerInterface
+from ..protocol import Request, Response, ServerConfig
+from .minimax import MiniMaxProvider, _clamp_temperature, _strip_think_tags
+
+
+class AsyncMiniMaxProvider(AsyncServerInterface):
+    """Async MiniMax API implementation of the Judge interface.
+
+    Uses the same OpenAI-compatible endpoint as :class:`MiniMaxProvider`
+    but through an asynchronous client (``AsyncOpenAI`` or ``aiohttp``).
+    """
+
+    MINIMAX_BASE_URL = MiniMaxProvider.MINIMAX_BASE_URL
+
+    def __init__(self, config: Optional[ServerConfig] = None):
+        super().__init__(config)
+        self.api_key = os.getenv("MINIMAX_API_KEY", "")
+        self.api_url = f"{self.MINIMAX_BASE_URL}/chat/completions"
+
+        self.use_async_client = False
+        try:
+            from openai import AsyncOpenAI
+
+            self.async_client = AsyncOpenAI(
+                api_key=self.api_key,
+                base_url=self.MINIMAX_BASE_URL,
+            )
+            self.use_async_client = True
+        except ImportError:
+            eval_logger.warning(
+                "AsyncOpenAI client not available, using aiohttp for MiniMax"
+            )
+
+    def is_available(self) -> bool:
+        return bool(self.api_key)
+
+    async def evaluate_async(self, request: Request) -> Response:
+        """Evaluate using MiniMax API asynchronously."""
+        if not self.is_available():
+            raise ValueError("MiniMax API key not configured (set MINIMAX_API_KEY)")
+
+        config = request.config or self.config
+        messages = self.prepare_messages(request)
+
+        if request.images:
+            messages = self._add_images_to_messages(messages, request.images)
+
+        payload = {
+            "model": config.model_name,
+            "messages": messages,
+            "temperature": _clamp_temperature(config.temperature),
+            "max_tokens": config.max_tokens,
+        }
+
+        if config.top_p is not None:
+            payload["top_p"] = config.top_p
+
+        if config.response_format == "json":
+            payload["response_format"] = {"type": "json_object"}
+
+        async with self.semaphore:
+            for attempt in range(config.num_retries):
+                try:
+                    if self.use_async_client:
+                        response = await self.async_client.chat.completions.create(
+                            **payload
+                        )
+                        content = response.choices[0].message.content
+                        model_used = response.model
+                        usage = (
+                            response.usage.model_dump()
+                            if hasattr(response.usage, "model_dump")
+                            else None
+                        )
+                        raw_response = response
+                    else:
+                        response = await self._make_async_request(
+                            payload, config.timeout
+                        )
+                        content = response["choices"][0]["message"]["content"]
+                        model_used = response["model"]
+                        usage = response.get("usage")
+                        raw_response = response
+
+                    content = _strip_think_tags(content)
+
+                    # Log usage
+                    if (
+                        self.use_async_client
+                        and hasattr(response, "usage")
+                        and response.usage
+                    ):
+                        log_usage(
+                            model_name=model_used or config.model_name,
+                            task_name=None,
+                            input_tokens=getattr(
+                                response.usage, "prompt_tokens", 0
+                            )
+                            or 0,
+                            output_tokens=getattr(
+                                response.usage, "completion_tokens", 0
+                            )
+                            or 0,
+                            reasoning_tokens=0,
+                            source="judge",
+                        )
+                    elif not self.use_async_client and isinstance(usage, dict):
+                        log_usage(
+                            model_name=model_used or config.model_name,
+                            task_name=None,
+                            input_tokens=usage.get("prompt_tokens", 0) or 0,
+                            output_tokens=usage.get("completion_tokens", 0) or 0,
+                            reasoning_tokens=0,
+                            source="judge",
+                        )
+
+                    return Response(
+                        content=content.strip(),
+                        model_used=model_used,
+                        usage=usage,
+                        raw_response=raw_response,
+                    )
+
+                except Exception as e:
+                    eval_logger.warning(
+                        f"MiniMax async attempt {attempt + 1}/{config.num_retries} "
+                        f"failed: {e}"
+                    )
+                    if attempt < config.num_retries - 1:
+                        await asyncio.sleep(config.retry_delay)
+                    else:
+                        eval_logger.error(
+                            f"All {config.num_retries} MiniMax async attempts failed"
+                        )
+                        raise
+
+    async def _make_async_request(self, payload: Dict, timeout: int) -> Dict:
+        """Make async HTTP request to MiniMax API."""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                self.api_url,
+                headers=headers,
+                json=payload,
+                timeout=aiohttp.ClientTimeout(total=timeout),
+            ) as response:
+                response.raise_for_status()
+                return await response.json()
+
+    def _add_images_to_messages(
+        self, messages: List[Dict], images: List[Union[str, bytes]]
+    ) -> List[Dict]:
+        """Add images to messages – reuse from MiniMaxProvider."""
+        return MiniMaxProvider._add_images_to_messages(self, messages, images)
+
+    def _encode_image(self, image_path: str) -> str:
+        """Encode image to base64 – reuse from MiniMaxProvider."""
+        return MiniMaxProvider._encode_image(self, image_path)