refactor: unify OpenAI model naming via Registry V2 conventions (#1084)

* refactor: unify OpenAI model naming via Registry V2 conventions

Rename canonical model IDs to shorter, consistent names:
- openai_compatible -> openai
- async_openai_compatible_chat -> async_openai

Old names preserved as backward-compatible aliases via new
MODEL_ALIASES dict wired into _build_builtin_manifests().

File renames:
- models/simple/openai_compatible.py -> models/simple/openai.py
- models/chat/openai_compatible.py -> models/chat/openai.py

* style: fix isort import formatting in chat/openai.py

Author: Luodian

docs/current_tasks.md

@@ -522,8 +522,8 @@ python -m lmms_eval --tasks list_with_num
 | `vllm_generate` | VLLMGenerate | vLLM generation mode |
 | `sglang` | Sglang | SGLang serving backend |
 | `huggingface` | Huggingface | Generic HuggingFace models |
-| `openai_compatible` | OpenAICompatible | OpenAI-compatible APIs |
-| `async_openai` | AsyncOpenAIChat | Async OpenAI chat |
+| `openai` | OpenAICompatible | OpenAI-compatible APIs (aliases: `openai_compatible`, `openai_compatible_chat`) |
+| `async_openai` | AsyncOpenAIChat | Async OpenAI chat (alias: `async_openai_compatible_chat`) |
 
 ### Simple/Legacy Models
 | Model Name | Class | Modality |

lmms_eval/models/__init__.py

@@ -63,7 +63,7 @@
     "mplug_owl_video": "mplug_Owl",
     "ola": "Ola",
     "omnivinci": "OmniVinci",
-    "openai_compatible": "OpenAICompatible",
+    "openai": "OpenAICompatible",
     "oryx": "Oryx",
     "phi3v": "Phi3v",
     "phi4_multimodal": "Phi4",
@@ -107,7 +107,7 @@
     "qwen3_vl": "Qwen3_VL",
     "qwen2_5_vl": "Qwen2_5_VL",
     "thyme": "Thyme",
-    "openai_compatible": "OpenAICompatible",
+    "openai": "OpenAICompatible",
     "vllm": "VLLM",
     "vllm_generate": "VLLMGenerate",
     "sglang": "Sglang",
@@ -117,6 +117,11 @@
     "llava_onevision1_5": "Llava_OneVision1_5",
 }
 
+MODEL_ALIASES: dict[str, tuple[str, ...]] = {
+    "openai": ("openai_compatible", "openai_compatible_chat"),
+    "async_openai": ("async_openai_compatible_chat",),
+}
+
 
 def _build_class_path(
     model_name: str,
@@ -136,11 +141,13 @@ def _build_builtin_manifests() -> list[ModelManifest]:
     for model_id in model_ids:
         simple_class = AVAILABLE_SIMPLE_MODELS.get(model_id)
         chat_class = AVAILABLE_CHAT_TEMPLATE_MODELS.get(model_id)
+        aliases = MODEL_ALIASES.get(model_id, ())
         manifests.append(
             ModelManifest(
                 model_id=model_id,
                 simple_class_path=(_build_class_path(model_id, "simple", simple_class) if simple_class else None),
                 chat_class_path=(_build_class_path(model_id, "chat", chat_class) if chat_class else None),
+                aliases=aliases,
             ),
         )
     return manifests

lmms_eval/models/chat/async_openai.py

@@ -31,7 +31,7 @@
 load_dotenv(verbose=True)
 
 
-@register_model("async_openai_compatible_chat")
+@register_model("async_openai")
 class AsyncOpenAIChat(lmms):
     is_simple = False
 

…ms_eval/models/chat/openai_compatible.py lmms_eval/models/chat/openai.pylmms_eval/models/chat/openai_compatible.py renamed to lmms_eval/models/chat/openai.py lmms_eval/models/chat/openai_compatible.py renamed to lmms_eval/models/chat/openai.py

@@ -14,9 +14,7 @@
     is_rate_limit_error,
 )
 from lmms_eval.models.model_utils.gen_metrics import log_metrics
-from lmms_eval.models.simple.openai_compatible import (
-    OpenAICompatible as OpenAICompatibleSimple,
-)
+from lmms_eval.models.simple.openai import OpenAICompatible as OpenAICompatibleSimple
 from lmms_eval.protocol import ChatMessages
 
 VideoReader, _ = optional_import("decord", "VideoReader")
@@ -25,7 +23,7 @@
 load_dotenv(verbose=True)
 
 
-@register_model("openai_compatible_chat")
+@register_model("openai")
 class OpenAICompatible(OpenAICompatibleSimple):
     is_simple = False
 

…_eval/models/simple/openai_compatible.py lmms_eval/models/simple/openai.pylmms_eval/models/simple/openai_compatible.py renamed to lmms_eval/models/simple/openai.py lmms_eval/models/simple/openai_compatible.py renamed to lmms_eval/models/simple/openai.py

@@ -37,7 +37,7 @@
 load_dotenv(verbose=True)
 
 
-@register_model("openai_compatible")
+@register_model("openai")
 class OpenAICompatible(lmms):
     def __init__(
         self,

lmms_eval/tui/discovery.py

@@ -122,7 +122,7 @@ def get_popular_tasks() -> list[tuple[str, str]]:
 
 def get_popular_models() -> list[tuple[str, str]]:
     return [
-        ("openai_compatible", "OpenAI Compatible API"),
+        ("openai", "OpenAI Compatible API"),
         ("qwen2_5_vl", "Qwen2.5-VL"),
         ("qwen2_5_vl_chat", "Qwen2.5-VL Chat"),
         ("llava_onevision", "LLaVA-OneVision"),

test/eval/test_model_registry_v2.py

@@ -77,10 +77,10 @@ def test_representative_aliases_are_resolvable(self):
         )
         registry.register_manifest(
             ModelManifest(
-                model_id="openai_compatible",
-                simple_class_path="lmms_eval.models.simple.openai_compatible.OpenAICompatible",
-                chat_class_path="lmms_eval.models.chat.openai_compatible.OpenAICompatible",
-                aliases=("openai_compatible_chat",),
+                model_id="openai",
+                simple_class_path="lmms_eval.models.simple.openai.OpenAICompatible",
+                chat_class_path="lmms_eval.models.chat.openai.OpenAICompatible",
+                aliases=("openai_compatible", "openai_compatible_chat"),
             ),
         )
         registry.register_manifest(
@@ -94,7 +94,11 @@ def test_representative_aliases_are_resolvable(self):
         self.assertEqual(registry.resolve("vllm_chat").model_id, "vllm")
         self.assertEqual(
             registry.resolve("openai_compatible_chat").model_id,
-            "openai_compatible",
+            "openai",
+        )
+        self.assertEqual(
+            registry.resolve("openai_compatible").model_id,
+            "openai",
         )
         self.assertEqual(registry.resolve("sglang_runtime").model_id, "sglang")