Switch to new together annotator. (#1053)

* Switching over to new Together dedicated endpoint for ensemble evaluator.

Author: wpietri

plugins/huggingface/modelgauge/suts/huggingface_chat_completion.py

@@ -7,10 +7,10 @@
 from pydantic import BaseModel
 
 from modelgauge.auth.huggingface_inference_token import HuggingFaceInferenceToken
-from modelgauge.prompt import TextPrompt
+from modelgauge.prompt import TextPrompt, ChatPrompt
 from modelgauge.secret_values import InjectSecret
 from modelgauge.sut import PromptResponseSUT, SUTOptions, SUTResponse, TokenProbability, TopTokens
-from modelgauge.sut_capabilities import AcceptsTextPrompt, ProducesPerTokenLogProbabilities
+from modelgauge.sut_capabilities import AcceptsTextPrompt, ProducesPerTokenLogProbabilities, AcceptsChatPrompt
 from modelgauge.sut_decorator import modelgauge_sut
 from modelgauge.sut_registry import SUTS
 
@@ -92,7 +92,7 @@ def translate_response(
         return SUTResponse(text=text, top_logprobs=logprobs)
 
 
-@modelgauge_sut(capabilities=[AcceptsTextPrompt, ProducesPerTokenLogProbabilities])
+@modelgauge_sut(capabilities=[AcceptsTextPrompt, AcceptsChatPrompt, ProducesPerTokenLogProbabilities])
 class HuggingFaceChatCompletionDedicatedSUT(BaseHuggingFaceChatCompletionSUT):
     """A Hugging Face SUT that is hosted on a dedicated inference endpoint and uses the chat_completion API."""
 
@@ -135,6 +135,16 @@ def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> Hugg
             **options.model_dump(),
         )
 
+    def translate_chat_prompt(self, prompt: ChatPrompt, options: SUTOptions) -> HuggingFaceChatCompletionRequest:
+        logprobs = None
+        if options.top_logprobs is not None:
+            logprobs = True
+        return HuggingFaceChatCompletionRequest(
+            messages=[ChatMessage(role=p.role.lower(), content=p.text) for p in prompt.messages],
+            logprobs=logprobs,
+            **options.model_dump(),
+        )
+
 
 @modelgauge_sut(capabilities=[AcceptsTextPrompt, ProducesPerTokenLogProbabilities])
 class HuggingFaceChatCompletionServerlessSUT(BaseHuggingFaceChatCompletionSUT):
@@ -181,6 +191,7 @@ def translate_text_prompt(self, prompt: TextPrompt, options: SUTOptions) -> Hugg
     "llama-3-1-tulu-3-8b": "bzk",  # check
     "llama-3-1-tulu-3-70b": "ome",
     "mistral-nemo-instruct-2407": "mgt",
+    "mixtral-8x22b-instruct-v0-1": "kog",
     "olmo-2-1124-13b-instruct": "ibo",
     "olmo-2-0325-32b-instruct": "yft",
     "qwen1-5-110b-chat": "gek",

plugins/huggingface/tests/test_huggingface_chat_completion.py

@@ -14,7 +14,8 @@
 from unittest.mock import Mock, patch
 
 from modelgauge.auth.huggingface_inference_token import HuggingFaceInferenceToken
-from modelgauge.prompt import TextPrompt
+from modelgauge.prompt import TextPrompt, ChatPrompt, ChatRole
+import modelgauge.prompt
 from modelgauge.sut import SUTOptions, SUTResponse, TokenProbability, TopTokens
 from modelgauge.suts.huggingface_chat_completion import (
     ChatMessage,
@@ -69,6 +70,17 @@ def test_huggingface_chat_completion_translate_text_prompt_request(fake_sut, top
     assert request == _make_sut_request(top_logprobs)
 
 
+@pytest.mark.parametrize("top_logprobs", [None, 2])
+def test_huggingface_chat_completion_translate_chat_prompt_request(fake_sut, top_logprobs):
+    request = fake_sut.translate_chat_prompt(
+        ChatPrompt(messages=[modelgauge.prompt.ChatMessage(text="some text prompt", role=ChatRole.user)]),
+        _make_sut_options(top_logprobs),
+    )
+
+    assert isinstance(request, HuggingFaceChatCompletionRequest)
+    assert request == _make_sut_request(top_logprobs)
+
+
 @pytest.mark.parametrize(
     "endpoint_status",
     [

src/modelgauge/secret_values.py

@@ -2,9 +2,10 @@
 from dataclasses import dataclass
 from typing import Generic, List, Mapping, Optional, Sequence, Type, TypeVar
 
-from modelgauge.general import get_concrete_subclasses
 from pydantic import BaseModel
 
+from modelgauge.general import get_concrete_subclasses
+
 
 class SecretDescription(BaseModel):
     """How to look up a secret and how to get the value if you don't have it."""
@@ -160,3 +161,8 @@ def inject(self, raw_secrets: RawSecrets) -> SecretType:
 
     def __repr__(self):
         return f"InjectSecret({self.secret_class.__name__})"
+
+
+class InjectAllSecrets(Injector):
+    def inject(self, raw_secrets: RawSecrets):
+        return raw_secrets

src/modelgauge/suts/together_client.py

@@ -387,6 +387,8 @@ def translate_response(self, request: TogetherInferenceRequest, response: Togeth
     "Mistral-7B-Instruct-v0.2": "mistralai/Mistral-7B-Instruct-v0.2",
     "Mixtral-8x7B-Instruct-v0.1": "mistralai/Mixtral-8x7B-Instruct-v0.1",
     "mistral-8x22b-instruct": "mistralai/Mixtral-8x22B-Instruct-v0.1",
+    "mistral-8x22b-instruct-nim": "mlc_ai_safety_2/nim/mistralai/mixtral-8x22b-instruct-v01-c5e6181e",
+    "mistral-8x22b-instruct-dedicated-together": "mlc_ai_safety_2/mistralai/Mixtral-8x22B-Instruct-v0.1-177be485",
     "deepseek-R1": "deepseek-ai/DeepSeek-R1",
     "deepseek-v3-together": "deepseek-ai/DeepSeek-V3",
     # No longer supported as of 2025-04-25