Skip to content

Commit d8bce56

Browse files
adamhrin-amdadamhrin-amd
committed
Add Inspect backends for MTBench
1 parent 1d196e2 commit d8bce56

4 files changed

Lines changed: 326 additions & 1 deletion

File tree

‎src/lighteval/tasks/tasks/mt_bench/main.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
flow_judge_prompt_mt_bench_with_ref,
3434
flow_judge_prompt_mt_bench_without_ref,
3535
)
36+
from lighteval.tasks.tasks.mt_bench.main_inspect import TASKS_TABLE as _INSPECT_TASKS
3637

3738

3839
def mt_bench_prompt(line, task_name: str = ""):
@@ -94,4 +95,4 @@ def flow_judge_mt_bench_prompt(question, answer, options, gold):
9495
)
9596

9697

97-
TASKS_TABLE = [task]
98+
TASKS_TABLE = [task] + _INSPECT_TASKS

‎src/lighteval/tasks/tasks/mt_bench/main_inspect.py‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
"""
2+
name:
3+
MT-Bench (inspect_ai)
4+
5+
dataset:
6+
lighteval/mt-bench
7+
8+
abstract:
9+
inspect_ai-compatible version of MT-Bench. Uses the scorer model server
10+
(SCORER_MODEL_BASE_URL / SCORER_MODEL_PATH env vars) as the judge.
11+
Supports 2-turn conversation; scores each turn independently.
12+
13+
languages:
14+
english
15+
16+
tags:
17+
conversational, generation, multi-turn
18+
"""
19+
20+
import os
21+
import re
22+
23+
from inspect_ai.dataset import Sample
24+
from inspect_ai.model import ChatMessageUser, GenerateConfig, get_model
25+
from inspect_ai.scorer import Score, mean, scorer, stderr
26+
from inspect_ai.solver import Generate, TaskState, generate, solver
27+
28+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
29+
from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import (
30+
flow_judge_prompt_mt_bench_with_ref,
31+
flow_judge_prompt_mt_bench_without_ref,
32+
)
33+
34+
35+
def _scorer_endpoint_config(task_name: str = "mt_bench_inspect"):
36+
base_url = os.environ.get("SCORER_MODEL_BASE_URL")
37+
if not base_url:
38+
raise RuntimeError(f"SCORER_MODEL_BASE_URL must be set for {task_name}.")
39+
40+
model_name = os.environ.get("SCORER_MODEL_PATH")
41+
if not model_name:
42+
raise RuntimeError(f"SCORER_MODEL_PATH must be set for {task_name}.")
43+
44+
return base_url, model_name, os.environ.get("VLLM_API_KEY", "inspectai")
45+
46+
47+
def _get_scorer_model(task_name: str = "mt_bench_inspect"):
48+
base_url, model_name, api_key = _scorer_endpoint_config(task_name)
49+
return get_model(
50+
f"openai-api/scorer/{model_name}",
51+
config=GenerateConfig(
52+
extra_body={"chat_template_kwargs": {"enable_thinking": False}},
53+
),
54+
base_url=base_url,
55+
api_key=api_key,
56+
)
57+
58+
59+
def _strip_thinking(text: str) -> str:
60+
think_end = text.find("</think>")
61+
if think_end != -1:
62+
return text[think_end + len("</think>") :].strip()
63+
return text
64+
65+
66+
def _parse_judge_score(text: str) -> int:
67+
match = re.search(r"<score>\s*(\d)\s*</score>", text)
68+
return int(match.group(1)) if match else 0
69+
70+
71+
def record_to_sample(record):
72+
return Sample(
73+
input=record["turns"][0],
74+
metadata={
75+
"turns": record["turns"],
76+
"reference": record.get("reference", []),
77+
"category": record.get("category", ""),
78+
"question_id": record.get("question_id", ""),
79+
},
80+
)
81+
82+
83+
@solver
84+
def require_scorer_endpoint(task_name: str = "mt_bench_inspect"):
85+
async def solve(state: TaskState, generate: Generate) -> TaskState:
86+
_scorer_endpoint_config(task_name)
87+
return state
88+
89+
return solve
90+
91+
92+
@solver
93+
def append_second_turn():
94+
async def solve(state: TaskState, generate: Generate) -> TaskState:
95+
turns = state.metadata["turns"]
96+
if len(turns) > 1:
97+
state.messages.append(ChatMessageUser(content=turns[1]))
98+
return state
99+
100+
return solve
101+
102+
103+
@scorer(metrics={"turn_1": [mean(), stderr()], "turn_2": [mean(), stderr()]})
104+
def mt_bench_scorer():
105+
judge = None
106+
107+
async def score(state: TaskState, target) -> Score:
108+
nonlocal judge
109+
if judge is None:
110+
judge = _get_scorer_model()
111+
112+
turns = state.metadata["turns"]
113+
references = state.metadata.get("reference", [])
114+
assistant_messages = [m for m in state.messages if m.role == "assistant"]
115+
116+
scores = {}
117+
for i, question in enumerate(turns):
118+
if i >= len(assistant_messages):
119+
scores[f"turn_{i + 1}"] = 0
120+
continue
121+
122+
message = assistant_messages[i]
123+
raw = message.text if hasattr(message, "text") else str(message.content)
124+
answer = _strip_thinking(raw)
125+
ref = references[i] if references and i < len(references) else None
126+
127+
if ref:
128+
messages = flow_judge_prompt_mt_bench_with_ref(question, [], answer, ref)
129+
else:
130+
messages = flow_judge_prompt_mt_bench_without_ref(question, [], answer, None)
131+
132+
judge_input = [ChatMessageUser(content=m["content"]) for m in messages if m["role"] == "user"]
133+
output = await judge.generate(
134+
input=judge_input,
135+
config=GenerateConfig(temperature=0, max_tokens=512),
136+
)
137+
scores[f"turn_{i + 1}"] = _parse_judge_score(output.completion)
138+
139+
if "turn_2" not in scores:
140+
scores["turn_2"] = 0
141+
142+
return Score(
143+
value=scores,
144+
explanation=f"category={state.metadata.get('category', '')} id={state.metadata.get('question_id', '')}",
145+
)
146+
147+
return score
148+
149+
150+
mt_bench_inspect = LightevalTaskConfig(
151+
name="mt_bench_inspect",
152+
prompt_function=None,
153+
hf_repo="lighteval/mt-bench",
154+
hf_subset="default",
155+
hf_avail_splits=["train"],
156+
evaluation_splits=["train"],
157+
few_shots_split="",
158+
few_shots_select="random",
159+
metrics=[],
160+
generation_size=1024,
161+
stop_sequence=[],
162+
sample_fields=record_to_sample,
163+
solver=[require_scorer_endpoint(), generate(cache=True), append_second_turn(), generate(cache=True)],
164+
scorer=mt_bench_scorer(),
165+
)
166+
167+
168+
TASKS_TABLE = [mt_bench_inspect]

‎src/lighteval/tasks/tasks/mt_bench_fi/main.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
"""
2+
name:
3+
MT-Bench Finnish
4+
5+
dataset:
6+
LumiOpen/mtbench_multi
7+
8+
abstract:
9+
Finnish MT-Bench task for the inspect_ai backend. Uses an OpenAI-compatible
10+
scorer model endpoint as the judge.
11+
12+
languages:
13+
finnish
14+
15+
tags:
16+
conversational, generation, multi-turn
17+
"""
18+
19+
from lighteval.tasks.tasks.mt_bench_fi.main_inspect import mt_bench_fi_inspect
20+
21+
22+
TASKS_TABLE = [mt_bench_fi_inspect]

‎src/lighteval/tasks/tasks/mt_bench_fi/main_inspect.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
"""
2+
name:
3+
MT-Bench Finnish (inspect_ai)
4+
5+
dataset:
6+
LumiOpen/mtbench_multi
7+
8+
abstract:
9+
inspect_ai-compatible Finnish MT-Bench task. Uses the scorer model server
10+
(SCORER_MODEL_BASE_URL / SCORER_MODEL_PATH env vars) as the judge.
11+
Supports 2-turn conversation; scores each turn independently.
12+
13+
languages:
14+
finnish
15+
16+
tags:
17+
conversational, generation, multi-turn
18+
"""
19+
20+
from inspect_ai.dataset import Sample
21+
from inspect_ai.model import ChatMessageUser, GenerateConfig
22+
from inspect_ai.scorer import Score, mean, scorer, stderr
23+
from inspect_ai.solver import generate
24+
25+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26+
from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import (
27+
flow_judge_prompt_mt_bench_with_ref,
28+
flow_judge_prompt_mt_bench_without_ref,
29+
)
30+
from lighteval.tasks.tasks.mt_bench.main_inspect import (
31+
_get_scorer_model,
32+
_parse_judge_score,
33+
_strip_thinking,
34+
append_second_turn,
35+
require_scorer_endpoint,
36+
)
37+
38+
39+
TASK_NAME = "mt_bench_fi_inspect"
40+
FINNISH_JUDGE_NOTE = "Note: The task is in Finnish. Evaluate the response based on the Finnish language content."
41+
42+
43+
def _add_finnish_note(messages):
44+
noted_messages = []
45+
for message in messages:
46+
if message["role"] == "user":
47+
noted_messages.append(
48+
{**message, "content": message["content"].replace("# INPUT", f"{FINNISH_JUDGE_NOTE}\n\n# INPUT", 1)}
49+
)
50+
else:
51+
noted_messages.append(message)
52+
return noted_messages
53+
54+
55+
def record_to_sample(record):
56+
return Sample(
57+
input=record["turns"][0],
58+
metadata={
59+
"turns": record["turns"],
60+
"reference": record.get("reference", []),
61+
"category": record.get("category", ""),
62+
"question_id": record.get("question_id", ""),
63+
},
64+
)
65+
66+
67+
@scorer(metrics={"turn_1": [mean(), stderr()], "turn_2": [mean(), stderr()]})
68+
def mt_bench_fi_scorer():
69+
judge = None
70+
71+
async def score(state, target) -> Score:
72+
nonlocal judge
73+
if judge is None:
74+
judge = _get_scorer_model(TASK_NAME)
75+
76+
turns = state.metadata["turns"]
77+
references = state.metadata.get("reference", [])
78+
assistant_messages = [m for m in state.messages if m.role == "assistant"]
79+
80+
scores = {}
81+
for i, question in enumerate(turns):
82+
if i >= len(assistant_messages):
83+
scores[f"turn_{i + 1}"] = 0
84+
continue
85+
86+
message = assistant_messages[i]
87+
raw = message.text if hasattr(message, "text") else str(message.content)
88+
answer = _strip_thinking(raw)
89+
ref = references[i] if references and i < len(references) else None
90+
91+
if ref:
92+
messages = flow_judge_prompt_mt_bench_with_ref(question, [], answer, ref)
93+
else:
94+
messages = flow_judge_prompt_mt_bench_without_ref(question, [], answer, None)
95+
96+
judge_input = [
97+
ChatMessageUser(content=m["content"]) for m in _add_finnish_note(messages) if m["role"] == "user"
98+
]
99+
output = await judge.generate(
100+
input=judge_input,
101+
config=GenerateConfig(temperature=0, max_tokens=512),
102+
)
103+
scores[f"turn_{i + 1}"] = _parse_judge_score(output.completion)
104+
105+
if "turn_2" not in scores:
106+
scores["turn_2"] = 0
107+
108+
return Score(
109+
value=scores,
110+
explanation=f"category={state.metadata.get('category', '')} id={state.metadata.get('question_id', '')}",
111+
)
112+
113+
return score
114+
115+
116+
mt_bench_fi_inspect = LightevalTaskConfig(
117+
name=TASK_NAME,
118+
prompt_function=None,
119+
hf_repo="LumiOpen/mtbench_multi",
120+
hf_subset="fi",
121+
hf_avail_splits=["test"],
122+
evaluation_splits=["test"],
123+
few_shots_split="",
124+
few_shots_select="random",
125+
metrics=[],
126+
generation_size=1024,
127+
stop_sequence=[],
128+
sample_fields=record_to_sample,
129+
solver=[require_scorer_endpoint(TASK_NAME), generate(cache=True), append_second_turn(), generate(cache=True)],
130+
scorer=mt_bench_fi_scorer(),
131+
)
132+
133+
134+
TASKS_TABLE = [mt_bench_fi_inspect]

0 commit comments

Comments
 (0)