Skip to content

Commit 1262e8d

Browse files
adamhrin-amdadamhrin-amd
committed
Add Inspect backends for MTBench
1 parent 1d196e2 commit 1262e8d

4 files changed

Lines changed: 318 additions & 1 deletion

File tree

‎src/lighteval/tasks/tasks/mt_bench/main.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
flow_judge_prompt_mt_bench_with_ref,
3434
flow_judge_prompt_mt_bench_without_ref,
3535
)
36+
from lighteval.tasks.tasks.mt_bench.main_inspect import TASKS_TABLE as _INSPECT_TASKS
3637

3738

3839
def mt_bench_prompt(line, task_name: str = ""):
@@ -94,4 +95,4 @@ def flow_judge_mt_bench_prompt(question, answer, options, gold):
9495
)
9596

9697

97-
TASKS_TABLE = [task]
98+
TASKS_TABLE = [task] + _INSPECT_TASKS

‎src/lighteval/tasks/tasks/mt_bench/main_inspect.py‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
"""
2+
name:
3+
MT-Bench (inspect_ai)
4+
5+
dataset:
6+
lighteval/mt-bench
7+
8+
abstract:
9+
inspect_ai-compatible version of MT-Bench. Uses the scorer model server
10+
(SCORER_MODEL_BASE_URL / SCORER_MODEL_PATH env vars) as the judge.
11+
Supports 2-turn conversation; scores each turn independently.
12+
13+
languages:
14+
english
15+
16+
tags:
17+
conversational, generation, multi-turn
18+
"""
19+
20+
import os
21+
import re
22+
23+
from inspect_ai.dataset import Sample
24+
from inspect_ai.model import ChatMessageUser, GenerateConfig, get_model
25+
from inspect_ai.scorer import Score, mean, scorer, stderr
26+
from inspect_ai.solver import Generate, TaskState, generate, solver
27+
28+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
29+
from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import (
30+
flow_judge_prompt_mt_bench_with_ref,
31+
flow_judge_prompt_mt_bench_without_ref,
32+
)
33+
34+
35+
def _scorer_endpoint_config(task_name: str = "mt_bench_inspect"):
36+
base_url = os.environ.get("SCORER_MODEL_BASE_URL")
37+
if not base_url:
38+
raise RuntimeError(f"SCORER_MODEL_BASE_URL must be set for {task_name}.")
39+
40+
model_name = os.environ.get("SCORER_MODEL_PATH")
41+
if not model_name:
42+
raise RuntimeError(f"SCORER_MODEL_PATH must be set for {task_name}.")
43+
44+
return base_url, model_name, os.environ.get("VLLM_API_KEY", "inspectai")
45+
46+
47+
def _get_scorer_model(task_name: str = "mt_bench_inspect"):
48+
base_url, model_name, api_key = _scorer_endpoint_config(task_name)
49+
return get_model(
50+
f"openai-api/scorer/{model_name}",
51+
config=GenerateConfig(
52+
extra_body={"chat_template_kwargs": {"enable_thinking": False}},
53+
),
54+
base_url=base_url,
55+
api_key=api_key,
56+
)
57+
58+
59+
def _parse_judge_score(text: str) -> int:
60+
match = re.search(r"<score>\s*(\d)\s*</score>", text)
61+
return int(match.group(1)) if match else 0
62+
63+
64+
def record_to_sample(record):
65+
return Sample(
66+
input=record["turns"][0],
67+
metadata={
68+
"turns": record["turns"],
69+
"reference": record.get("reference", []),
70+
"category": record.get("category", ""),
71+
"question_id": record.get("question_id", ""),
72+
},
73+
)
74+
75+
76+
@solver
77+
def require_scorer_endpoint(task_name: str = "mt_bench_inspect"):
78+
async def solve(state: TaskState, generate: Generate) -> TaskState:
79+
_scorer_endpoint_config(task_name)
80+
return state
81+
82+
return solve
83+
84+
85+
@solver
86+
def append_second_turn():
87+
async def solve(state: TaskState, generate: Generate) -> TaskState:
88+
turns = state.metadata["turns"]
89+
if len(turns) > 1:
90+
state.messages.append(ChatMessageUser(content=turns[1]))
91+
return state
92+
93+
return solve
94+
95+
96+
@scorer(metrics={"turn_1": [mean(), stderr()], "turn_2": [mean(), stderr()]})
97+
def mt_bench_scorer():
98+
judge = None
99+
100+
async def score(state: TaskState, target) -> Score:
101+
nonlocal judge
102+
if judge is None:
103+
judge = _get_scorer_model()
104+
105+
turns = state.metadata["turns"]
106+
references = state.metadata.get("reference", [])
107+
assistant_messages = [m for m in state.messages if m.role == "assistant"]
108+
109+
scores = {}
110+
for i, question in enumerate(turns):
111+
if i >= len(assistant_messages):
112+
scores[f"turn_{i + 1}"] = 0
113+
continue
114+
115+
message = assistant_messages[i]
116+
raw = message.text if hasattr(message, "text") else str(message.content)
117+
answer = raw
118+
ref = references[i] if references and i < len(references) else None
119+
120+
if ref:
121+
messages = flow_judge_prompt_mt_bench_with_ref(question, [], answer, ref)
122+
else:
123+
messages = flow_judge_prompt_mt_bench_without_ref(question, [], answer, None)
124+
125+
judge_input = [ChatMessageUser(content=m["content"]) for m in messages if m["role"] == "user"]
126+
output = await judge.generate(
127+
input=judge_input,
128+
config=GenerateConfig(temperature=0, max_tokens=512),
129+
)
130+
scores[f"turn_{i + 1}"] = _parse_judge_score(output.completion)
131+
132+
if "turn_2" not in scores:
133+
scores["turn_2"] = 0
134+
135+
return Score(
136+
value=scores,
137+
explanation=f"category={state.metadata.get('category', '')} id={state.metadata.get('question_id', '')}",
138+
)
139+
140+
return score
141+
142+
143+
mt_bench_inspect = LightevalTaskConfig(
144+
name="mt_bench_inspect",
145+
prompt_function=None,
146+
hf_repo="lighteval/mt-bench",
147+
hf_subset="default",
148+
hf_avail_splits=["train"],
149+
evaluation_splits=["train"],
150+
few_shots_split="",
151+
few_shots_select="random",
152+
metrics=[],
153+
generation_size=1024,
154+
stop_sequence=[],
155+
sample_fields=record_to_sample,
156+
solver=[require_scorer_endpoint(), generate(cache=True), append_second_turn(), generate(cache=True)],
157+
scorer=mt_bench_scorer(),
158+
)
159+
160+
161+
TASKS_TABLE = [mt_bench_inspect]

‎src/lighteval/tasks/tasks/mt_bench_fi/main.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
"""
2+
name:
3+
MT-Bench Finnish
4+
5+
dataset:
6+
LumiOpen/mtbench_multi
7+
8+
abstract:
9+
Finnish MT-Bench task for the inspect_ai backend. Uses an OpenAI-compatible
10+
scorer model endpoint as the judge.
11+
12+
languages:
13+
finnish
14+
15+
tags:
16+
conversational, generation, multi-turn
17+
"""
18+
19+
from lighteval.tasks.tasks.mt_bench_fi.main_inspect import mt_bench_fi_inspect
20+
21+
22+
TASKS_TABLE = [mt_bench_fi_inspect]

‎src/lighteval/tasks/tasks/mt_bench_fi/main_inspect.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
"""
2+
name:
3+
MT-Bench Finnish (inspect_ai)
4+
5+
dataset:
6+
LumiOpen/mtbench_multi
7+
8+
abstract:
9+
inspect_ai-compatible Finnish MT-Bench task. Uses the scorer model server
10+
(SCORER_MODEL_BASE_URL / SCORER_MODEL_PATH env vars) as the judge.
11+
Supports 2-turn conversation; scores each turn independently.
12+
13+
languages:
14+
finnish
15+
16+
tags:
17+
conversational, generation, multi-turn
18+
"""
19+
20+
from inspect_ai.dataset import Sample
21+
from inspect_ai.model import ChatMessageUser, GenerateConfig
22+
from inspect_ai.scorer import Score, mean, scorer, stderr
23+
from inspect_ai.solver import generate
24+
25+
from lighteval.tasks.lighteval_task import LightevalTaskConfig
26+
from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import (
27+
flow_judge_prompt_mt_bench_with_ref,
28+
flow_judge_prompt_mt_bench_without_ref,
29+
)
30+
from lighteval.tasks.tasks.mt_bench.main_inspect import (
31+
_get_scorer_model,
32+
_parse_judge_score,
33+
append_second_turn,
34+
require_scorer_endpoint,
35+
)
36+
37+
38+
TASK_NAME = "mt_bench_fi_inspect"
39+
FINNISH_JUDGE_NOTE = "Note: The task is in Finnish. Evaluate the response based on the Finnish language content."
40+
41+
42+
def _add_finnish_note(messages):
43+
noted_messages = []
44+
for message in messages:
45+
if message["role"] == "user":
46+
noted_messages.append(
47+
{**message, "content": message["content"].replace("# INPUT", f"{FINNISH_JUDGE_NOTE}\n\n# INPUT", 1)}
48+
)
49+
else:
50+
noted_messages.append(message)
51+
return noted_messages
52+
53+
54+
def record_to_sample(record):
55+
return Sample(
56+
input=record["turns"][0],
57+
metadata={
58+
"turns": record["turns"],
59+
"reference": record.get("reference", []),
60+
"category": record.get("category", ""),
61+
"question_id": record.get("question_id", ""),
62+
},
63+
)
64+
65+
66+
@scorer(metrics={"turn_1": [mean(), stderr()], "turn_2": [mean(), stderr()]})
67+
def mt_bench_fi_scorer():
68+
judge = None
69+
70+
async def score(state, target) -> Score:
71+
nonlocal judge
72+
if judge is None:
73+
judge = _get_scorer_model(TASK_NAME)
74+
75+
turns = state.metadata["turns"]
76+
references = state.metadata.get("reference", [])
77+
assistant_messages = [m for m in state.messages if m.role == "assistant"]
78+
79+
scores = {}
80+
for i, question in enumerate(turns):
81+
if i >= len(assistant_messages):
82+
scores[f"turn_{i + 1}"] = 0
83+
continue
84+
85+
message = assistant_messages[i]
86+
raw = message.text if hasattr(message, "text") else str(message.content)
87+
answer = raw
88+
ref = references[i] if references and i < len(references) else None
89+
90+
if ref:
91+
messages = flow_judge_prompt_mt_bench_with_ref(question, [], answer, ref)
92+
else:
93+
messages = flow_judge_prompt_mt_bench_without_ref(question, [], answer, None)
94+
95+
judge_input = [
96+
ChatMessageUser(content=m["content"]) for m in _add_finnish_note(messages) if m["role"] == "user"
97+
]
98+
output = await judge.generate(
99+
input=judge_input,
100+
config=GenerateConfig(temperature=0, max_tokens=512),
101+
)
102+
scores[f"turn_{i + 1}"] = _parse_judge_score(output.completion)
103+
104+
if "turn_2" not in scores:
105+
scores["turn_2"] = 0
106+
107+
return Score(
108+
value=scores,
109+
explanation=f"category={state.metadata.get('category', '')} id={state.metadata.get('question_id', '')}",
110+
)
111+
112+
return score
113+
114+
115+
mt_bench_fi_inspect = LightevalTaskConfig(
116+
name=TASK_NAME,
117+
prompt_function=None,
118+
hf_repo="LumiOpen/mtbench_multi",
119+
hf_subset="fi",
120+
hf_avail_splits=["test"],
121+
evaluation_splits=["test"],
122+
few_shots_split="",
123+
few_shots_select="random",
124+
metrics=[],
125+
generation_size=1024,
126+
stop_sequence=[],
127+
sample_fields=record_to_sample,
128+
solver=[require_scorer_endpoint(TASK_NAME), generate(cache=True), append_second_turn(), generate(cache=True)],
129+
scorer=mt_bench_fi_scorer(),
130+
)
131+
132+
133+
TASKS_TABLE = [mt_bench_fi_inspect]

0 commit comments

Comments
 (0)