Skip to content

Commit 1d23344

Browse files
chaanimchaanim
authored
Multilingual AIME 2026 and English 25/26 for consintency with Multilingual Math-500 (#8)
1 parent f9ab622 commit 1d23344

1 file changed

Lines changed: 246 additions & 7 deletions

File tree

  • src/lighteval/tasks/multilingual/tasks

‎src/lighteval/tasks/multilingual/tasks/maime.py‎

Lines changed: 246 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
"""
22
name:
3-
mAIME2025
3+
mAIME2025, mAIME2026
44
55
dataset:
6-
LumiOpen/mAIME2025
6+
LumiOpen/mAIME2025, LumiOpen/mAIME2026
77
88
abstract:
9-
The Multilingual AIME 2025 (mAIME2025) is a multilingual version of the
10-
2025 AIME (American Invitational Mathematics Examination) problems,
11-
professionally translated into European languages. This dataset contains
12-
all 30 problems from AIME I and AIME II 2025, translated and
13-
human-reviewed by native speakers to preserve mathematical accuracy and
9+
The Multilingual AIME (mAIME) datasets are multilingual versions of the
10+
AIME (American Invitational Mathematics Examination) problems,
11+
professionally translated into European languages. Each dataset contains
12+
all 30 problems from AIME I and AIME II for the respective year, translated
13+
and human-reviewed by native speakers to preserve mathematical accuracy and
1414
LaTeX formatting.
1515
1616
languages:
@@ -35,6 +35,11 @@
3535

3636
# Per-language prompt templates
3737
MATH_PROMPT_TEMPLATES = {
38+
"en": dedent("""
39+
Solve the following math problem efficiently and clearly. The last line of your response should be of the following format: 'Therefore, the final answer is: $\\boxed{{ANSWER}}$. I hope it is correct' (without quotes) where ANSWER is just the final number or expression that solves the problem. Think step by step before answering.
40+
41+
{prompt}
42+
"""),
3843
"fi": dedent("""
3944
Ratkaise seuraava matemaattinen tehtävä tehokkaasti ja selkeästi.
4045
Vastauksesi viimeisen rivin tulee olla seuraavassa muodossa:
@@ -59,6 +64,10 @@ def record_to_sample(record):
5964
return Sample(input=record["question"], target=record["solution"])
6065

6166

67+
def record_to_sample_en(record):
68+
return Sample(input=record["problem"], target=record["answer"])
69+
70+
6271
def _maime_prompt_fn(lang: str):
6372
template = MATH_PROMPT_TEMPLATES[lang]
6473

@@ -73,6 +82,71 @@ def maime_prompt(line, task_name: str = None):
7382
return maime_prompt
7483

7584

85+
def _maime_prompt_fn_en(lang: str):
86+
template = MATH_PROMPT_TEMPLATES[lang]
87+
88+
def maime_prompt_en(line, task_name: str = None):
89+
return Doc(
90+
task_name=task_name,
91+
query=template.format(prompt=line["problem"]),
92+
choices=[line["answer"]],
93+
gold_index=0,
94+
)
95+
96+
return maime_prompt_en
97+
98+
99+
# English tasks — AIME 2025
100+
maime25_en = LightevalTaskConfig(
101+
name="maime25:en",
102+
prompt_function=_maime_prompt_fn_en("en"),
103+
sample_fields=record_to_sample_en,
104+
solver=[prompt_template(MATH_PROMPT_TEMPLATES["en"]), generate(cache=True)],
105+
scorer=math_scorer(),
106+
hf_repo="yentinglin/aime_2025",
107+
hf_subset="default",
108+
hf_avail_splits=["train"],
109+
evaluation_splits=["train"],
110+
few_shots_split=None,
111+
few_shots_select=None,
112+
generation_size=None,
113+
metrics=[
114+
Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}),
115+
Metrics.avg_at_n_math(sample_params={"n": 1}),
116+
],
117+
version=1,
118+
)
119+
120+
maime25_en_avg = LightevalTaskConfig(
121+
name="maime25_avg:en",
122+
prompt_function=_maime_prompt_fn_en("en"),
123+
sample_fields=record_to_sample_en,
124+
hf_repo="yentinglin/aime_2025",
125+
hf_subset="default",
126+
hf_avail_splits=["train"],
127+
evaluation_splits=["train"],
128+
few_shots_split=None,
129+
few_shots_select=None,
130+
generation_size=None,
131+
metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})],
132+
version=1,
133+
)
134+
135+
maime25_en_gpassk = LightevalTaskConfig(
136+
name="maime25_gpassk:en",
137+
prompt_function=_maime_prompt_fn_en("en"),
138+
sample_fields=record_to_sample_en,
139+
hf_repo="yentinglin/aime_2025",
140+
hf_subset="default",
141+
hf_avail_splits=["train"],
142+
evaluation_splits=["train"],
143+
few_shots_split=None,
144+
few_shots_select=None,
145+
generation_size=None,
146+
metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})],
147+
version=1,
148+
)
149+
76150
# Danish tasks
77151
maime25_da = LightevalTaskConfig(
78152
name="maime25:da",
@@ -175,11 +249,176 @@ def maime_prompt(line, task_name: str = None):
175249
version=1,
176250
)
177251

252+
# English tasks — AIME 2026
253+
maime26_en = LightevalTaskConfig(
254+
name="maime26:en",
255+
prompt_function=_maime_prompt_fn_en("en"),
256+
sample_fields=record_to_sample_en,
257+
solver=[prompt_template(MATH_PROMPT_TEMPLATES["en"]), generate(cache=True)],
258+
scorer=math_scorer(),
259+
hf_repo="math-ai/aime26",
260+
hf_subset="default",
261+
hf_avail_splits=["test"],
262+
evaluation_splits=["test"],
263+
few_shots_split=None,
264+
few_shots_select=None,
265+
generation_size=None,
266+
metrics=[
267+
Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}),
268+
Metrics.avg_at_n_math(sample_params={"n": 1}),
269+
],
270+
version=1,
271+
)
272+
273+
maime26_en_avg = LightevalTaskConfig(
274+
name="maime26_avg:en",
275+
prompt_function=_maime_prompt_fn_en("en"),
276+
sample_fields=record_to_sample_en,
277+
hf_repo="math-ai/aime26",
278+
hf_subset="default",
279+
hf_avail_splits=["test"],
280+
evaluation_splits=["test"],
281+
few_shots_split=None,
282+
few_shots_select=None,
283+
generation_size=None,
284+
metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})],
285+
version=1,
286+
)
287+
288+
maime26_en_gpassk = LightevalTaskConfig(
289+
name="maime26_gpassk:en",
290+
prompt_function=_maime_prompt_fn_en("en"),
291+
sample_fields=record_to_sample_en,
292+
hf_repo="math-ai/aime26",
293+
hf_subset="default",
294+
hf_avail_splits=["test"],
295+
evaluation_splits=["test"],
296+
few_shots_split=None,
297+
few_shots_select=None,
298+
generation_size=None,
299+
metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})],
300+
version=1,
301+
)
302+
303+
# Danish tasks — AIME 2026
304+
maime26_da = LightevalTaskConfig(
305+
name="maime26:da",
306+
prompt_function=_maime_prompt_fn("da"),
307+
sample_fields=record_to_sample,
308+
solver=[prompt_template(MATH_PROMPT_TEMPLATES["da"]), generate(cache=True)],
309+
scorer=math_scorer(),
310+
hf_repo="LumiOpen/mAIME2026",
311+
hf_subset="da_combined",
312+
hf_avail_splits=["test"],
313+
evaluation_splits=["test"],
314+
few_shots_split=None,
315+
few_shots_select=None,
316+
generation_size=None,
317+
metrics=[
318+
Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}),
319+
Metrics.avg_at_n_math(sample_params={"n": 1}),
320+
],
321+
version=1,
322+
)
323+
324+
maime26_da_avg = LightevalTaskConfig(
325+
name="maime26_avg:da",
326+
prompt_function=_maime_prompt_fn("da"),
327+
sample_fields=record_to_sample,
328+
hf_repo="LumiOpen/mAIME2026",
329+
hf_subset="da_combined",
330+
hf_avail_splits=["test"],
331+
evaluation_splits=["test"],
332+
few_shots_split=None,
333+
few_shots_select=None,
334+
generation_size=None,
335+
metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})],
336+
version=1,
337+
)
338+
339+
maime26_da_gpassk = LightevalTaskConfig(
340+
name="maime26_gpassk:da",
341+
prompt_function=_maime_prompt_fn("da"),
342+
sample_fields=record_to_sample,
343+
hf_repo="LumiOpen/mAIME2026",
344+
hf_subset="da_combined",
345+
hf_avail_splits=["test"],
346+
evaluation_splits=["test"],
347+
few_shots_split=None,
348+
few_shots_select=None,
349+
generation_size=None,
350+
metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})],
351+
version=1,
352+
)
353+
354+
# Finnish tasks — AIME 2026
355+
maime26_fi = LightevalTaskConfig(
356+
name="maime26:fi",
357+
prompt_function=_maime_prompt_fn("fi"),
358+
sample_fields=record_to_sample,
359+
solver=[prompt_template(MATH_PROMPT_TEMPLATES["fi"]), generate(cache=True)],
360+
scorer=math_scorer(),
361+
hf_repo="LumiOpen/mAIME2026",
362+
hf_subset="fi_combined",
363+
hf_avail_splits=["test"],
364+
evaluation_splits=["test"],
365+
few_shots_split=None,
366+
few_shots_select=None,
367+
generation_size=None,
368+
metrics=[
369+
Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}),
370+
Metrics.avg_at_n_math(sample_params={"n": 1}),
371+
],
372+
version=1,
373+
)
374+
375+
maime26_fi_avg = LightevalTaskConfig(
376+
name="maime26_avg:fi",
377+
prompt_function=_maime_prompt_fn("fi"),
378+
sample_fields=record_to_sample,
379+
hf_repo="LumiOpen/mAIME2026",
380+
hf_subset="fi_combined",
381+
hf_avail_splits=["test"],
382+
evaluation_splits=["test"],
383+
few_shots_split=None,
384+
few_shots_select=None,
385+
generation_size=None,
386+
metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})],
387+
version=1,
388+
)
389+
390+
maime26_fi_gpassk = LightevalTaskConfig(
391+
name="maime26_gpassk:fi",
392+
prompt_function=_maime_prompt_fn("fi"),
393+
sample_fields=record_to_sample,
394+
hf_repo="LumiOpen/mAIME2026",
395+
hf_subset="fi_combined",
396+
hf_avail_splits=["test"],
397+
evaluation_splits=["test"],
398+
few_shots_split=None,
399+
few_shots_select=None,
400+
generation_size=None,
401+
metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})],
402+
version=1,
403+
)
404+
178405
TASKS_TABLE = [
406+
maime25_en,
407+
maime25_en_avg,
408+
maime25_en_gpassk,
179409
maime25_da,
180410
maime25_da_avg,
181411
maime25_da_gpassk,
182412
maime25_fi,
183413
maime25_fi_avg,
184414
maime25_fi_gpassk,
415+
maime26_en,
416+
maime26_en_avg,
417+
maime26_en_gpassk,
418+
maime26_da,
419+
maime26_da_avg,
420+
maime26_da_gpassk,
421+
maime26_fi,
422+
maime26_fi_avg,
423+
maime26_fi_gpassk,
185424
]

0 commit comments

Comments
 (0)