Skip to content

Commit b0f9680

Browse files
authored
Add GSM8K Platinum to Laguna regression (#2841)
1 parent a41b67c commit b0f9680

1 file changed

Lines changed: 13 additions & 0 deletions

File tree

tests/models/test_laguna.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,19 @@ class TestLagunaXS2(ModelTest):
2929
EVAL_BATCH_SIZE = "auto"
3030

3131
EVAL_TASKS_FAST = {
32+
"gsm8k_platinum_cot": {
33+
"chat_template": False,
34+
"evalution_suite_kwargs": {
35+
"max_new_tokens": 256,
36+
"stream": True,
37+
"max_rows": 128,
38+
},
39+
"acc,num": {
40+
"value": 0.3828125,
41+
"floor_pct": 0.50,
42+
"ceil_pct": 1.0,
43+
},
44+
},
3245
"arc_challenge": {
3346
"chat_template": False,
3447
"evalution_suite_kwargs": {

0 commit comments

Comments
 (0)