Skip to content

Commit 6567fb5

Browse files
esokulluesokullu
authored
Merge pull request #81 from esokullu/codex/tencent-hy3-openrouter-blog
[codex] Add Tencent Hy3 planner benchmark
2 parents 112f7be + b6f7cb6 commit 6567fb5

104 files changed

Lines changed: 5139 additions & 1 deletion

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/001.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "001",
3+
"user": "go to firefox extensions management pagr",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3763,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "about:addons"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17253,
22+
"completion_tokens": 48,
23+
"total_tokens": 17301,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 25,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/002.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "002",
3+
"user": "open gmail",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3482,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://mail.google.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17248,
22+
"completion_tokens": 38,
23+
"total_tokens": 17286,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 14,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/003.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "003",
3+
"user": "take me to youtube",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3246,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://www.youtube.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17249,
22+
"completion_tokens": 37,
23+
"total_tokens": 17286,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 13,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/004.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "004",
3+
"user": "open my github",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3486,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://github.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17248,
22+
"completion_tokens": 39,
23+
"total_tokens": 17287,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 16,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/005.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "005",
3+
"user": "show me my downloads",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3990,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "list_downloads",
13+
"args": {
14+
"limit": 20
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": "I'll pull up your recent downloads.",
20+
"usage": {
21+
"prompt_tokens": 17249,
22+
"completion_tokens": 51,
23+
"total_tokens": 17300,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 22,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/006.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "006",
3+
"user": "i wanna check netflix",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 2926,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://www.netflix.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": "I'll take you to Netflix.",
20+
"usage": {
21+
"prompt_tokens": 17250,
22+
"completion_tokens": 45,
23+
"total_tokens": 17295,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 13,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/007.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "007",
3+
"user": "open linkedin in this tab",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 2018,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://www.linkedin.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17251,
22+
"completion_tokens": 54,
23+
"total_tokens": 17305,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 17152,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 28,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/008.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "008",
3+
"user": "wikipedia please",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3082,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "navigate",
13+
"args": {
14+
"url": "https://www.wikipedia.org"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17248,
22+
"completion_tokens": 38,
23+
"total_tokens": 17286,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 14,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

‎test/llm/results/2026-07-07-openrouter-tencent-hy3-free_chrome_tencent_hy3_free_frozen/009.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"id": "009",
3+
"user": "open chatgpt in a new tab",
4+
"mode": "act",
5+
"tab": {
6+
"url": "about:home",
7+
"title": "New Tab"
8+
},
9+
"latencyMs": 3430,
10+
"error": null,
11+
"firstToolCall": {
12+
"name": "new_tab",
13+
"args": {
14+
"url": "https://chat.openai.com"
15+
}
16+
},
17+
"toolCallSource": "tool_calls",
18+
"finishReason": "tool_calls",
19+
"content": null,
20+
"usage": {
21+
"prompt_tokens": 17253,
22+
"completion_tokens": 43,
23+
"total_tokens": 17296,
24+
"cost": 0,
25+
"is_byok": false,
26+
"prompt_tokens_details": {
27+
"cached_tokens": 0,
28+
"cache_write_tokens": 0,
29+
"audio_tokens": 0,
30+
"video_tokens": 0
31+
},
32+
"cost_details": {
33+
"upstream_inference_cost": 0,
34+
"upstream_inference_prompt_cost": 0,
35+
"upstream_inference_completions_cost": 0
36+
},
37+
"completion_tokens_details": {
38+
"reasoning_tokens": 18,
39+
"image_tokens": 0,
40+
"audio_tokens": 0
41+
}
42+
}
43+
}

0 commit comments

Comments
 (0)