Skip to content

Commit 01ed329

Browse files
authored
add reference benchmark dimensions for Quadtrix (#39)
## Summary Introduces configuration for real C++ and Python Quadtrix benchmark runs, including warmup, token generation, and training step dimensions.
2 parents 09d841d + 124bdcf commit 01ed329

1 file changed

Lines changed: 59 additions & 0 deletions

File tree

benchmark/benchmark_config.json

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"schema_version": 1,
3+
"purpose": "Reference benchmark dimensions for the real C++ and Python Quadtrix benchmark suites.",
4+
"common": {
5+
"runs": 10,
6+
"warmup": 3,
7+
"quick_runs": 2,
8+
"quick_warmup": 1,
9+
"generate_tokens": 32,
10+
"quick_generate_tokens": 4,
11+
"train_steps": 5,
12+
"quick_train_steps": 1,
13+
"max_data_chars": 1000000,
14+
"quick_max_data_chars": 50000
15+
},
16+
"suites": {
17+
"data": [
18+
"tokenizer_or_char_encode",
19+
"batch_sample_to_device"
20+
],
21+
"primitive": [
22+
"matmul_3d",
23+
"attention_scores_or_softmax3d",
24+
"layer_norm"
25+
],
26+
"forward": [
27+
"batch1_seq8",
28+
"batch1_full_context",
29+
"configured_batch_full_context"
30+
],
31+
"training": [
32+
"adamw_step_forward_backward_update"
33+
],
34+
"generation": [
35+
"empty_prompt",
36+
"short_prompt",
37+
"long_prompt"
38+
]
39+
},
40+
"metrics": [
41+
"avg_ms",
42+
"median_ms",
43+
"min_ms",
44+
"max_ms",
45+
"p90_ms",
46+
"p95_ms",
47+
"std_ms",
48+
"tokens_per_sec",
49+
"loss",
50+
"parameter_mb_fp32",
51+
"memory_mb"
52+
],
53+
"outputs": {
54+
"cpp_json": "benchmark/results/cpp_benchmark.json",
55+
"cpp_csv": "benchmark/results/cpp_benchmark.csv",
56+
"python_json": "benchmark/results/python_benchmark.json",
57+
"python_csv": "benchmark/results/python_benchmark.csv"
58+
}
59+
}

0 commit comments

Comments
 (0)