File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change 1+ {
2+ "schema_version" : 1 ,
3+ "purpose" : " Reference benchmark dimensions for the real C++ and Python Quadtrix benchmark suites." ,
4+ "common" : {
5+ "runs" : 10 ,
6+ "warmup" : 3 ,
7+ "quick_runs" : 2 ,
8+ "quick_warmup" : 1 ,
9+ "generate_tokens" : 32 ,
10+ "quick_generate_tokens" : 4 ,
11+ "train_steps" : 5 ,
12+ "quick_train_steps" : 1 ,
13+ "max_data_chars" : 1000000 ,
14+ "quick_max_data_chars" : 50000
15+ },
16+ "suites" : {
17+ "data" : [
18+ " tokenizer_or_char_encode" ,
19+ " batch_sample_to_device"
20+ ],
21+ "primitive" : [
22+ " matmul_3d" ,
23+ " attention_scores_or_softmax3d" ,
24+ " layer_norm"
25+ ],
26+ "forward" : [
27+ " batch1_seq8" ,
28+ " batch1_full_context" ,
29+ " configured_batch_full_context"
30+ ],
31+ "training" : [
32+ " adamw_step_forward_backward_update"
33+ ],
34+ "generation" : [
35+ " empty_prompt" ,
36+ " short_prompt" ,
37+ " long_prompt"
38+ ]
39+ },
40+ "metrics" : [
41+ " avg_ms" ,
42+ " median_ms" ,
43+ " min_ms" ,
44+ " max_ms" ,
45+ " p90_ms" ,
46+ " p95_ms" ,
47+ " std_ms" ,
48+ " tokens_per_sec" ,
49+ " loss" ,
50+ " parameter_mb_fp32" ,
51+ " memory_mb"
52+ ],
53+ "outputs" : {
54+ "cpp_json" : " benchmark/results/cpp_benchmark.json" ,
55+ "cpp_csv" : " benchmark/results/cpp_benchmark.csv" ,
56+ "python_json" : " benchmark/results/python_benchmark.json" ,
57+ "python_csv" : " benchmark/results/python_benchmark.csv"
58+ }
59+ }
You can’t perform that action at this time.
0 commit comments