Skip to content

Commit e3b12cd

Browse files
alexey-milovidovalexey-milovidov
authored
Merge pull request #845 from ClickHouse/fix-cold-run-cheating-starrocks-doris
Fix starrocks and doris cheating at cold runs.
2 parents 584f78a + efe0a7b commit e3b12cd

14 files changed

Lines changed: 622 additions & 431 deletions

‎doris/benchmark.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ sudo systemctl stop unattended-upgrades
4949
# Start Backend
5050
sudo sysctl -w vm.max_map_count=2000000
5151
ulimit -n 65535
52+
# Disable internal caches so that the cold run (1st of 3 tries) is actually cold.
53+
# Without this, the BE process keeps decoded data in its own in-memory page cache
54+
# (`storage_page_cache`, default ~20% of RAM) and segment cache, which `drop_caches`
55+
# does not clear, so first-run timings reflect a warm cache and underreport
56+
# cold-run latency.
57+
printf "\ndisable_storage_page_cache = true\n" >> "$DORIS_HOME"/be/conf/be.conf
58+
printf "\nsegment_cache_capacity = 0\n" >> "$DORIS_HOME"/be/conf/be.conf
5259
"$DORIS_HOME"/be/bin/start_be.sh --daemon
5360

5461
# Wait for Frontend ready

‎doris/results/c6a.2xlarge.json‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,58 @@
11
{
22
"system": "Apache Doris",
3-
"date": "2026-03-31",
3+
"date": "2026-05-03",
44
"machine": "c6a.2xlarge",
55
"cluster_size": 1,
66
"proprietary": "no",
77
"hardware": "cpu",
88
"tuned": "no",
9-
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative", "lukewarm-cold-run"],
10-
"load_time": 665,
11-
"data_size": 16360275237,
9+
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","lukewarm-cold-run"],
10+
"load_time": 420,
11+
"data_size": 13766121272,
1212
"result": [
13-
[0.00,0.00,0.00],
14-
[0.64,0.05,0.05],
15-
[1.14,0.08,0.08],
16-
[2.09,0.11,0.11],
17-
[0.58,0.59,0.51],
18-
[1.96,1.45,1.43],
19-
[0.17,0.01,0.00],
20-
[0.08,0.05,0.05],
21-
[1.16,0.74,0.77],
22-
[2.45,2.54,2.49],
23-
[1.00,0.14,0.18],
24-
[0.77,0.17,0.16],
25-
[1.59,1.07,0.96],
26-
[1.73,1.70,1.71],
27-
[2.79,2.86,2.84],
28-
[0.91,0.93,0.98],
29-
[2.71,2.68,2.77],
30-
[0.48,0.40,0.38],
31-
[6.02,5.14,5.31],
32-
[0.02,0.01,0.01],
33-
[8.56,1.95,1.90],
34-
[8.69,2.13,2.36],
35-
[19.35,4.75,4.40],
36-
[2.65,0.42,0.25],
37-
[0.16,0.13,0.10],
38-
[0.89,0.50,0.47],
39-
[0.10,0.06,0.06],
40-
[10.44,3.03,2.94],
41-
[19.72,18.93,18.76],
42-
[0.22,0.08,0.07],
43-
[6.17,0.87,0.60],
44-
[3.59,1.02,0.91],
45-
[6.54,6.84,6.56],
46-
[9.55,8.43,8.52],
47-
[9.38,8.55,8.49],
48-
[1.27,1.07,1.04],
49-
[1.99,0.16,0.16],
50-
[0.51,0.09,0.09],
51-
[0.39,0.08,0.09],
52-
[0.89,0.27,0.28],
53-
[0.21,0.07,0.07],
54-
[0.38,0.07,0.07],
55-
[0.10,0.07,0.08]
13+
[0.01, 0, 0],
14+
[0.71, 0.09, 0.09],
15+
[1.68, 0.14, 0.16],
16+
[1.99, 0.31, 0.33],
17+
[1.48, 0.64, 0.66],
18+
[1.81, 1.5, 1.46],
19+
[0.04, 0.01, 0],
20+
[0.15, 0.08, 0.06],
21+
[3.39, 0.9, 0.92],
22+
[5.47, 2.81, 2.71],
23+
[2.7, 0.43, 0.4],
24+
[2.81, 0.44, 0.45],
25+
[2.29, 1.07, 1.09],
26+
[3.45, 2.04, 2.01],
27+
[2.93, 2.75, 2.81],
28+
[1.56, 1.06, 1.02],
29+
[3.96, 2.98, 3],
30+
[4.06, 0.77, 0.76],
31+
[6.75, 5.02, 5.07],
32+
[0.05, 0.02, 0.02],
33+
[8.44, 2.08, 2.59],
34+
[10.23, 2.33, 2.35],
35+
[21.94, 4.83, 4.7],
36+
[4.26, 0.63, 0.57],
37+
[0.2, 0.2, 0.12],
38+
[1.9, 0.66, 0.6],
39+
[0.16, 0.13, 0.1],
40+
[9.22, 2.64, 2.62],
41+
[18.39, 18.72, 18.62],
42+
[0.23, 0.12, 0.13],
43+
[6.33, 0.96, 0.96],
44+
[7.94, 1.38, 1.34],
45+
[6.75, 6.63, 6.28],
46+
[9.16, 7.47, 7.7],
47+
[9.04, 7.75, 7.63],
48+
[1.29, 1.15, 1.1],
49+
[2.25, 0.17, 0.17],
50+
[1.47, 0.14, 0.16],
51+
[1.26, 0.13, 0.16],
52+
[1.77, 0.33, 0.34],
53+
[1, 0.1, 0.11],
54+
[1.53, 0.12, 0.11],
55+
[0.73, 0.1, 0.1]
5656
]
5757
}
5858

‎doris/results/c6a.4xlarge.json‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,58 @@
11
{
22
"system": "Apache Doris",
3-
"date": "2026-03-31",
3+
"date": "2026-05-03",
44
"machine": "c6a.4xlarge",
55
"cluster_size": 1,
66
"proprietary": "no",
77
"hardware": "cpu",
88
"tuned": "no",
9-
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative", "lukewarm-cold-run"],
10-
"load_time": 475,
11-
"data_size": 17105042742,
9+
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","lukewarm-cold-run"],
10+
"load_time": 147,
11+
"data_size": 13787416162,
1212
"result": [
13-
[0.00,0.00,0.01],
14-
[0.12,0.04,0.04],
15-
[0.92,0.06,0.06],
16-
[1.80,0.10,0.11],
17-
[0.36,0.36,0.33],
18-
[1.52,0.88,0.89],
19-
[0.03,0.00,0.00],
20-
[0.05,0.03,0.03],
21-
[0.53,0.49,0.48],
22-
[1.70,1.63,1.64],
23-
[0.36,0.10,0.09],
24-
[0.71,0.10,0.12],
25-
[0.60,0.57,0.60],
26-
[1.14,1.09,1.11],
27-
[0.85,0.78,0.80],
28-
[0.55,0.53,0.55],
29-
[1.61,1.61,1.60],
30-
[0.20,0.21,0.19],
31-
[2.79,2.72,2.65],
32-
[0.02,0.01,0.01],
33-
[8.21,1.00,1.01],
34-
[7.24,0.85,0.84],
35-
[15.12,1.87,1.84],
36-
[1.93,0.21,0.08],
37-
[0.12,0.06,0.05],
38-
[0.16,0.15,0.18],
39-
[0.11,0.05,0.07],
40-
[7.31,1.96,2.10],
41-
[10.84,10.58,10.41],
42-
[0.17,0.09,0.06],
43-
[2.92,0.49,0.40],
44-
[2.96,0.71,0.60],
45-
[4.06,3.97,3.74],
46-
[8.73,5.42,5.54],
47-
[8.53,5.62,5.63],
48-
[0.66,0.58,0.58],
49-
[1.23,0.08,0.08],
50-
[0.98,0.05,0.05],
51-
[0.62,0.04,0.04],
52-
[1.14,0.18,0.17],
53-
[0.53,0.05,0.05],
54-
[0.94,0.05,0.03],
55-
[0.28,0.04,0.04]
13+
[0.01, 0.01, 0],
14+
[0.12, 0.05, 0.05],
15+
[1.13, 0.09, 0.09],
16+
[1.71, 0.18, 0.17],
17+
[1.34, 0.38, 0.34],
18+
[1.64, 0.93, 0.95],
19+
[0.03, 0.01, 0],
20+
[0.08, 0.05, 0.04],
21+
[2.38, 0.51, 0.51],
22+
[4.06, 1.73, 1.75],
23+
[1.94, 0.22, 0.22],
24+
[2.38, 0.24, 0.25],
25+
[2.09, 0.68, 0.67],
26+
[3.31, 1.25, 1.25],
27+
[2.03, 0.88, 0.89],
28+
[1.21, 0.57, 0.55],
29+
[3.15, 1.74, 1.66],
30+
[3.07, 0.37, 0.38],
31+
[5.61, 2.87, 2.89],
32+
[0.04, 0.01, 0.02],
33+
[8.19, 1.1, 1.08],
34+
[9.26, 1.07, 1.07],
35+
[19.18, 2.08, 2.06],
36+
[3.35, 0.76, 0.3],
37+
[0.55, 0.13, 0.09],
38+
[1.95, 0.26, 0.26],
39+
[0.47, 0.09, 0.09],
40+
[8.65, 1.93, 1.92],
41+
[9.45, 10.22, 10.57],
42+
[0.14, 0.09, 0.09],
43+
[6.2, 0.6, 0.59],
44+
[7.56, 0.83, 0.84],
45+
[5.3, 4.15, 4.04],
46+
[8.67, 5.28, 5.26],
47+
[8.72, 5.35, 5.35],
48+
[0.65, 0.64, 0.59],
49+
[1.56, 0.1, 0.09],
50+
[1.62, 0.08, 0.1],
51+
[1.38, 0.08, 0.07],
52+
[2.02, 0.25, 0.22],
53+
[1.08, 0.06, 0.06],
54+
[1.52, 0.06, 0.06],
55+
[0.7, 0.05, 0.05]
5656
]
5757
}
5858

‎doris/results/c6a.metal.json‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,58 @@
11
{
22
"system": "Apache Doris",
3-
"date": "2026-03-31",
3+
"date": "2026-05-03",
44
"machine": "c6a.metal",
55
"cluster_size": 1,
66
"proprietary": "no",
77
"hardware": "cpu",
88
"tuned": "no",
9-
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative", "lukewarm-cold-run"],
10-
"load_time": 408,
11-
"data_size": 17361426786,
9+
"tags": ["C++","column-oriented","MySQL compatible","ClickHouse derivative","lukewarm-cold-run"],
10+
"load_time": 97,
11+
"data_size": 13777301248,
1212
"result": [
13-
[0.01,0.00,0.01],
14-
[0.10,0.03,0.03],
15-
[0.30,0.05,0.05],
16-
[1.45,0.05,0.05],
17-
[0.10,0.07,0.08],
18-
[1.39,0.16,0.16],
19-
[0.03,0.01,0.00],
20-
[0.05,0.04,0.04],
21-
[0.40,0.11,0.12],
22-
[0.61,0.60,0.65],
23-
[0.11,0.05,0.05],
24-
[0.51,0.05,0.06],
25-
[0.15,0.14,0.14],
26-
[0.25,0.23,0.24],
27-
[0.17,0.16,0.15],
28-
[0.11,0.12,0.11],
29-
[0.31,0.31,0.31],
30-
[0.07,0.07,0.07],
31-
[0.56,0.49,0.48],
32-
[0.02,0.01,0.01],
33-
[8.06,0.12,0.12],
34-
[0.07,0.07,0.07],
35-
[7.18,0.10,0.10],
36-
[0.51,0.08,0.07],
37-
[0.06,0.06,0.06],
38-
[0.07,0.07,0.07],
39-
[0.06,0.06,0.06],
40-
[0.37,0.33,0.34],
41-
[6.66,0.96,0.94],
42-
[0.09,0.08,0.07],
43-
[0.77,0.09,0.09],
44-
[2.65,0.15,0.13],
45-
[0.75,0.70,0.74],
46-
[1.02,1.00,1.00],
47-
[1.03,1.02,1.06],
48-
[0.15,0.14,0.14],
49-
[0.09,0.05,0.04],
50-
[0.04,0.03,0.03],
51-
[0.10,0.04,0.04],
52-
[0.29,0.25,0.27],
53-
[0.09,0.04,0.05],
54-
[0.08,0.04,0.04],
55-
[0.04,0.04,0.04]
13+
[0, 0.01, 0.01],
14+
[0.06, 0.03, 0.03],
15+
[0.63, 0.05, 0.05],
16+
[1.41, 0.06, 0.07],
17+
[1.25, 0.09, 0.09],
18+
[1.64, 0.18, 0.18],
19+
[0.03, 0.01, 0.01],
20+
[0.05, 0.04, 0.04],
21+
[2.47, 0.15, 0.14],
22+
[3.78, 0.68, 0.68],
23+
[1.38, 0.08, 0.09],
24+
[2.39, 0.09, 0.1],
25+
[1.77, 0.16, 0.17],
26+
[3, 0.26, 0.26],
27+
[1.67, 0.19, 0.19],
28+
[0.99, 0.13, 0.14],
29+
[3.19, 0.32, 0.32],
30+
[2.89, 0.09, 0.1],
31+
[5.01, 0.49, 0.47],
32+
[0.03, 0.02, 0.02],
33+
[8.05, 0.35, 0.34],
34+
[9.17, 0.35, 0.35],
35+
[17.89, 0.7, 0.69],
36+
[5.78, 0.94, 0.24],
37+
[2.17, 0.75, 0.1],
38+
[1.55, 0.09, 0.1],
39+
[2.36, 0.78, 0.11],
40+
[8.38, 0.48, 0.49],
41+
[6.71, 1.04, 1.03],
42+
[0.11, 0.06, 0.06],
43+
[4.46, 0.14, 0.15],
44+
[6.51, 0.18, 0.17],
45+
[4.43, 0.67, 0.75],
46+
[8.31, 1.06, 1.1],
47+
[8.21, 1.11, 1.13],
48+
[0.37, 0.15, 0.15],
49+
[0.8, 0.04, 0.04],
50+
[0.94, 0.03, 0.04],
51+
[1.02, 0.04, 0.04],
52+
[1.25, 0.09, 0.07],
53+
[0.87, 0.04, 0.04],
54+
[0.98, 0.04, 0.05],
55+
[0.57, 0.03, 0.04]
5656
]
5757
}
5858

0 commit comments

Comments
 (0)