Skip to content

Commit 9610c7e

Browse files
morningmanclaudexylaaaaakaka11chen
authored
branch-4.0: [fix](test) fix unstable cases (#63500)
external_table_p0/hive/test_hive_compress_type.groovy external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy export_p0/outfile/csv/test_outfile_empty_data.groovy external_table_p0/broker_load/test_broker_load_func.groovy bp #61073 #62360 #62423 --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Chenjunwei <138805230+xylaaaaa@users.noreply.github.com> Co-authored-by: Qi Chen <chenqi@selectdb.com>
1 parent 810a336 commit 9610c7e

5 files changed

Lines changed: 126 additions & 50 deletions

File tree

‎regression-test/suites/export_p0/outfile/csv/test_outfile_empty_data.groovy‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,21 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
5151
// broker
5252
String broker_name = "hdfs"
5353

54+
// check whether the broker named `hdfs` exists. If not, skip broker-related cases.
55+
def brokerExists = {String name ->
56+
def brokers = sql """ SHOW BROKER """
57+
for (def row : brokers) {
58+
if (row[0] == name) {
59+
return true
60+
}
61+
}
62+
return false
63+
}
64+
def has_broker = brokerExists(broker_name)
65+
if (!has_broker) {
66+
logger.info("broker `${broker_name}` does not exist, will skip broker-related cases.")
67+
}
68+
5469
def export_table_name = "outfile_empty_data_test"
5570

5671
def create_table = {table_name, column_define ->
@@ -133,23 +148,25 @@ suite("test_outfile_empty_data", "external,hive,tvf,external_docker") {
133148
create_table(export_table_name, doris_column_define);
134149
// test outfile empty data to hdfs directly
135150
def outfile_to_hdfs_directly_url = outfile_to_HDFS_directly()
136-
// test outfile empty data to hdfs with broker
137-
def outfile_to_hdfs_with_broker_url= outfile_to_HDFS_with_broker()
138151
// test outfile empty data to s3 directly
139152
def outfile_to_s3_directly_url = outfile_to_S3_directly()
140-
qt_select_base1 """ SELECT * FROM ${export_table_name} ORDER BY user_id; """
153+
qt_select_base1 """ SELECT * FROM ${export_table_name} ORDER BY user_id; """
141154

142155
qt_select_tvf1 """ select * from HDFS(
143156
"uri" = "${outfile_to_hdfs_directly_url}0.csv",
144157
"hadoop.username" = "${hdfsUserName}",
145158
"format" = "${format}");
146159
"""
147160

148-
qt_select_tvf2 """ select * from HDFS(
149-
"uri" = "${outfile_to_hdfs_with_broker_url}0.csv",
150-
"hadoop.username" = "${hdfsUserName}",
151-
"format" = "${format}");
152-
"""
161+
if (has_broker) {
162+
// test outfile empty data to hdfs with broker
163+
def outfile_to_hdfs_with_broker_url= outfile_to_HDFS_with_broker()
164+
qt_select_tvf2 """ select * from HDFS(
165+
"uri" = "${outfile_to_hdfs_with_broker_url}0.csv",
166+
"hadoop.username" = "${hdfsUserName}",
167+
"format" = "${format}");
168+
"""
169+
}
153170

154171
qt_select_tvf3 """ SELECT * FROM S3 (
155172
"uri" = "http://${bucket}.${s3_endpoint}${outfile_to_s3_directly_url.substring(5 + bucket.length(), outfile_to_s3_directly_url.length())}0.csv",

‎regression-test/suites/external_table_p0/broker_load/test_broker_load_func.groovy‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@ suite("test_broker_load_func", "p0,external,hive,external_docker,external_docker
2525

2626
String database_name = "test_broker_load_func"
2727
String broker_name = "hdfs"
28+
29+
// check whether the broker named `hdfs` exists. If not, skip this case.
30+
def brokers = sql """ SHOW BROKER """
31+
def has_broker = false
32+
for (def row : brokers) {
33+
if (row[0] == broker_name) {
34+
has_broker = true
35+
break
36+
}
37+
}
38+
if (!has_broker) {
39+
logger.info("broker `${broker_name}` does not exist, skip test_broker_load_func.")
40+
return
41+
}
42+
2843
def uuid = UUID.randomUUID().toString().replaceAll("-", "")
2944
def test_load_label="label_test_broker_load_func_${uuid}"
3045
String table_name="simple"

‎regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,6 @@ suite("test_hive_compress_type", "p0,external,hive,external_docker,external_dock
2323
return;
2424
}
2525

26-
def backends = sql """show backends"""
27-
def backendNum = backends.size()
28-
logger.info("get backendNum: ${backendNum}")
29-
// `parallel_fragment_exec_instance_num` may be displayed as
30-
// `deprecated_parallel_fragment_exec_instance_num` in newer branches.
31-
def parallelExecInstanceRows = sql("show variables like '%parallel_fragment_exec_instance_num%'")
32-
assertTrue(parallelExecInstanceRows.size() > 0)
33-
def parallelExecInstanceNum = (parallelExecInstanceRows[0][1] as String).toInteger()
34-
logger.info("get ${parallelExecInstanceRows[0][0]}: ${parallelExecInstanceNum}")
35-
3626
for (String hivePrefix : ["hive3"]) {
3727
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
3828
String catalog_name = "${hivePrefix}_test_hive_compress_type"
@@ -45,37 +35,6 @@ suite("test_hive_compress_type", "p0,external,hive,external_docker,external_dock
4535
);"""
4636
sql """use `${catalog_name}`.`multi_catalog`"""
4737

48-
// table test_compress_partitioned has 6 partitions with different compressed file: plain, gzip, bzip2, deflate
49-
sql """set file_split_size=0"""
50-
// COUNT pushdown split behavior depends on:
51-
// totalFileNum < parallel_fragment_exec_instance_num * backendNum
52-
// test_compress_partitioned currently has 16 files.
53-
def expectedSplitNum = 16
54-
if (backendNum > 1) {
55-
expectedSplitNum = (16 < parallelExecInstanceNum * backendNum) ? 28 : 16
56-
}
57-
explain {
58-
sql("select count(*) from test_compress_partitioned")
59-
contains "inputSplitNum=${expectedSplitNum}, totalFileSize=734675596, scanRanges=${expectedSplitNum}"
60-
contains "partition=8/8"
61-
}
62-
qt_q21 """select count(*) from test_compress_partitioned where dt="gzip" or dt="mix""""
63-
qt_q22 """select count(*) from test_compress_partitioned"""
64-
order_qt_q23 """select * from test_compress_partitioned where watchid=4611870011201662970"""
65-
66-
sql """set file_split_size=8388608"""
67-
explain {
68-
sql("select count(*) from test_compress_partitioned")
69-
contains "inputSplitNum=16, totalFileSize=734675596, scanRanges=16"
70-
contains "partition=8/8"
71-
}
72-
73-
qt_q31 """select count(*) from test_compress_partitioned where dt="gzip" or dt="mix""""
74-
qt_q32 """select count(*) from test_compress_partitioned"""
75-
order_qt_q33 """select * from test_compress_partitioned where watchid=4611870011201662970"""
76-
sql """set file_split_size=0"""
77-
78-
7938
order_qt_q42 """ select count(*) from parquet_lz4_compression ; """
8039
order_qt_q43 """ select * from parquet_lz4_compression
8140
order by col_int,col_smallint,col_tinyint,col_bigint,col_float,col_double,col_boolean,col_string,col_char,col_varchar,col_date,col_timestamp,col_decimal

‎regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc
405405
}
406406
sql """ set enable_ext_func_pred_pushdown = "false"; """
407407
explain {
408-
sql ("select k6, k8 from test1 where nvl(k6, null) = 1 and k8 = 1;")
408+
sql ("select k6, k8 from test1 where nvl(k6, 1) = k6 and k8 = 1;")
409409

410410
contains "QUERY: SELECT `k6`, `k8` FROM `doris_test`.`test1` WHERE ((`k8` = 1))"
411411
}

‎regression-test/suites/external_table_p2/hive/test_hive_compress_type_large_data.groovy‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_hive_compress_type_large_data", "p2,external") {
19+
String enabled = context.config.otherConfigs.get("enableHiveTest")
20+
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
21+
logger.info("diable Hive test.")
22+
return;
23+
}
24+
25+
def backends = sql """show backends"""
26+
def backendNum = backends.size()
27+
logger.info("get backendNum: ${backendNum}")
28+
29+
// Set parallel_pipeline_task_num explicitly so getParallelExecInstanceNum() returns
30+
// a known value, regardless of fuzzy mode randomization or BE core count.
31+
def parallelExecInstanceNum = 8
32+
sql """set parallel_pipeline_task_num = ${parallelExecInstanceNum}"""
33+
34+
for (String hivePrefix : ["hive3"]) {
35+
String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
36+
String catalog_name = "${hivePrefix}_test_hive_compress_type_large_data"
37+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
38+
39+
sql """drop catalog if exists ${catalog_name}"""
40+
sql """create catalog if not exists ${catalog_name} properties (
41+
"type"="hms",
42+
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
43+
);"""
44+
sql """use `${catalog_name}`.`multi_catalog`"""
45+
46+
// table test_compress_partitioned has 16 files across 8 partitions (734MB total).
47+
// With count pushdown, needSplit depends on totalFileNum vs parallelNum * backendNum.
48+
// When needSplit=false: each file = 1 split = 16 splits.
49+
// When needSplit=true and file_split_size=0: splits by dynamic size = 28 splits.
50+
// When needSplit=true and file_split_size=8MB: splits by 8MB = 82 splits.
51+
def needSplit = (backendNum > 1) && (16 < parallelExecInstanceNum * backendNum)
52+
53+
sql """set file_split_size=0"""
54+
def expectedSplitNum1 = needSplit ? 28 : 16
55+
explain {
56+
sql("select count(*) from test_compress_partitioned")
57+
contains "inputSplitNum=${expectedSplitNum1}, totalFileSize=734675596, scanRanges=${expectedSplitNum1}"
58+
contains "partition=8/8"
59+
}
60+
61+
def countMix1 = sql """select count(*) from test_compress_partitioned where dt="gzip" or dt="mix""""
62+
assertEquals(600005, countMix1[0][0])
63+
def countAll1 = sql """select count(*) from test_compress_partitioned"""
64+
assertEquals(1510010, countAll1[0][0])
65+
def countWatchId1 = sql """select count(*) from test_compress_partitioned where watchid=4611870011201662970"""
66+
assertEquals(15, countWatchId1[0][0])
67+
68+
sql """set file_split_size=8388608"""
69+
def expectedSplitNum2 = needSplit ? 82 : 16
70+
explain {
71+
sql("select count(*) from test_compress_partitioned")
72+
contains "inputSplitNum=${expectedSplitNum2}, totalFileSize=734675596, scanRanges=${expectedSplitNum2}"
73+
contains "partition=8/8"
74+
}
75+
76+
def countMix2 = sql """select count(*) from test_compress_partitioned where dt="gzip" or dt="mix""""
77+
assertEquals(600005, countMix2[0][0])
78+
def countAll2 = sql """select count(*) from test_compress_partitioned"""
79+
assertEquals(1510010, countAll2[0][0])
80+
def countWatchId2 = sql """select count(*) from test_compress_partitioned where watchid=4611870011201662970"""
81+
assertEquals(15, countWatchId2[0][0])
82+
83+
sql """set file_split_size=0"""
84+
}
85+
}

0 commit comments

Comments
 (0)