Skip to content

Commit 6f179f5

Browse files
zml1206zml1206
authored
[MINOR] Remove stale complex type scan fallback config from tests (#12476)
1 parent 630d5ac commit 6f179f5

14 files changed

Lines changed: 915 additions & 982 deletions

File tree

‎backends-velox/src/test/scala/org/apache/gluten/execution/VeloxOrcDataTypeValidationSuite.scala‎

Lines changed: 83 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -331,108 +331,102 @@ class VeloxOrcDataTypeValidationSuite extends VeloxWholeStageTransformerSuite {
331331
}
332332

333333
test("Array type") {
334-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
335-
// Validation: BatchScan.
336-
runQueryAndCompare("select array from type1") {
337-
checkGlutenPlan[BatchScanExecTransformer]
338-
}
334+
// Validation: BatchScan.
335+
runQueryAndCompare("select array from type1") {
336+
checkGlutenPlan[BatchScanExecTransformer]
337+
}
339338

340-
// Validation: BatchScan Project Aggregate Expand Sort Limit
341-
runQueryAndCompare(
342-
"select int, array from type1 " +
343-
" group by grouping sets(int, array) sort by array, int limit 1") {
344-
df =>
345-
{
346-
val executedPlan = getExecutedPlan(df)
347-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
348-
}
349-
}
339+
// Validation: BatchScan Project Aggregate Expand Sort Limit
340+
runQueryAndCompare(
341+
"select int, array from type1 " +
342+
" group by grouping sets(int, array) sort by array, int limit 1") {
343+
df =>
344+
{
345+
val executedPlan = getExecutedPlan(df)
346+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
347+
}
348+
}
350349

351-
// Validation: BroadHashJoin, Filter, Project
352-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
353-
runQueryAndCompare(
354-
"select type1.array from type1," +
355-
" type2 where type1.array = type2.array") { _ => }
350+
// Validation: BroadHashJoin, Filter, Project
351+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
352+
runQueryAndCompare(
353+
"select type1.array from type1," +
354+
" type2 where type1.array = type2.array") { _ => }
356355

357-
// Validation: ShuffledHashJoin, Filter, Project
358-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
359-
runQueryAndCompare(
360-
"select type1.array from type1," +
361-
" type2 where type1.array = type2.array") { _ => }
362-
}
356+
// Validation: ShuffledHashJoin, Filter, Project
357+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
358+
runQueryAndCompare(
359+
"select type1.array from type1," +
360+
" type2 where type1.array = type2.array") { _ => }
363361
}
364362

365363
test("Map type") {
366-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
367-
// Validation: BatchScan Project Limit
368-
runQueryAndCompare("select map from type1 limit 1") {
369-
df =>
370-
{
371-
val executedPlan = getExecutedPlan(df)
372-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
373-
}
374-
}
375-
// Validation: BatchScan Project Aggregate Sort Limit
376-
// TODO validate Expand operator support map type ?
377-
runQueryAndCompare(
378-
"select map['key'] from type1 group by map['key']" +
379-
" sort by map['key'] limit 1") {
380-
df =>
381-
{
382-
val executedPlan = getExecutedPlan(df)
383-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
384-
}
385-
}
364+
// Validation: BatchScan Project Limit
365+
runQueryAndCompare("select map from type1 limit 1") {
366+
df =>
367+
{
368+
val executedPlan = getExecutedPlan(df)
369+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
370+
}
371+
}
372+
// Validation: BatchScan Project Aggregate Sort Limit
373+
// TODO validate Expand operator support map type ?
374+
runQueryAndCompare(
375+
"select map['key'] from type1 group by map['key']" +
376+
" sort by map['key'] limit 1") {
377+
df =>
378+
{
379+
val executedPlan = getExecutedPlan(df)
380+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
381+
}
382+
}
386383

387-
// Validation: BroadHashJoin, Filter, Project
388-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
389-
runQueryAndCompare(
390-
"select type1.map['key'] from type1," +
391-
" type2 where type1.map['key'] = type2.map['key']") { _ => }
384+
// Validation: BroadHashJoin, Filter, Project
385+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
386+
runQueryAndCompare(
387+
"select type1.map['key'] from type1," +
388+
" type2 where type1.map['key'] = type2.map['key']") { _ => }
392389

393-
// Validation: ShuffledHashJoin, Filter, Project
394-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
395-
runQueryAndCompare(
396-
"select type1.map['key'] from type1," +
397-
" type2 where type1.map['key'] = type2.map['key']") { _ => }
398-
}
390+
// Validation: ShuffledHashJoin, Filter, Project
391+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
392+
runQueryAndCompare(
393+
"select type1.map['key'] from type1," +
394+
" type2 where type1.map['key'] = type2.map['key']") { _ => }
399395
}
400396

401397
test("Struct type") {
402-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
403-
// Validation: BatchScan Project Limit
404-
runQueryAndCompare("select struct from type1") {
405-
df =>
406-
{
407-
val executedPlan = getExecutedPlan(df)
408-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
409-
}
410-
}
411-
// Validation: BatchScan Project Aggregate Sort Limit
412-
// TODO validate Expand operator support Struct type ?
413-
runQueryAndCompare(
414-
"select int, struct.struct_1 from type1 " +
415-
"sort by struct.struct_1 limit 1") {
416-
df =>
417-
{
418-
val executedPlan = getExecutedPlan(df)
419-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
420-
assert(executedPlan.exists(plan => plan.isInstanceOf[ProjectExecTransformer]))
421-
}
422-
}
398+
// Validation: BatchScan Project Limit
399+
runQueryAndCompare("select struct from type1") {
400+
df =>
401+
{
402+
val executedPlan = getExecutedPlan(df)
403+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
404+
}
405+
}
406+
// Validation: BatchScan Project Aggregate Sort Limit
407+
// TODO validate Expand operator support Struct type ?
408+
runQueryAndCompare(
409+
"select int, struct.struct_1 from type1 " +
410+
"sort by struct.struct_1 limit 1") {
411+
df =>
412+
{
413+
val executedPlan = getExecutedPlan(df)
414+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
415+
assert(executedPlan.exists(plan => plan.isInstanceOf[ProjectExecTransformer]))
416+
}
417+
}
423418

424-
// Validation: BroadHashJoin, Filter, Project
425-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
426-
runQueryAndCompare(
427-
"select type1.struct.struct_1 from type1," +
428-
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
419+
// Validation: BroadHashJoin, Filter, Project
420+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
421+
runQueryAndCompare(
422+
"select type1.struct.struct_1 from type1," +
423+
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
429424

430-
// Validation: ShuffledHashJoin, Filter, Project
431-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
432-
runQueryAndCompare(
433-
"select type1.struct.struct_1 from type1," +
434-
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
435-
}
425+
// Validation: ShuffledHashJoin, Filter, Project
426+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
427+
runQueryAndCompare(
428+
"select type1.struct.struct_1 from type1," +
429+
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
436430
}
437431

438432
test("Decimal type") {

‎backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala‎

Lines changed: 83 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -332,108 +332,102 @@ class VeloxParquetDataTypeValidationSuite extends VeloxWholeStageTransformerSuit
332332
}
333333

334334
test("Array type") {
335-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
336-
// Validation: BatchScan.
337-
runQueryAndCompare("select array from type1") {
338-
checkGlutenPlan[BatchScanExecTransformer]
339-
}
335+
// Validation: BatchScan.
336+
runQueryAndCompare("select array from type1") {
337+
checkGlutenPlan[BatchScanExecTransformer]
338+
}
340339

341-
// Validation: BatchScan Project Aggregate Expand Sort Limit
342-
runQueryAndCompare(
343-
"select int, array from type1 " +
344-
" group by grouping sets(int, array) sort by array, int limit 1") {
345-
df =>
346-
{
347-
val executedPlan = getExecutedPlan(df)
348-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
349-
}
350-
}
340+
// Validation: BatchScan Project Aggregate Expand Sort Limit
341+
runQueryAndCompare(
342+
"select int, array from type1 " +
343+
" group by grouping sets(int, array) sort by array, int limit 1") {
344+
df =>
345+
{
346+
val executedPlan = getExecutedPlan(df)
347+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
348+
}
349+
}
351350

352-
// Validation: BroadHashJoin, Filter, Project
353-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
354-
runQueryAndCompare(
355-
"select type1.array from type1," +
356-
" type2 where type1.array = type2.array") { _ => }
351+
// Validation: BroadHashJoin, Filter, Project
352+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
353+
runQueryAndCompare(
354+
"select type1.array from type1," +
355+
" type2 where type1.array = type2.array") { _ => }
357356

358-
// Validation: ShuffledHashJoin, Filter, Project
359-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
360-
runQueryAndCompare(
361-
"select type1.array from type1," +
362-
" type2 where type1.array = type2.array") { _ => }
363-
}
357+
// Validation: ShuffledHashJoin, Filter, Project
358+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
359+
runQueryAndCompare(
360+
"select type1.array from type1," +
361+
" type2 where type1.array = type2.array") { _ => }
364362
}
365363

366364
test("Map type") {
367-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
368-
// Validation: BatchScan Project Limit
369-
runQueryAndCompare("select map from type1 limit 1") {
370-
df =>
371-
{
372-
val executedPlan = getExecutedPlan(df)
373-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
374-
}
375-
}
376-
// Validation: BatchScan Project Aggregate Sort Limit
377-
// TODO validate Expand operator support map type ?
378-
runQueryAndCompare(
379-
"select map['key'] from type1 group by map['key']" +
380-
" sort by map['key'] limit 1") {
381-
df =>
382-
{
383-
val executedPlan = getExecutedPlan(df)
384-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
385-
}
386-
}
365+
// Validation: BatchScan Project Limit
366+
runQueryAndCompare("select map from type1 limit 1") {
367+
df =>
368+
{
369+
val executedPlan = getExecutedPlan(df)
370+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
371+
}
372+
}
373+
// Validation: BatchScan Project Aggregate Sort Limit
374+
// TODO validate Expand operator support map type ?
375+
runQueryAndCompare(
376+
"select map['key'] from type1 group by map['key']" +
377+
" sort by map['key'] limit 1") {
378+
df =>
379+
{
380+
val executedPlan = getExecutedPlan(df)
381+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
382+
}
383+
}
387384

388-
// Validation: BroadHashJoin, Filter, Project
389-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
390-
runQueryAndCompare(
391-
"select type1.map['key'] from type1," +
392-
" type2 where type1.map['key'] = type2.map['key']") { _ => }
385+
// Validation: BroadHashJoin, Filter, Project
386+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
387+
runQueryAndCompare(
388+
"select type1.map['key'] from type1," +
389+
" type2 where type1.map['key'] = type2.map['key']") { _ => }
393390

394-
// Validation: ShuffledHashJoin, Filter, Project
395-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
396-
runQueryAndCompare(
397-
"select type1.map['key'] from type1," +
398-
" type2 where type1.map['key'] = type2.map['key']") { _ => }
399-
}
391+
// Validation: ShuffledHashJoin, Filter, Project
392+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
393+
runQueryAndCompare(
394+
"select type1.map['key'] from type1," +
395+
" type2 where type1.map['key'] = type2.map['key']") { _ => }
400396
}
401397

402398
test("Struct type") {
403-
withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled", "false")) {
404-
// Validation: BatchScan Project Limit
405-
runQueryAndCompare("select struct from type1") {
406-
df =>
407-
{
408-
val executedPlan = getExecutedPlan(df)
409-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
410-
}
411-
}
412-
// Validation: BatchScan Project Aggregate Sort Limit
413-
// TODO validate Expand operator support Struct type ?
414-
runQueryAndCompare(
415-
"select int, struct.struct_1 from type1 " +
416-
"sort by struct.struct_1 limit 1") {
417-
df =>
418-
{
419-
val executedPlan = getExecutedPlan(df)
420-
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
421-
assert(executedPlan.exists(plan => plan.isInstanceOf[ProjectExecTransformer]))
422-
}
423-
}
399+
// Validation: BatchScan Project Limit
400+
runQueryAndCompare("select struct from type1") {
401+
df =>
402+
{
403+
val executedPlan = getExecutedPlan(df)
404+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
405+
}
406+
}
407+
// Validation: BatchScan Project Aggregate Sort Limit
408+
// TODO validate Expand operator support Struct type ?
409+
runQueryAndCompare(
410+
"select int, struct.struct_1 from type1 " +
411+
"sort by struct.struct_1 limit 1") {
412+
df =>
413+
{
414+
val executedPlan = getExecutedPlan(df)
415+
assert(executedPlan.exists(plan => plan.isInstanceOf[BatchScanExecTransformer]))
416+
assert(executedPlan.exists(plan => plan.isInstanceOf[ProjectExecTransformer]))
417+
}
418+
}
424419

425-
// Validation: BroadHashJoin, Filter, Project
426-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
427-
runQueryAndCompare(
428-
"select type1.struct.struct_1 from type1," +
429-
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
420+
// Validation: BroadHashJoin, Filter, Project
421+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
422+
runQueryAndCompare(
423+
"select type1.struct.struct_1 from type1," +
424+
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
430425

431-
// Validation: ShuffledHashJoin, Filter, Project
432-
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
433-
runQueryAndCompare(
434-
"select type1.struct.struct_1 from type1," +
435-
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
436-
}
426+
// Validation: ShuffledHashJoin, Filter, Project
427+
super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
428+
runQueryAndCompare(
429+
"select type1.struct.struct_1 from type1," +
430+
" type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
437431
}
438432

439433
test("Decimal type") {

0 commit comments

Comments
 (0)