diff --git a/Cargo.toml b/Cargo.toml index c5d0c4239e..1912d9fd20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,9 +64,9 @@ cfg-if = "1" chrono = "0.4.41" clap = { version = "4.5.48", features = ["derive", "cargo"] } dashmap = "6" -datafusion = "52.1" -datafusion-cli = "52.1" -datafusion-sqllogictest = "52.1" +datafusion = { version = "52" } +datafusion-cli = { version = "52" } +datafusion-sqllogictest = { version = "52" } derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 7bb4e7ed0e..3a853496f8 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -189,13 +189,14 @@ impl DisplayAs for IcebergTableScan { ) -> std::fmt::Result { write!( f, - "IcebergTableScan projection:[{}] predicate:[{}]", + "IcebergTableScan projection:[{}] predicate:[{}] limit:[{}]", self.projection .clone() .map_or(String::new(), |v| v.join(",")), self.predicates .clone() - .map_or(String::from(""), |p| format!("{p}")) + .map_or(String::from(""), |p| format!("{p}")), + self.limit.map_or(String::from(""), |l| format!("{l}")) ) } } diff --git a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt index 54d74f5e70..5166c93d8b 100644 --- a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt @@ -29,7 +29,7 @@ physical_plan 01)FilterExec: data@1 = 0102 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,data] predicate:[data = 0102] +04)------IcebergTableScan projection:[id,data] predicate:[data = 0102] limit:[] # Verify empty result from empty table query I? diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt index 466a45f8c6..d6b97a337e 100644 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt @@ -40,7 +40,7 @@ physical_plan 01)FilterExec: is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] +04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] limit:[] # Query with is_active = true query ITT rowsort @@ -61,7 +61,7 @@ physical_plan 01)FilterExec: NOT is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] +04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] limit:[] # Query with is_active = false query ITT rowsort @@ -81,7 +81,7 @@ physical_plan 01)FilterExec: NOT is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] +04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] limit:[] # Query with is_active != true (includes false and NULL) query ITT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt index a160ca77f4..c171eb626e 100644 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt @@ -38,7 +38,7 @@ physical_plan 01)FilterExec: name@1 LIKE Al% 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] +04)------IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] limit:[] # Test LIKE filtering with case-sensitive match query IT rowsort @@ -58,7 +58,7 @@ physical_plan 01)FilterExec: name@1 NOT LIKE Al% 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] +04)------IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] limit:[] # Test NOT LIKE filtering query IT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt index 3427625291..dafb48bacb 100644 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt @@ -51,7 +51,7 @@ physical_plan 01)FilterExec: ts@1 = 1672921800000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] +04)------IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] limit:[] # Verify timestamp equality filtering works query I? @@ -70,7 +70,7 @@ physical_plan 01)FilterExec: ts@1 > 1673308800000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] +04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] limit:[] # Verify timestamp greater than filtering query I? rowsort @@ -100,7 +100,7 @@ physical_plan 01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] +04)------IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] limit:[] # Test timestamp range predicate filtering query I? rowsort @@ -166,7 +166,7 @@ physical_plan 01)FilterExec: ts@1 > 1672531200000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] +04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] limit:[] query I? SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP)