Skip to content

Display limit in IcebergTableScan fmt_as #2212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
toutane wants to merge 1 commit into from
Closed

Display limit in IcebergTableScan fmt_as #2212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ cfg-if = "1"
chrono = "0.4.41"
clap = { version = "4.5.48", features = ["derive", "cargo"] }
dashmap = "6"
datafusion = "52.1"
datafusion-cli = "52.1"
datafusion-sqllogictest = "52.1"
datafusion = { version = "52" }
datafusion-cli = { version = "52" }
datafusion-sqllogictest = { version = "52" }
derive_builder = "0.20"
dirs = "6"
enum-ordinalize = "4.3.0"
Expand Down
5 changes: 3 additions & 2 deletions crates/integrations/datafusion/src/physical_plan/scan.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,13 +189,14 @@ impl DisplayAs for IcebergTableScan {
) -> std::fmt::Result {
write!(
f,
"IcebergTableScan projection:[{}] predicate:[{}]",
"IcebergTableScan projection:[{}] predicate:[{}] limit:[{}]",
self.projection
.clone()
.map_or(String::new(), |v| v.join(",")),
self.predicates
.clone()
.map_or(String::from(""), |p| format!("{p}"))
.map_or(String::from(""), |p| format!("{p}")),
self.limit.map_or(String::from(""), |l| format!("{l}"))
)
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ physical_plan
01)FilterExec: data@1 = 0102
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,data] predicate:[data = 0102]
04)------IcebergTableScan projection:[id,data] predicate:[data = 0102] limit:[]

# Verify empty result from empty table
query I?
Expand Down
6 changes: 3 additions & 3 deletions crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ physical_plan
01)FilterExec: is_active@1
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true]
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] limit:[]

# Query with is_active = true
query ITT rowsort
Expand All @@ -61,7 +61,7 @@ physical_plan
01)FilterExec: NOT is_active@1
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false]
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] limit:[]

# Query with is_active = false
query ITT rowsort
Expand All @@ -81,7 +81,7 @@ physical_plan
01)FilterExec: NOT is_active@1
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false]
04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] limit:[]

# Query with is_active != true (includes false and NULL)
query ITT rowsort
Expand Down
4 changes: 2 additions & 2 deletions crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ physical_plan
01)FilterExec: name@1 LIKE Al%
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"]
04)------IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] limit:[]

# Test LIKE filtering with case-sensitive match
query IT rowsort
Expand All @@ -58,7 +58,7 @@ physical_plan
01)FilterExec: name@1 NOT LIKE Al%
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"]
04)------IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] limit:[]

# Test NOT LIKE filtering
query IT rowsort
Expand Down
8 changes: 4 additions & 4 deletions crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ physical_plan
01)FilterExec: ts@1 = 1672921800000000000
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00]
04)------IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] limit:[]

# Verify timestamp equality filtering works
query I?
Expand All @@ -70,7 +70,7 @@ physical_plan
01)FilterExec: ts@1 > 1673308800000000000
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00]
04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] limit:[]

# Verify timestamp greater than filtering
query I? rowsort
Expand Down Expand Up @@ -100,7 +100,7 @@ physical_plan
01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)]
04)------IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] limit:[]

# Test timestamp range predicate filtering
query I? rowsort
Expand Down Expand Up @@ -166,7 +166,7 @@ physical_plan
01)FilterExec: ts@1 > 1672531200000000
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
03)----CooperativeExec
04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00]
04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] limit:[]

query I?
SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP)
Expand Down
Loading