Skip to content

Commit 62d378e

Browse files
sundapengclaude
sundapeng
and
claude
committed
[core] Fix default partition handling and improve predicate pushdown correctness
- Handle __DEFAULT_PARTITION__ in buildPartitionPredicate() by generating isNull() instead of castFromString() which throws NumberFormatException on non-string partition types (e.g. INT) - Fix scan path to skip pushdown for unsupported predicates instead of returning empty results - Pass defaultPartitionName through all system table callers Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent e5185ab commit 62d378e

5 files changed

Lines changed: 62 additions & 32 deletions

File tree

‎paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,8 @@ public RecordReader<InternalRow> createReader(Split split) {
234234
snapshotReader,
235235
bucketsSplit.partitionPredicate,
236236
partitionKeys,
237-
partitionType);
237+
partitionType,
238+
fileStoreTable.coreOptions().partitionDefaultName());
238239
if (!hasResults) {
239240
return new IteratorRecordReader<>(Collections.emptyIterator());
240241
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/FileKeyRangesTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,11 @@ public Plan innerPlan() {
170170
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
171171
boolean hasResults =
172172
PartitionPredicateHelper.applyPartitionFilter(
173-
snapshotReader, partitionPredicate, partitionKeys, partitionType);
173+
snapshotReader,
174+
partitionPredicate,
175+
partitionKeys,
176+
partitionType,
177+
fileStoreTable.coreOptions().partitionDefaultName());
174178
if (!hasResults) {
175179
return Collections::emptyList;
176180
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,11 @@ public Plan innerPlan() {
196196
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
197197
boolean hasResults =
198198
PartitionPredicateHelper.applyPartitionFilter(
199-
snapshotReader, partitionPredicate, partitionKeys, partitionType);
199+
snapshotReader,
200+
partitionPredicate,
201+
partitionKeys,
202+
partitionType,
203+
fileStoreTable.coreOptions().partitionDefaultName());
200204
if (!hasResults) {
201205
return Collections::emptyList;
202206
}

‎paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -455,13 +455,16 @@ private List<Partition> listPartitionEntries(@Nullable LeafPredicate partitionPr
455455
if (partitionPredicate != null) {
456456
List<String> partitionKeys = fileStoreTable.partitionKeys();
457457
RowType partitionType = fileStoreTable.schema().logicalPartitionType();
458+
String defaultPartitionName = fileStoreTable.coreOptions().partitionDefaultName();
458459
Predicate partPred =
459460
PartitionPredicateHelper.buildPartitionPredicate(
460-
partitionPredicate, partitionKeys, partitionType);
461-
if (partPred == null) {
462-
return Collections.emptyList();
461+
partitionPredicate,
462+
partitionKeys,
463+
partitionType,
464+
defaultPartitionName);
465+
if (partPred != null) {
466+
scan.withPartitionFilter(partPred);
463467
}
464-
scan.withPartitionFilter(partPred);
465468
}
466469

467470
List<PartitionEntry> partitionEntries = scan.listPartitionEntries();

‎paimon-core/src/main/java/org/apache/paimon/utils/PartitionPredicateHelper.java‎

Lines changed: 43 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,17 @@ public class PartitionPredicateHelper {
4242

4343
/**
4444
* Build a partition-typed predicate from a string-based leaf predicate on the "partition"
45-
* column.
45+
* column. Only Equal and In predicates are supported for pushdown.
4646
*
47-
* @return the predicate on partition fields, or {@code null} if the partition spec is invalid
48-
* (indicating no results should be returned)
47+
* @return the predicate on partition fields, or {@code null} if the predicate cannot be pushed
48+
* down (unsupported predicate type or invalid partition spec)
4949
*/
5050
@Nullable
5151
public static Predicate buildPartitionPredicate(
52-
LeafPredicate partitionPredicate, List<String> partitionKeys, RowType partitionType) {
52+
LeafPredicate partitionPredicate,
53+
List<String> partitionKeys,
54+
RowType partitionType,
55+
String defaultPartitionName) {
5356
if (partitionPredicate.function() instanceof Equal) {
5457
LinkedHashMap<String, String> partSpec =
5558
parsePartitionSpec(
@@ -60,10 +63,13 @@ public static Predicate buildPartitionPredicate(
6063
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
6164
List<Predicate> predicates = new ArrayList<>();
6265
for (int i = 0; i < partitionKeys.size(); i++) {
63-
Object value =
64-
TypeUtils.castFromString(
65-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
66-
predicates.add(partBuilder.equal(i, value));
66+
String strValue = partSpec.get(partitionKeys.get(i));
67+
if (defaultPartitionName.equals(strValue)) {
68+
predicates.add(partBuilder.isNull(i));
69+
} else {
70+
Object value = TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
71+
predicates.add(partBuilder.equal(i, value));
72+
}
6773
}
6874
return PredicateBuilder.and(predicates);
6975
} else if (partitionPredicate.function() instanceof In) {
@@ -77,15 +83,22 @@ public static Predicate buildPartitionPredicate(
7783
}
7884
List<Predicate> andPredicates = new ArrayList<>();
7985
for (int i = 0; i < partitionKeys.size(); i++) {
80-
Object value =
81-
TypeUtils.castFromString(
82-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
83-
andPredicates.add(partBuilder.equal(i, value));
86+
String strValue = partSpec.get(partitionKeys.get(i));
87+
if (defaultPartitionName.equals(strValue)) {
88+
andPredicates.add(partBuilder.isNull(i));
89+
} else {
90+
Object value =
91+
TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
92+
andPredicates.add(partBuilder.equal(i, value));
93+
}
8494
}
8595
orPredicates.add(PredicateBuilder.and(andPredicates));
8696
}
8797
return orPredicates.isEmpty() ? null : PredicateBuilder.or(orPredicates);
88-
} else if (partitionPredicate.function() instanceof LeafBinaryFunction) {
98+
}
99+
// Range predicates (>, >=, <, <=) can be pushed down for simple value formats
100+
// used by BucketsTable, FilesTable, FileKeyRangesTable.
101+
if (partitionPredicate.function() instanceof LeafBinaryFunction) {
89102
LinkedHashMap<String, String> partSpec =
90103
parsePartitionSpec(
91104
partitionPredicate.literals().get(0).toString(), partitionKeys);
@@ -95,16 +108,19 @@ public static Predicate buildPartitionPredicate(
95108
PredicateBuilder partBuilder = new PredicateBuilder(partitionType);
96109
List<Predicate> predicates = new ArrayList<>();
97110
for (int i = 0; i < partitionKeys.size(); i++) {
98-
Object value =
99-
TypeUtils.castFromString(
100-
partSpec.get(partitionKeys.get(i)), partitionType.getTypeAt(i));
101-
predicates.add(
102-
new LeafPredicate(
103-
partitionPredicate.function(),
104-
partitionType.getTypeAt(i),
105-
i,
106-
partitionKeys.get(i),
107-
Collections.singletonList(value)));
111+
String strValue = partSpec.get(partitionKeys.get(i));
112+
if (defaultPartitionName.equals(strValue)) {
113+
predicates.add(partBuilder.isNull(i));
114+
} else {
115+
Object value = TypeUtils.castFromString(strValue, partitionType.getTypeAt(i));
116+
predicates.add(
117+
new LeafPredicate(
118+
partitionPredicate.function(),
119+
partitionType.getTypeAt(i),
120+
i,
121+
partitionKeys.get(i),
122+
Collections.singletonList(value)));
123+
}
108124
}
109125
return PredicateBuilder.and(predicates);
110126
}
@@ -115,13 +131,15 @@ public static boolean applyPartitionFilter(
115131
SnapshotReader snapshotReader,
116132
@Nullable LeafPredicate partitionPredicate,
117133
List<String> partitionKeys,
118-
RowType partitionType) {
134+
RowType partitionType,
135+
String defaultPartitionName) {
119136
if (partitionPredicate == null) {
120137
return true;
121138
}
122139

123140
Predicate predicate =
124-
buildPartitionPredicate(partitionPredicate, partitionKeys, partitionType);
141+
buildPartitionPredicate(
142+
partitionPredicate, partitionKeys, partitionType, defaultPartitionName);
125143
if (predicate == null) {
126144
return false;
127145
}

0 commit comments

Comments
 (0)