Skip to content

Commit 2c6e00f

Browse files
Wei-hao-LiWei-hao-Li
authored
Support extract value filter push down
1 parent 9fb66af commit 2c6e00f

7 files changed

Lines changed: 296 additions & 59 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java‎

Lines changed: 104 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public void extractUsNsTest() {
118118
}
119119

120120
@Test
121-
public void extractTimeFilterPushDownTest() {
121+
public void extractFilterPushDownTest() {
122122
String[] expectedHeader = new String[] {"time", "s1"};
123123
String[] retArray =
124124
new String[] {
@@ -129,23 +129,34 @@ public void extractTimeFilterPushDownTest() {
129129
expectedHeader,
130130
retArray,
131131
DATABASE_NAME);
132-
// verify the pushdown result is same with non-pushdown
133132
tableResultSetEqualTest(
134133
"SELECT time, s1 FROM table1 where extract(hour from ts) > 1",
135134
expectedHeader,
136135
retArray,
137136
DATABASE_NAME);
137+
// verify the pushdown result is same with non-pushdown
138+
tableResultSetEqualTest(
139+
"SELECT time, s1 FROM table1 where extract(hour from ts) + 1 > 2",
140+
expectedHeader,
141+
retArray,
142+
DATABASE_NAME);
138143
tableResultSetEqualTest(
139144
"SELECT time, s1 FROM table1 where extract(hour from time) >= 2",
140145
expectedHeader,
141146
retArray,
142147
DATABASE_NAME);
143-
// verify the pushdown result is same with non-pushdown
144148
tableResultSetEqualTest(
145149
"SELECT time, s1 FROM table1 where extract(hour from ts) >= 2",
146150
expectedHeader,
147151
retArray,
148152
DATABASE_NAME);
153+
// verify the pushdown result is same with non-pushdown
154+
tableResultSetEqualTest(
155+
"SELECT time, s1 FROM table1 where extract(hour from ts) + 1>= 3",
156+
expectedHeader,
157+
retArray,
158+
DATABASE_NAME);
159+
149160
retArray =
150161
new String[] {
151162
getTimeStrUTC8("2025-07-08T10:18:51") + ",2,",
@@ -162,6 +173,12 @@ public void extractTimeFilterPushDownTest() {
162173
expectedHeader,
163174
retArray,
164175
DATABASE_NAME);
176+
tableResultSetEqualTest(
177+
"SELECT time, s1 FROM table1 where extract(hour from ts) + 1 > 10",
178+
"+08:00",
179+
expectedHeader,
180+
retArray,
181+
DATABASE_NAME);
165182
tableResultSetEqualTest(
166183
"SELECT time, s1 FROM table1 where extract(hour from time) >= 10",
167184
"+08:00",
@@ -174,6 +191,12 @@ public void extractTimeFilterPushDownTest() {
174191
expectedHeader,
175192
retArray,
176193
DATABASE_NAME);
194+
tableResultSetEqualTest(
195+
"SELECT time, s1 FROM table1 where extract(hour from ts)+1>= 11",
196+
"+08:00",
197+
expectedHeader,
198+
retArray,
199+
DATABASE_NAME);
177200

178201
expectedHeader = new String[] {"time", "s1"};
179202
retArray =
@@ -190,6 +213,11 @@ public void extractTimeFilterPushDownTest() {
190213
expectedHeader,
191214
retArray,
192215
DATABASE_NAME);
216+
tableResultSetEqualTest(
217+
"SELECT time, s1 FROM table1 where extract(hour from ts) + 1< 2",
218+
expectedHeader,
219+
retArray,
220+
DATABASE_NAME);
193221
tableResultSetEqualTest(
194222
"SELECT time, s1 FROM table1 where extract(hour from time) <= 0",
195223
expectedHeader,
@@ -200,6 +228,11 @@ public void extractTimeFilterPushDownTest() {
200228
expectedHeader,
201229
retArray,
202230
DATABASE_NAME);
231+
tableResultSetEqualTest(
232+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1<= 1",
233+
expectedHeader,
234+
retArray,
235+
DATABASE_NAME);
203236
tableResultSetEqualTest(
204237
"SELECT time, s1 FROM table1 where extract(hour from time) = 0",
205238
expectedHeader,
@@ -210,6 +243,11 @@ public void extractTimeFilterPushDownTest() {
210243
expectedHeader,
211244
retArray,
212245
DATABASE_NAME);
246+
tableResultSetEqualTest(
247+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1= 1",
248+
expectedHeader,
249+
retArray,
250+
DATABASE_NAME);
213251
retArray =
214252
new String[] {
215253
getTimeStrUTC8("2025-07-09T08:17:50") + ",3,",
@@ -226,6 +264,12 @@ public void extractTimeFilterPushDownTest() {
226264
expectedHeader,
227265
retArray,
228266
DATABASE_NAME);
267+
tableResultSetEqualTest(
268+
"SELECT time, s1 FROM table1 where extract(hour from ts) + 1 < 10",
269+
"+08:00",
270+
expectedHeader,
271+
retArray,
272+
DATABASE_NAME);
229273
tableResultSetEqualTest(
230274
"SELECT time, s1 FROM table1 where extract(hour from time) <= 8",
231275
"+08:00",
@@ -238,6 +282,12 @@ public void extractTimeFilterPushDownTest() {
238282
expectedHeader,
239283
retArray,
240284
DATABASE_NAME);
285+
tableResultSetEqualTest(
286+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1<= 9",
287+
"+08:00",
288+
expectedHeader,
289+
retArray,
290+
DATABASE_NAME);
241291
tableResultSetEqualTest(
242292
"SELECT time, s1 FROM table1 where extract(hour from time) = 8",
243293
"+08:00",
@@ -250,6 +300,12 @@ public void extractTimeFilterPushDownTest() {
250300
expectedHeader,
251301
retArray,
252302
DATABASE_NAME);
303+
tableResultSetEqualTest(
304+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1= 9",
305+
"+08:00",
306+
expectedHeader,
307+
retArray,
308+
DATABASE_NAME);
253309

254310
retArray =
255311
new String[] {
@@ -266,6 +322,27 @@ public void extractTimeFilterPushDownTest() {
266322
expectedHeader,
267323
retArray,
268324
DATABASE_NAME);
325+
tableResultSetEqualTest(
326+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1!= 1",
327+
expectedHeader,
328+
retArray,
329+
DATABASE_NAME);
330+
tableResultSetEqualTest(
331+
"SELECT time, s1 FROM table1 where extract(hour from time) between 1 and 2",
332+
expectedHeader,
333+
retArray,
334+
DATABASE_NAME);
335+
tableResultSetEqualTest(
336+
"SELECT time, s1 FROM table1 where extract(hour from ts) between 1 and 2",
337+
expectedHeader,
338+
retArray,
339+
DATABASE_NAME);
340+
tableResultSetEqualTest(
341+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1 between 2 and 3",
342+
expectedHeader,
343+
retArray,
344+
DATABASE_NAME);
345+
269346
retArray =
270347
new String[] {
271348
getTimeStrUTC8("2025-07-08T09:18:51") + ",1,",
@@ -283,6 +360,30 @@ public void extractTimeFilterPushDownTest() {
283360
expectedHeader,
284361
retArray,
285362
DATABASE_NAME);
363+
tableResultSetEqualTest(
364+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1 != 9",
365+
"+08:00",
366+
expectedHeader,
367+
retArray,
368+
DATABASE_NAME);
369+
tableResultSetEqualTest(
370+
"SELECT time, s1 FROM table1 where extract(hour from time) between 9 and 10",
371+
"+08:00",
372+
expectedHeader,
373+
retArray,
374+
DATABASE_NAME);
375+
tableResultSetEqualTest(
376+
"SELECT time, s1 FROM table1 where extract(hour from ts) between 9 and 10",
377+
"+08:00",
378+
expectedHeader,
379+
retArray,
380+
DATABASE_NAME);
381+
tableResultSetEqualTest(
382+
"SELECT time, s1 FROM table1 where extract(hour from ts) +1 between 10 and 11",
383+
"+08:00",
384+
expectedHeader,
385+
retArray,
386+
DATABASE_NAME);
286387
}
287388

288389
@Test

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression;
3030
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral;
3131
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
32+
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract;
3233
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral;
3334
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression;
3435
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression;
@@ -47,13 +48,15 @@
4748
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference;
4849
import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
4950

51+
import com.google.common.collect.ImmutableList;
5052
import org.apache.tsfile.common.conf.TSFileConfig;
5153
import org.apache.tsfile.common.regexp.LikePattern;
5254
import org.apache.tsfile.enums.TSDataType;
5355
import org.apache.tsfile.read.common.type.Type;
5456
import org.apache.tsfile.read.filter.basic.Filter;
5557
import org.apache.tsfile.read.filter.factory.FilterFactory;
5658
import org.apache.tsfile.read.filter.factory.ValueFilterApi;
59+
import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators;
5760
import org.apache.tsfile.utils.Binary;
5861

5962
import javax.annotation.Nullable;
@@ -72,18 +75,24 @@
7275
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor.getLongValue;
7376
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral;
7477
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference;
78+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn;
7579
import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn;
80+
import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP;
7681

7782
public class ConvertPredicateToFilterVisitor
7883
extends PredicateVisitor<Filter, ConvertPredicateToFilterVisitor.Context> {
7984

8085
@Nullable private final String timeColumnName;
8186
private final ConvertPredicateToTimeFilterVisitor timeFilterVisitor;
87+
private final ZoneId zoneId;
88+
private final TimeUnit currPrecision;
8289

8390
public ConvertPredicateToFilterVisitor(
8491
@Nullable String timeColumnName, ZoneId zoneId, TimeUnit currPrecision) {
8592
this.timeColumnName = timeColumnName;
8693
this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision);
94+
this.zoneId = zoneId;
95+
this.currPrecision = currPrecision;
8796
}
8897

8998
@Override
@@ -168,6 +177,48 @@ public static <T extends Comparable<T>> Filter constructCompareFilter(
168177
}
169178
}
170179

180+
private Filter constructExtractCompareFilter(
181+
ComparisonExpression.Operator operator,
182+
SymbolReference symbolReference,
183+
Extract.Field field,
184+
Literal literal,
185+
Context context) {
186+
187+
if (!context.isMeasurementColumn(symbolReference)) {
188+
throw new IllegalStateException(
189+
String.format("Only support measurement column in filter: %s", symbolReference));
190+
}
191+
192+
int measurementIndex = context.getMeasurementIndex(symbolReference.getName());
193+
long value = getValue(literal, TIMESTAMP);
194+
ExtractTimeFilterOperators.Field field1 =
195+
ExtractTimeFilterOperators.Field.values()[field.ordinal()];
196+
197+
switch (operator) {
198+
case EQUAL:
199+
return ValueFilterApi.extractValueEq(
200+
measurementIndex, value, field1, zoneId, currPrecision);
201+
case NOT_EQUAL:
202+
return ValueFilterApi.extractValueNotEq(
203+
measurementIndex, value, field1, zoneId, currPrecision);
204+
case GREATER_THAN:
205+
return ValueFilterApi.extractValueGt(
206+
measurementIndex, value, field1, zoneId, currPrecision);
207+
case GREATER_THAN_OR_EQUAL:
208+
return ValueFilterApi.extractValueGtEq(
209+
measurementIndex, value, field1, zoneId, currPrecision);
210+
case LESS_THAN:
211+
return ValueFilterApi.extractValueLt(
212+
measurementIndex, value, field1, zoneId, currPrecision);
213+
case LESS_THAN_OR_EQUAL:
214+
return ValueFilterApi.extractValueLtEq(
215+
measurementIndex, value, field1, zoneId, currPrecision);
216+
default:
217+
throw new IllegalArgumentException(
218+
String.format("Unsupported extract comparison operator %s", operator));
219+
}
220+
}
221+
171222
@SuppressWarnings("unchecked")
172223
public static <T extends Comparable<T>> T getValue(Literal value, Type dataType) {
173224
try {
@@ -273,6 +324,22 @@ && isSymbolReference(right)
273324
&& context.isMeasurementColumn((SymbolReference) right)) {
274325
return constructCompareFilter(
275326
node.getOperator().flip(), (SymbolReference) right, (Literal) left, context);
327+
} else if (context.isExtractMeasurementColumn(left) && isLiteral(right)) {
328+
Extract extract = (Extract) left;
329+
return constructExtractCompareFilter(
330+
node.getOperator(),
331+
(SymbolReference) extract.getExpression(),
332+
extract.getField(),
333+
(Literal) right,
334+
context);
335+
} else if (isLiteral(left) && context.isExtractMeasurementColumn(right)) {
336+
Extract extract = (Extract) right;
337+
return constructExtractCompareFilter(
338+
node.getOperator().flip(),
339+
(SymbolReference) extract.getExpression(),
340+
extract.getField(),
341+
(Literal) left,
342+
context);
276343
} else {
277344
throw new IllegalStateException(
278345
String.format("%s is not supported in value push down", node));
@@ -307,7 +374,10 @@ protected Filter visitBetweenPredicate(BetweenPredicate node, Context context) {
307374

308375
if (isTimeColumn(firstExpression, timeColumnName)
309376
|| isTimeColumn(secondExpression, timeColumnName)
310-
|| isTimeColumn(thirdExpression, timeColumnName)) {
377+
|| isTimeColumn(thirdExpression, timeColumnName)
378+
|| isExtractTimeColumn(firstExpression, timeColumnName)
379+
|| isExtractTimeColumn(secondExpression, timeColumnName)
380+
|| isExtractTimeColumn(thirdExpression, timeColumnName)) {
311381
return timeFilterVisitor.process(node, null);
312382
}
313383

@@ -331,6 +401,33 @@ protected Filter visitBetweenPredicate(BetweenPredicate node, Context context) {
331401
(SymbolReference) thirdExpression,
332402
(Literal) firstExpression,
333403
context);
404+
} else if (context.isExtractMeasurementColumn(firstExpression)) {
405+
checkArgument(isLiteral(secondExpression));
406+
checkArgument(isLiteral(thirdExpression));
407+
long minValue = getLongValue(secondExpression);
408+
long maxValue = getLongValue(thirdExpression);
409+
Extract extract = (Extract) firstExpression;
410+
int measurementIndex =
411+
context.getMeasurementIndex(((SymbolReference) extract.getExpression()).getName());
412+
ExtractTimeFilterOperators.Field field =
413+
ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()];
414+
415+
if (minValue == maxValue) {
416+
return ValueFilterApi.extractValueEq(
417+
measurementIndex, minValue, field, zoneId, currPrecision);
418+
}
419+
return FilterFactory.and(
420+
ImmutableList.of(
421+
ValueFilterApi.extractValueGtEq(
422+
measurementIndex, minValue, field, zoneId, currPrecision),
423+
ValueFilterApi.extractValueLtEq(
424+
measurementIndex, maxValue, field, zoneId, currPrecision)));
425+
} else if (context.isExtractMeasurementColumn(secondExpression)) {
426+
throw new IllegalStateException(
427+
"Should not reach here before PredicateCombineIntoTableScanChecker support Extract push-down in third child");
428+
} else if (context.isExtractMeasurementColumn(thirdExpression)) {
429+
throw new IllegalStateException(
430+
"Should not reach here before PredicateCombineIntoTableScanChecker support Extract push-down in third child");
334431
} else {
335432
throw new IllegalStateException(
336433
String.format("%s is not supported in value push down", node));
@@ -429,5 +526,14 @@ public boolean isMeasurementColumn(SymbolReference symbolReference) {
429526
ColumnSchema schema = schemaMap.get(Symbol.from(symbolReference));
430527
return schema != null && schema.getColumnCategory() == TsTableColumnCategory.FIELD;
431528
}
529+
530+
public boolean isExtractMeasurementColumn(Expression expression) {
531+
if (expression instanceof Extract
532+
&& ((Extract) expression).getExpression() instanceof SymbolReference) {
533+
ColumnSchema schema = schemaMap.get(Symbol.from(((Extract) expression).getExpression()));
534+
return schema != null && schema.getColumnCategory() == TsTableColumnCategory.FIELD;
535+
}
536+
return false;
537+
}
432538
}
433539
}

0 commit comments

Comments
 (0)