From f24cedcc33f7c5642c27042008412dd99a1564ff Mon Sep 17 00:00:00 2001 From: Weihao Li <18110526956@163.com> Date: Wed, 2 Jul 2025 12:00:32 +0800 Subject: [PATCH 1/8] implement Signed-off-by: Weihao Li <18110526956@163.com> --- .../relational/ColumnTransformerBuilder.java | 23 +++ .../analyzer/ExpressionAnalyzer.java | 13 ++ .../analyzer/StatementAnalyzer.java | 17 +++ .../relational/planner/IrTypeAnalyzer.java | 6 + .../planner/ir/ExpressionRewriter.java | 6 + .../planner/ir/ExpressionTreeRewriter.java | 20 +++ .../plan/relational/sql/ast/AstVisitor.java | 4 + .../plan/relational/sql/ast/Expression.java | 3 + .../plan/relational/sql/ast/Extract.java | 139 ++++++++++++++++++ .../sql/ast/TableExpressionType.java | 3 +- .../relational/sql/parser/AstBuilder.java | 14 ++ .../unary/scalar/ExtractTransformer.java | 106 +++++++++++++ .../apache/iotdb/db/utils/DateTimeUtils.java | 18 ++- .../relational/grammar/sql/RelationalSql.g4 | 1 + 14 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index d0b7e35e94ce4..0e915ab258cb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DecimalLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; @@ -122,6 +123,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.EndsWith2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.EndsWithColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ExpColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ExtractTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.FloorColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.FormatColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.LTrim2ColumnTransformer; @@ -343,6 +345,27 @@ protected ColumnTransformer visitCast(Cast node, Context context) { return getColumnTransformerFromCacheAndAddReferenceCount(node, context); } + @Override + protected ColumnTransformer visitExtract(Extract node, Context context) { + if (!context.cache.containsKey(node)) { + if (context.hasSeen.containsKey(node)) { + ColumnTransformer columnTransformer = context.hasSeen.get(node); + appendIdentityColumnTransformer( + node, + columnTransformer.getType(), + getTSDataType(columnTransformer.getType()), + context, + columnTransformer); + } else { + ColumnTransformer child = this.process(node.getExpression(), context); + context.cache.put( + node, + new ExtractTransformer(INT64, child, node.getField(), context.sessionInfo.getZoneId())); + } + } + return getColumnTransformerFromCacheAndAddReferenceCount(node, context); + } + @Override protected ColumnTransformer visitBooleanLiteral(BooleanLiteral node, Context context) { ColumnTransformer res = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index fad9bc32d2f81..01fa13e0924d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -104,6 +105,7 @@ import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; import javax.annotation.Nullable; @@ -1451,6 +1453,17 @@ protected Type visitParameter(Parameter node, StackableAstVisitorContext context) { + Type type = process(node.getExpression(), context); + + if (!(type instanceof TimestampType)) { + throw new SemanticException(String.format("Cannot extract from %s", type)); + } + + return setExpressionType(node, INT64); + } + @Override protected Type visitBetweenPredicate( BetweenPredicate node, StackableAstVisitorContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index a0afd0453afb8..2ec3676acaee1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -95,6 +95,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; @@ -1996,6 +1997,22 @@ protected List visitRow(Row node, Scope context) { String.format("%s are not supported now", node.getClass().getSimpleName())); } + @Override + protected List visitExtract(Extract node, Scope context) { + List childResult = process(node.getExpression(), context); + if (expandedExpressions == null) { + // no Columns need to be expanded + return Collections.singletonList(node); + } + + ImmutableList.Builder resultBuilder = new ImmutableList.Builder<>(); + for (Expression expression : childResult) { + resultBuilder.add(new Extract(expression, node.getField())); + } + + return resultBuilder.build(); + } + @Override protected List visitSearchedCaseExpression( SearchedCaseExpression node, Scope context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index 1ecf2dad77dc7..0aa43ef8123e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentUser; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; @@ -306,6 +307,11 @@ protected Type visitArithmeticUnary(ArithmeticUnaryExpression node, Context cont return setExpressionType(node, process(node.getValue(), context)); } + @Override + protected Type visitExtract(Extract node, Context context) { + return setExpressionType(node, INT64); + } + @Override protected Type visitArithmeticBinary(ArithmeticBinaryExpression node, Context context) { ImmutableList.Builder argumentTypes = ImmutableList.builder(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index d232101d6037c..159655d4255b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -174,6 +175,11 @@ public Expression rewriteDereferenceExpression( return rewriteExpression(node, context, treeRewriter); } + public Expression rewriteExtract( + Extract node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + // public Expression rewriteBindExpression(BindExpression node, C context, // ExpressionTreeRewriter treeRewriter) { // return rewriteExpression(node, context, treeRewriter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index 585829e25401d..d84ac593306fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -625,6 +626,25 @@ public Expression visitDereferenceExpression(DereferenceExpression node, Context return node; } + @Override + protected Expression visitExtract(Extract node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteExtract(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + Expression expression = rewrite(node.getExpression(), context.get()); + + if (node.getExpression() != expression) { + return new Extract(expression, node.getField()); + } + + return node; + } + @Override public Expression visitCast(Cast node, Context context) { if (!context.isDefaultRewrite()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index fd3eda8e6f059..e48bb38da9399 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -229,6 +229,10 @@ protected R visitNotExpression(NotExpression node, C context) { return visitExpression(node, context); } + protected R visitExtract(Extract node, C context) { + return visitExpression(node, context); + } + protected R visitWindowDefinition(WindowDefinition node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java index 53eadfb149cc9..f4bc64e25523e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java @@ -161,6 +161,9 @@ public static Expression deserialize(ByteBuffer byteBuffer) { case 31: expression = new Row(byteBuffer); break; + case 32: + expression = new Extract(byteBuffer); + break; default: throw new IllegalArgumentException("Invalid expression type: " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java new file mode 100644 index 0000000000000..efeb92c527820 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.Immutable; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +@Immutable +public class Extract extends Expression { + private final Expression expression; + private final Field field; + + public enum Field { + YEAR, + QUARTER, + MONTH, + WEEK, + DAY, + DAY_OF_MONTH, + DAY_OF_WEEK, + DOW, + DAY_OF_YEAR, + DOY, + HOUR, + MINUTE, + SECOND, + MS, + US, + NS + } + + public Extract(Expression expression, Field field) { + this(null, expression, field); + } + + public Extract(NodeLocation location, Expression expression, Field field) { + super(location); + requireNonNull(expression, "expression is null"); + requireNonNull(field, "field is null"); + + this.expression = expression; + this.field = field; + } + + public Extract(ByteBuffer byteBuffer) { + super(null); + expression = deserialize(byteBuffer); + field = Field.values()[ReadWriteIOUtils.readInt(byteBuffer)]; + } + + @Override + protected void serialize(ByteBuffer byteBuffer) { + expression.serialize(byteBuffer); + ReadWriteIOUtils.write(field.ordinal(), byteBuffer); + } + + @Override + protected void serialize(DataOutputStream stream) throws IOException { + expression.serialize(stream); + ReadWriteIOUtils.write(field.ordinal(), stream); + } + + @Override + public TableExpressionType getExpressionType() { + return TableExpressionType.EXTRACT; + } + + public Expression getExpression() { + return expression; + } + + public Field getField() { + return field; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitExtract(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(expression); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Extract that = (Extract) o; + return Objects.equals(expression, that.expression) && (field == that.field); + } + + @Override + public int hashCode() { + return Objects.hash(expression, field); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + Extract otherExtract = (Extract) other; + return field.equals(otherExtract.field); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java index 78ef4feb09e75..9d0b7ae0b4aa1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java @@ -50,7 +50,8 @@ public enum TableExpressionType { WHEN_CLAUSE((short) 28), CURRENT_DATABASE((short) 29), CURRENT_USER((short) 30), - ROW((short) 31); + ROW((short) 31), + EXTRACT((short) 32); TableExpressionType(short type) { this.type = type; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 26fd39070fd4f..2237bc7829aac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -91,6 +91,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; @@ -282,6 +283,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Long.parseLong; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; @@ -3090,6 +3092,18 @@ public Node visitCurrentUser(RelationalSqlParser.CurrentUserContext ctx) { return new CurrentUser(getLocation(ctx)); } + @Override + public Node visitExtract(RelationalSqlParser.ExtractContext context) { + String fieldString = context.identifier().getText(); + Extract.Field field; + try { + field = Extract.Field.valueOf(fieldString.toUpperCase(ENGLISH)); + } catch (IllegalArgumentException e) { + throw parseError("Invalid EXTRACT field: " + fieldString, context); + } + return new Extract(getLocation(context), (Expression) visit(context.valueExpression()), field); + } + @Override public Node visitSubqueryExpression(RelationalSqlParser.SubqueryExpressionContext ctx) { return new SubqueryExpression(getLocation(ctx), (Query) visit(ctx.query())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java new file mode 100644 index 0000000000000..05d52981c0aaf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.time.ZoneId; +import java.util.function.Function; + +import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_MS_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_NS_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_US_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.convertToZonedDateTime; + +public class ExtractTransformer extends UnaryColumnTransformer { + private final Function evaluateFunction; + + public ExtractTransformer( + Type type, ColumnTransformer child, Extract.Field field, ZoneId zoneId) { + super(type, child); + this.evaluateFunction = constructEvaluateFunction(field, zoneId); + } + + private Function constructEvaluateFunction(Extract.Field field, ZoneId zoneId) { + switch (field) { + case YEAR: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getYear(); + case QUARTER: + return timestamp -> (convertToZonedDateTime(timestamp, zoneId).getMonthValue() + 2L) / 3L; + case MONTH: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getMonthValue(); + case WEEK: + return timestamp -> convertToZonedDateTime(timestamp, zoneId).getLong(ALIGNED_WEEK_OF_YEAR); + case DAY: + case DAY_OF_MONTH: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getDayOfMonth(); + case DAY_OF_WEEK: + case DOW: + return timestamp -> + (long) convertToZonedDateTime(timestamp, zoneId).getDayOfWeek().getValue(); + case DAY_OF_YEAR: + case DOY: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getDayOfYear(); + case HOUR: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getHour(); + case MINUTE: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getMinute(); + case SECOND: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getSecond(); + case MS: + return EXTRACT_TIMESTAMP_MS_PART; + case US: + return EXTRACT_TIMESTAMP_US_PART; + case NS: + return EXTRACT_TIMESTAMP_NS_PART; + default: + throw new UnsupportedOperationException("Unexpected extract field: " + field); + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (!column.isNull(i)) { + columnBuilder.writeLong(evaluateFunction.apply(column.getLong(i))); + } else { + columnBuilder.appendNull(); + } + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (selection[i] && !column.isNull(i)) { + columnBuilder.writeLong(evaluateFunction.apply(column.getLong(i))); + } else { + columnBuilder.appendNull(); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index a94bda7303387..e1114bbe4491d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -86,22 +86,38 @@ public static long correctPrecision(long millis) { } } - private static Function CAST_TIMESTAMP_TO_MS; + public static final Function CAST_TIMESTAMP_TO_MS; + + public static final Function EXTRACT_TIMESTAMP_MS_PART; + public static final Function EXTRACT_TIMESTAMP_US_PART; + public static final Function EXTRACT_TIMESTAMP_NS_PART; static { switch (CommonDescriptor.getInstance().getConfig().getTimestampPrecision()) { case "us": case "microsecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000; + + EXTRACT_TIMESTAMP_MS_PART = timestamp -> (timestamp % 1000_000) / 1000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> timestamp % 1000; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; case "ns": case "nanosecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000000; + + EXTRACT_TIMESTAMP_MS_PART = timestamp -> (timestamp % 1000_000_000) / 1000_000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> (timestamp % 1000_000) / 1000; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> timestamp % 1000; break; case "ms": case "millisecond": default: CAST_TIMESTAMP_TO_MS = timestamp -> timestamp; + + EXTRACT_TIMESTAMP_MS_PART = timestamp -> timestamp % 1000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> 0L; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; } } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 91bb547c27feb..8b02a79ffc50a 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1115,6 +1115,7 @@ primaryExpression trimSource=valueExpression ')' #trim | TRIM '(' trimSource=valueExpression ',' trimChar=valueExpression ')' #trim | SUBSTRING '(' valueExpression FROM valueExpression (FOR valueExpression)? ')' #substring + | EXTRACT '(' identifier FROM valueExpression ')' #extract | DATE_BIN '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBin | DATE_BIN_GAPFILL '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBinGapFill | '(' expression ')' #parenthesizedExpression From a4a69990f3a3a7858bd9f969c8680f60c8ba93b8 Mon Sep 17 00:00:00 2001 From: Weihao Li <18110526956@163.com> Date: Thu, 3 Jul 2025 22:38:15 +0800 Subject: [PATCH 2/8] fix Signed-off-by: Weihao Li <18110526956@163.com> --- .../plan/relational/analyzer/AggregationAnalyzer.java | 6 ++++++ .../plan/relational/sql/ast/DefaultTraversalVisitor.java | 5 +++++ .../plan/relational/sql/util/ExpressionFormatter.java | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java index 4e3e12f723ab0..93f7847c5faaf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; @@ -212,6 +213,11 @@ protected Boolean visitNullIfExpression(NullIfExpression node, Void context) { return process(node.getFirst(), context) && process(node.getSecond(), context); } + @Override + protected Boolean visitExtract(Extract node, Void context) { + return process(node.getExpression(), context); + } + @Override protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { return process(node.getMin(), context) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java index 164c86d425ca1..f22af4b020e1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java @@ -20,6 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; public abstract class DefaultTraversalVisitor extends AstVisitor { + @Override + protected Void visitExtract(Extract node, C context) { + process(node.getExpression(), context); + return null; + } @Override protected Void visitArithmeticBinary(ArithmeticBinaryExpression node, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index 47e00fb0fb69a..9f1afa6a5f38b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -184,6 +185,11 @@ protected String visitCurrentTime(CurrentTime node, Void context) { return builder.toString(); } + @Override + protected String visitExtract(Extract node, Void context) { + return "EXTRACT(" + node.getField() + " FROM " + process(node.getExpression(), context) + ")"; + } + @Override protected String visitBooleanLiteral(BooleanLiteral node, Void context) { return literalFormatter From 7f07f68822deb6114f6e01671412f03954832d2e Mon Sep 17 00:00:00 2001 From: Weihao Li <18110526956@163.com> Date: Wed, 16 Jul 2025 21:23:01 +0800 Subject: [PATCH 3/8] implement push-down Signed-off-by: Weihao Li <18110526956@163.com> --- .../fragment/FragmentInstanceContext.java | 6 +- .../plan/analyze/PredicateUtils.java | 13 ++- .../plan/planner/TableOperatorGenerator.java | 24 +++++- .../planner/plan/TableModelTimePredicate.java | 6 +- .../plan/planner/plan/TimePredicate.java | 4 +- .../planner/plan/TreeModelTimePredicate.java | 4 +- .../analyzer/ExpressionAnalyzer.java | 11 ++- .../ConvertPredicateToFilterVisitor.java | 7 +- .../ConvertPredicateToTimeFilterVisitor.java | 80 ++++++++++++++++++- .../PredicateCombineIntoTableScanChecker.java | 23 ++++-- .../ir/GlobalTimePredicateExtractVisitor.java | 15 +++- .../PushPredicateIntoTableScan.java | 13 ++- .../apache/iotdb/db/utils/DateTimeUtils.java | 16 ++-- pom.xml | 2 +- 14 files changed, 181 insertions(+), 43 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 8b9a2a274a14c..8fc462bbd4285 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp; @@ -235,7 +236,10 @@ private FragmentInstanceContext( this.sessionInfo = sessionInfo; this.dataRegion = dataRegion; this.globalTimeFilter = - globalTimePredicate == null ? null : globalTimePredicate.convertPredicateToTimeFilter(); + globalTimePredicate == null + ? null + : globalTimePredicate.convertPredicateToTimeFilter( + sessionInfo.getZoneId(), TimestampPrecisionUtils.currPrecision); this.dataNodeQueryContextMap = dataNodeQueryContextMap; this.dataNodeQueryContext = dataNodeQueryContextMap.get(id.getQueryId()); this.memoryReservationManager = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java index 9fb41fc152d1c..cf4b3a8294c25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class PredicateUtils { @@ -282,13 +283,15 @@ public static Filter convertPredicateToTimeFilter(Expression predicate) { } public static Filter convertPredicateToTimeFilter( - org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate) { + org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate, + ZoneId zoneId, + TimeUnit currPrecision) { if (predicate == null) { return null; } return predicate.accept( new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate - .ConvertPredicateToTimeFilterVisitor(), + .ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision), null); } @@ -311,13 +314,15 @@ public static Filter convertPredicateToFilter( org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate, Map measurementColumnsIndexMap, Map schemaMap, - String timeColumnName) { + String timeColumnName, + ZoneId zoneId, + TimeUnit currPrecision) { if (predicate == null) { return null; } return predicate.accept( new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate - .ConvertPredicateToFilterVisitor(timeColumnName), + .ConvertPredicateToFilterVisitor(timeColumnName, zoneId, currPrecision), new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate .ConvertPredicateToFilterVisitor.Context(measurementColumnsIndexMap, schemaMap)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 8856a2ec43d9a..ebb704f518f73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -226,6 +226,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.datastructure.SortKey; import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; @@ -606,7 +607,11 @@ private void calculateSeriesScanOptionsList() { Filter timeFilter = null; if (node.getTimePredicate().isPresent()) { Expression timePredicate = node.getTimePredicate().get(); - timeFilter = timePredicate.accept(new ConvertPredicateToTimeFilterVisitor(), null); + timeFilter = + timePredicate.accept( + new ConvertPredicateToTimeFilterVisitor( + context.getZoneId(), TimestampPrecisionUtils.currPrecision), + null); context .getDriverContext() .getFragmentInstanceContext() @@ -649,7 +654,9 @@ private void calculateSeriesScanOptionsList() { pushDownPredicateForCurrentMeasurement, Collections.singletonMap(measurementSchema.getMeasurementName(), 0), commonParameter.columnSchemaMap, - commonParameter.timeColumnName)); + commonParameter.timeColumnName, + context.getZoneId(), + TimestampPrecisionUtils.currPrecision)); } if (isSingleColumn || (pushDownOffsetAndLimitToLeftChildSeriesScanOperator @@ -1226,7 +1233,11 @@ private SeriesScanOptions.Builder getSeriesScanOptionsBuilder( LocalExecutionPlanContext context, @NotNull Expression timePredicate) { SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); - Filter timeFilter = timePredicate.accept(new ConvertPredicateToTimeFilterVisitor(), null); + Filter timeFilter = + timePredicate.accept( + new ConvertPredicateToTimeFilterVisitor( + context.getZoneId(), TimestampPrecisionUtils.currPrecision), + null); context.getDriverContext().getFragmentInstanceContext().setTimeFilterForTableModel(timeFilter); // time filter may be stateful, so we need to copy it scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); @@ -3104,7 +3115,12 @@ private SeriesScanOptions buildSeriesScanOptions( if (pushDownPredicate != null) { scanOptionsBuilder.withPushDownFilter( convertPredicateToFilter( - pushDownPredicate, measurementColumnsIndexMap, columnSchemaMap, timeColumnName)); + pushDownPredicate, + measurementColumnsIndexMap, + columnSchemaMap, + timeColumnName, + context.getZoneId(), + TimestampPrecisionUtils.currPrecision)); } return scanOptionsBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java index b299c04e6d7a8..dcdb79bf79e78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java @@ -27,7 +27,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.time.ZoneId; import java.util.Objects; +import java.util.concurrent.TimeUnit; public class TableModelTimePredicate implements TimePredicate { @@ -44,8 +46,8 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public Filter convertPredicateToTimeFilter() { - return PredicateUtils.convertPredicateToTimeFilter(timePredicate); + public Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision) { + return PredicateUtils.convertPredicateToTimeFilter(timePredicate, zoneId, currPrecision); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java index 440c34a65aab4..d424aa82d09a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java @@ -27,12 +27,14 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; public interface TimePredicate { void serialize(DataOutputStream stream) throws IOException; - Filter convertPredicateToTimeFilter(); + Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision); static TimePredicate deserialize(ByteBuffer byteBuffer) { // 0 for tree model, 1 for table model diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java index 5cc00125aa794..132077d801b61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java @@ -27,7 +27,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.time.ZoneId; import java.util.Objects; +import java.util.concurrent.TimeUnit; public class TreeModelTimePredicate implements TimePredicate { @@ -44,7 +46,7 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public Filter convertPredicateToTimeFilter() { + public Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision) { return PredicateUtils.convertPredicateToTimeFilter(timePredicate); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 01fa13e0924d0..cf72834a1ce64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -1455,10 +1455,15 @@ protected Type visitParameter(Parameter node, StackableAstVisitorContext context) { - Type type = process(node.getExpression(), context); + if (node.getExpression() instanceof LongLiteral) { + // Don't visit child here to avoid setting its Type to INT32 + setExpressionType(node.getExpression(), INT64); + } else { + Type type = process(node.getExpression(), context); - if (!(type instanceof TimestampType)) { - throw new SemanticException(String.format("Cannot extract from %s", type)); + if (!(type instanceof TimestampType)) { + throw new SemanticException(String.format("Cannot extract from %s", type)); + } } return setExpressionType(node, INT64); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index 1704ccab7e0a9..0c9f42b3e63e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -58,11 +58,13 @@ import javax.annotation.Nullable; +import java.time.ZoneId; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -78,9 +80,10 @@ public class ConvertPredicateToFilterVisitor @Nullable private final String timeColumnName; private final ConvertPredicateToTimeFilterVisitor timeFilterVisitor; - public ConvertPredicateToFilterVisitor(@Nullable String timeColumnName) { + public ConvertPredicateToFilterVisitor( + @Nullable String timeColumnName, ZoneId zoneId, TimeUnit currPrecision) { this.timeColumnName = timeColumnName; - this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(); + this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java index b9ab2109a15cd..7368c5d28c2c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; @@ -36,20 +37,31 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; +import java.time.ZoneId; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; /** The caller must make sure that the Expression only contains valid time predicate */ public class ConvertPredicateToTimeFilterVisitor extends PredicateVisitor { + private final ZoneId zoneId; + private final TimeUnit currPrecision; + + public ConvertPredicateToTimeFilterVisitor(ZoneId zoneId, TimeUnit currPrecision) { + this.zoneId = zoneId; + this.currPrecision = currPrecision; + } @Override protected Filter visitInPredicate(InPredicate node, Void context) { @@ -109,7 +121,7 @@ protected Filter visitNotExpression(NotExpression node, Void context) { @Override protected Filter visitComparisonExpression(ComparisonExpression node, Void context) { long value; - if (node.getLeft() instanceof LongLiteral) { + if (node.getRight() instanceof SymbolReference) { value = getLongValue(node.getLeft()); switch (node.getOperator()) { case EQUAL: @@ -127,7 +139,7 @@ protected Filter visitComparisonExpression(ComparisonExpression node, Void conte default: throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); } - } else if (node.getRight() instanceof LongLiteral) { + } else if (node.getLeft() instanceof SymbolReference) { value = getLongValue(node.getRight()); switch (node.getOperator()) { case EQUAL: @@ -145,9 +157,51 @@ protected Filter visitComparisonExpression(ComparisonExpression node, Void conte default: throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); } + } else if (node.getRight() instanceof Extract) { + Extract extract = (Extract) node.getRight(); + value = getLongValue(node.getLeft()); + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + switch (node.getOperator()) { + case EQUAL: + return TimeFilterApi.extractTimeEq(value, field, zoneId, currPrecision); + case NOT_EQUAL: + return TimeFilterApi.extractTimeNotEq(value, field, zoneId, currPrecision); + case GREATER_THAN: + return TimeFilterApi.extractTimeLt(value, field, zoneId, currPrecision); + case GREATER_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeLtEq(value, field, zoneId, currPrecision); + case LESS_THAN: + return TimeFilterApi.extractTimeGt(value, field, zoneId, currPrecision); + case LESS_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeGtEq(value, field, zoneId, currPrecision); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } + } else if (node.getLeft() instanceof Extract) { + Extract extract = (Extract) node.getLeft(); + value = getLongValue(node.getRight()); + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + switch (node.getOperator()) { + case EQUAL: + return TimeFilterApi.extractTimeEq(value, field, zoneId, currPrecision); + case NOT_EQUAL: + return TimeFilterApi.extractTimeNotEq(value, field, zoneId, currPrecision); + case GREATER_THAN: + return TimeFilterApi.extractTimeGt(value, field, zoneId, currPrecision); + case GREATER_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeGtEq(value, field, zoneId, currPrecision); + case LESS_THAN: + return TimeFilterApi.extractTimeLt(value, field, zoneId, currPrecision); + case LESS_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeLtEq(value, field, zoneId, currPrecision); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } } else { throw new IllegalStateException( - "Either left or right operand of Time ComparisonExpression should be LongLiteral"); + "Either left or right operand of ComparisonExpression should have time column."); } } @@ -218,6 +272,26 @@ protected Filter visitBetweenPredicate(BetweenPredicate node, Void context) { value >= minValue, String.format("Predicate [%s] should be simplified in previous step", node)); return TimeFilterApi.gtEq(value); + } else if (firstExpression instanceof Extract) { + long minValue = getLongValue(secondExpression); + long maxValue = getLongValue(thirdExpression); + Extract extract = (Extract) firstExpression; + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + + if (minValue == maxValue) { + return TimeFilterApi.extractTimeEq(minValue, field, zoneId, currPrecision); + } + return FilterFactory.and( + ImmutableList.of( + TimeFilterApi.extractTimeGtEq(minValue, field, zoneId, currPrecision), + TimeFilterApi.extractTimeLtEq(maxValue, field, zoneId, currPrecision))); + } else if (secondExpression instanceof Extract) { + throw new IllegalStateException( + "Should not reach here before GlobalTimePredicateExtractVisitor support Extract push-down in second child"); + } else if (thirdExpression instanceof Extract) { + throw new IllegalStateException( + "Should not reach here before GlobalTimePredicateExtractVisitor support Extract push-down in third child"); } else { throw new IllegalStateException( "Three operand of between expression should have time column."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java index 579f15628765d..b0764213dfe57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java @@ -41,17 +41,23 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; public class PredicateCombineIntoTableScanChecker extends PredicateVisitor { private final Set measurementColumns; + private final String timeColumnName; - public static boolean check(Set measurementColumns, Expression expression) { - return new PredicateCombineIntoTableScanChecker(measurementColumns).process(expression); + public static boolean check( + Set measurementColumns, String timeColumnName, Expression expression) { + return new PredicateCombineIntoTableScanChecker(measurementColumns, timeColumnName) + .process(expression); } - public PredicateCombineIntoTableScanChecker(Set measurementColumns) { + public PredicateCombineIntoTableScanChecker( + Set measurementColumns, String timeColumnName) { this.measurementColumns = measurementColumns; + this.timeColumnName = timeColumnName; } @Override @@ -115,14 +121,19 @@ protected Boolean visitLogicalExpression(LogicalExpression node, Void context) { @Override protected Boolean visitComparisonExpression(ComparisonExpression node, Void context) { return (isMeasurementColumn(node.getLeft()) && isLiteral(node.getRight())) - || (isMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())); + || (isMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())) + || (isExtractTimeColumn(node.getLeft(), timeColumnName) && isLiteral(node.getRight())) + || (isExtractTimeColumn(node.getRight(), timeColumnName) && isLiteral(node.getLeft())); } @Override protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { return (isMeasurementColumn(node.getValue()) - && isLiteral(node.getMin()) - && isLiteral(node.getMax())); + && isLiteral(node.getMin()) + && isLiteral(node.getMax())) + || (isExtractTimeColumn(node.getValue(), timeColumnName) + && isLiteral(node.getMin()) + && isLiteral(node.getMax())); // TODO After Constant-Folding introduced /*|| (isLiteral(node.getValue()) && isMeasurementColumn(node.getMin()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java index f168eb50d8309..418d19fdd66d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; @@ -197,7 +198,8 @@ protected Pair visitBetweenPredicate( Expression thirdExpression = node.getMax(); boolean isTimeFilter = false; - if (isTimeColumn(firstExpression, context.timeColumnName)) { + if (isTimeColumn(firstExpression, context.timeColumnName) + || isExtractTimeColumn(firstExpression, context.timeColumnName)) { isTimeFilter = checkBetweenConstantSatisfy(secondExpression, thirdExpression); } // TODO After Constant-Folding introduced @@ -269,9 +271,18 @@ public static boolean isTimeColumn(Expression e, String timeColumnName) { && ((SymbolReference) e).getName().equalsIgnoreCase(timeColumnName); } + public static boolean isExtractTimeColumn(Expression e, String timeColumnName) { + return e instanceof Extract + && ((SymbolReference) ((Extract) e).getExpression()) + .getName() + .equalsIgnoreCase(timeColumnName); + } + private static boolean checkIsTimeFilter( Expression timeExpression, String timeColumnName, Expression valueExpression) { - return isTimeColumn(timeExpression, timeColumnName) && valueExpression instanceof LongLiteral; + return (isTimeColumn(timeExpression, timeColumnName) + || isExtractTimeColumn(timeExpression, timeColumnName)) + && valueExpression instanceof LongLiteral; } private static boolean checkBetweenConstantSatisfy(Expression e1, Expression e2) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index f6be45cd2aca2..4284c37a03a07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -68,6 +68,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -530,7 +531,7 @@ private SplitExpression splitPredicate(DeviceTableScanNode node, Expression pred if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, expression)) { metadataExpressions.add(expression); } else if (PredicateCombineIntoTableScanChecker.check( - measurementColumnNames, expression)) { + measurementColumnNames, timeColumnName, expression)) { expressionsCanPushDown.add(expression); } else { expressionsCannotPushDown.add(expression); @@ -543,7 +544,8 @@ private SplitExpression splitPredicate(DeviceTableScanNode node, Expression pred if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, predicate)) { metadataExpressions.add(predicate); - } else if (PredicateCombineIntoTableScanChecker.check(measurementColumnNames, predicate)) { + } else if (PredicateCombineIntoTableScanChecker.check( + measurementColumnNames, timeColumnName, predicate)) { expressionsCanPushDown.add(predicate); } else { expressionsCannotPushDown.add(predicate); @@ -616,7 +618,12 @@ private void getDeviceEntriesWithDataPartitions( final Filter timeFilter = tableScanNode .getTimePredicate() - .map(value -> value.accept(new ConvertPredicateToTimeFilterVisitor(), null)) + .map( + value -> + value.accept( + new ConvertPredicateToTimeFilterVisitor( + queryContext.getZoneId(), TimestampPrecisionUtils.currPrecision), + null)) .orElse(null); tableScanNode.setTimeFilter(timeFilter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index e1114bbe4491d..b413ae5ff372f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -87,7 +87,6 @@ public static long correctPrecision(long millis) { } public static final Function CAST_TIMESTAMP_TO_MS; - public static final Function EXTRACT_TIMESTAMP_MS_PART; public static final Function EXTRACT_TIMESTAMP_US_PART; public static final Function EXTRACT_TIMESTAMP_NS_PART; @@ -97,25 +96,22 @@ public static long correctPrecision(long millis) { case "us": case "microsecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000; - - EXTRACT_TIMESTAMP_MS_PART = timestamp -> (timestamp % 1000_000) / 1000; - EXTRACT_TIMESTAMP_US_PART = timestamp -> timestamp % 1000; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000_000L) / 1000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> Math.floorMod(timestamp, 1000L); EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; case "ns": case "nanosecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000000; - - EXTRACT_TIMESTAMP_MS_PART = timestamp -> (timestamp % 1000_000_000) / 1000_000; - EXTRACT_TIMESTAMP_US_PART = timestamp -> (timestamp % 1000_000) / 1000; - EXTRACT_TIMESTAMP_NS_PART = timestamp -> timestamp % 1000; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000_000_000L) / 1000_000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> Math.floorMod(timestamp, 1000_000L) / 1000; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> Math.floorMod(timestamp, 1000L); break; case "ms": case "millisecond": default: CAST_TIMESTAMP_TO_MS = timestamp -> timestamp; - - EXTRACT_TIMESTAMP_MS_PART = timestamp -> timestamp % 1000; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000L); EXTRACT_TIMESTAMP_US_PART = timestamp -> 0L; EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; diff --git a/pom.xml b/pom.xml index 0f2cc724f15d0..f855bf8f7a9e7 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ 0.14.1 1.9 1.5.6-3 - 2.1.0-250616-SNAPSHOT + 2.1.0-SNAPSHOT