diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 3bd2a17e3f8ea..7e4d69f9b1bd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -229,7 +229,24 @@ public static void tableResultSetEqualTest( expectedRetArray, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, - database); + database, + "+00:00"); + } + + public static void tableResultSetEqualTest( + String sql, + String timeZone, + String[] expectedHeader, + String[] expectedRetArray, + String database) { + tableResultSetEqualTest( + sql, + expectedHeader, + expectedRetArray, + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + database, + timeZone); } public static void tableResultSetEqualTest( @@ -239,9 +256,21 @@ public static void tableResultSetEqualTest( String userName, String password, String database) { + tableResultSetEqualTest( + sql, expectedHeader, expectedRetArray, userName, password, database, "+00:00"); + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password, + String database, + String timeZone) { try (Connection connection = EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { - connection.setClientInfo("time_zone", "+00:00"); + connection.setClientInfo("time_zone", timeZone); try (Statement statement = connection.createStatement()) { statement.execute("use " + database); try (ResultSet resultSet = statement.executeQuery(sql)) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable2IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable2IT.java new file mode 100644 index 0000000000000..7986961db36a7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable2IT.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.extract; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBExtractTable2IT extends IoTDBExtractTableIT { + public IoTDBExtractTable2IT() { + this.decimal = "123456"; + } + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("us"); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @Test + public void extractUsNsTest() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2"}; + String[] retArray = + new String[] { + getTimeStrUTC("2025-07-08T01:18:51") + ",456,0,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + getTimeStrUTC8("2025-07-08T09:18:51") + ",456,0,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable3IT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable3IT.java new file mode 100644 index 0000000000000..0d362dc08200b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTable3IT.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.extract; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBExtractTable3IT extends IoTDBExtractTableIT { + public IoTDBExtractTable3IT() { + this.decimal = "123456789"; + } + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecision("ns"); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @Test + public void extractUsNsTest() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2"}; + String[] retArray = + new String[] { + getTimeStrUTC("2025-07-08T01:18:51") + ",456,789,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + getTimeStrUTC8("2025-07-08T09:18:51") + ",456,789,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java new file mode 100644 index 0000000000000..8b065b1571f46 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.extract; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBExtractTableIT { + protected static final String DATABASE_NAME = "test"; + protected static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, ts TIMESTAMP FIELD, s1 INT32 FIELD)", + "INSERT INTO table1(time,device_id,ts,s1) values(2025/07/08 01:18:51.123456789,'d1',2025/07/08 01:18:51.123456789,1)", + "INSERT INTO table1(time,device_id,ts,s1) values(2025/07/08 02:18:51.123456789,'d1',2025/07/08 02:18:51.123456789,2)", + "INSERT INTO table1(time,device_id,ts,s1) values(2025/07/09 00:17:50.123456789,'d1',2025/07/09 00:17:50.123456789,3)", + "FLUSH" + }; + protected String decimal = "123"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void extractTest() { + + String[] expectedHeader = + new String[] { + "time", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", + "_col10", "_col11" + }; + String[] retArray = + new String[] {getTimeStrUTC("2025-07-08T01:18:51") + ",2025,3,7,27,8,2,189,1,18,51,123,"}; + tableResultSetEqualTest( + "SELECT time, extract(year from time),extract(quarter from time),extract(month from time),extract(week from time),extract(day from time)," + + "extract(dow from time),extract(doy from time),extract(hour from time),extract(minute from time),extract(second from time),extract(ms from time)" + + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] {getTimeStrUTC8("2025-07-08T09:18:51") + ",2025,3,7,27,8,2,189,9,18,51,123,"}; + tableResultSetEqualTest( + "SELECT time,extract(year from time),extract(quarter from time),extract(month from time),extract(week from time),extract(day from time)," + + "extract(dow from time),extract(doy from time),extract(hour from time),extract(minute from time),extract(second from time),extract(ms from time)" + + " FROM table1 order by time limit 1", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extractUsNsTest() { + String[] expectedHeader = new String[] {"time", "_col1", "_col2"}; + String[] retArray = + new String[] { + getTimeStrUTC("2025-07-08T01:18:51") + ",0,0,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + getTimeStrUTC8("2025-07-08T09:18:51") + ",0,0,", + }; + tableResultSetEqualTest( + "SELECT time, extract(us from time),extract(ns from time)" + + " FROM table1 order by time limit 1", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void extractTimeFilterPushDownTest() { + String[] expectedHeader = new String[] {"time", "s1"}; + String[] retArray = + new String[] { + getTimeStrUTC("2025-07-08T02:18:51") + ",2,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) > 1", + expectedHeader, + retArray, + DATABASE_NAME); + // verify the pushdown result is same with non-pushdown + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) > 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) >= 2", + expectedHeader, + retArray, + DATABASE_NAME); + // verify the pushdown result is same with non-pushdown + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) >= 2", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = + new String[] { + getTimeStrUTC8("2025-07-08T10:18:51") + ",2,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) > 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) > 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) >= 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) >= 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + + expectedHeader = new String[] {"time", "s1"}; + retArray = + new String[] { + getTimeStrUTC("2025-07-09T00:17:50") + ",3,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) < 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) < 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) <= 0", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) <= 0", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) = 0", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) = 0", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = + new String[] { + getTimeStrUTC8("2025-07-09T08:17:50") + ",3,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) < 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) < 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) <= 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) <= 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) = 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) = 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + + retArray = + new String[] { + getTimeStrUTC("2025-07-08T01:18:51") + ",1,", + getTimeStrUTC("2025-07-08T02:18:51") + ",2,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) != 0", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) != 0", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = + new String[] { + getTimeStrUTC8("2025-07-08T09:18:51") + ",1,", + getTimeStrUTC8("2025-07-08T10:18:51") + ",2,", + }; + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) != 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) != 8", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testExtractFromComplexExpression() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = new String[] {"0,"}; + tableResultSetEqualTest( + "SELECT extract(hour from cast(s1 AS TIMESTAMP))" + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testExtractFromConstant() { + String[] expectedHeader = new String[] {"_col0"}; + String[] retArray = new String[] {"1,"}; + tableResultSetEqualTest( + "SELECT extract(hour from 2025/07/08 01:18:51)" + " FROM table1 order by time limit 1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + protected String getTimeStrUTC(String time) { + return time + "." + decimal + 'Z'; + } + + protected String getTimeStrUTC8(String time) { + return time + "." + decimal + "+08:00"; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 8b9a2a274a14c..8fc462bbd4285 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp; @@ -235,7 +236,10 @@ private FragmentInstanceContext( this.sessionInfo = sessionInfo; this.dataRegion = dataRegion; this.globalTimeFilter = - globalTimePredicate == null ? null : globalTimePredicate.convertPredicateToTimeFilter(); + globalTimePredicate == null + ? null + : globalTimePredicate.convertPredicateToTimeFilter( + sessionInfo.getZoneId(), TimestampPrecisionUtils.currPrecision); this.dataNodeQueryContextMap = dataNodeQueryContextMap; this.dataNodeQueryContext = dataNodeQueryContextMap.get(id.getQueryId()); this.memoryReservationManager = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index d0b7e35e94ce4..0e915ab258cb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DecimalLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; @@ -122,6 +123,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.EndsWith2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.EndsWithColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ExpColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ExtractTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.FloorColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.FormatColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.LTrim2ColumnTransformer; @@ -343,6 +345,27 @@ protected ColumnTransformer visitCast(Cast node, Context context) { return getColumnTransformerFromCacheAndAddReferenceCount(node, context); } + @Override + protected ColumnTransformer visitExtract(Extract node, Context context) { + if (!context.cache.containsKey(node)) { + if (context.hasSeen.containsKey(node)) { + ColumnTransformer columnTransformer = context.hasSeen.get(node); + appendIdentityColumnTransformer( + node, + columnTransformer.getType(), + getTSDataType(columnTransformer.getType()), + context, + columnTransformer); + } else { + ColumnTransformer child = this.process(node.getExpression(), context); + context.cache.put( + node, + new ExtractTransformer(INT64, child, node.getField(), context.sessionInfo.getZoneId())); + } + } + return getColumnTransformerFromCacheAndAddReferenceCount(node, context); + } + @Override protected ColumnTransformer visitBooleanLiteral(BooleanLiteral node, Context context) { ColumnTransformer res = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java index 9fb41fc152d1c..cf4b3a8294c25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class PredicateUtils { @@ -282,13 +283,15 @@ public static Filter convertPredicateToTimeFilter(Expression predicate) { } public static Filter convertPredicateToTimeFilter( - org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate) { + org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate, + ZoneId zoneId, + TimeUnit currPrecision) { if (predicate == null) { return null; } return predicate.accept( new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate - .ConvertPredicateToTimeFilterVisitor(), + .ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision), null); } @@ -311,13 +314,15 @@ public static Filter convertPredicateToFilter( org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression predicate, Map measurementColumnsIndexMap, Map schemaMap, - String timeColumnName) { + String timeColumnName, + ZoneId zoneId, + TimeUnit currPrecision) { if (predicate == null) { return null; } return predicate.accept( new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate - .ConvertPredicateToFilterVisitor(timeColumnName), + .ConvertPredicateToFilterVisitor(timeColumnName, zoneId, currPrecision), new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate .ConvertPredicateToFilterVisitor.Context(measurementColumnsIndexMap, schemaMap)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 75e2275275b07..a4fd11867dc98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -232,6 +232,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.utils.datastructure.SortKey; import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider; @@ -614,7 +615,11 @@ private void calculateSeriesScanOptionsList() { Filter timeFilter = null; if (node.getTimePredicate().isPresent()) { Expression timePredicate = node.getTimePredicate().get(); - timeFilter = timePredicate.accept(new ConvertPredicateToTimeFilterVisitor(), null); + timeFilter = + timePredicate.accept( + new ConvertPredicateToTimeFilterVisitor( + context.getZoneId(), TimestampPrecisionUtils.currPrecision), + null); context .getDriverContext() .getFragmentInstanceContext() @@ -654,7 +659,9 @@ private void calculateSeriesScanOptionsList() { pushDownPredicateForCurrentMeasurement, Collections.singletonMap(symbol.getName(), 0), commonParameter.columnSchemaMap, - commonParameter.timeColumnName)); + commonParameter.timeColumnName, + context.getZoneId(), + TimestampPrecisionUtils.currPrecision)); } if (isSingleColumn || (pushDownOffsetAndLimitToLeftChildSeriesScanOperator @@ -1235,7 +1242,11 @@ private SeriesScanOptions.Builder getSeriesScanOptionsBuilder( LocalExecutionPlanContext context, @NotNull Expression timePredicate) { SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); - Filter timeFilter = timePredicate.accept(new ConvertPredicateToTimeFilterVisitor(), null); + Filter timeFilter = + timePredicate.accept( + new ConvertPredicateToTimeFilterVisitor( + context.getZoneId(), TimestampPrecisionUtils.currPrecision), + null); context.getDriverContext().getFragmentInstanceContext().setTimeFilterForTableModel(timeFilter); // time filter may be stateful, so we need to copy it scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); @@ -3113,7 +3124,12 @@ private SeriesScanOptions buildSeriesScanOptions( if (pushDownPredicate != null) { scanOptionsBuilder.withPushDownFilter( convertPredicateToFilter( - pushDownPredicate, measurementColumnsIndexMap, columnSchemaMap, timeColumnName)); + pushDownPredicate, + measurementColumnsIndexMap, + columnSchemaMap, + timeColumnName, + context.getZoneId(), + TimestampPrecisionUtils.currPrecision)); } return scanOptionsBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java index b299c04e6d7a8..dcdb79bf79e78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java @@ -27,7 +27,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.time.ZoneId; import java.util.Objects; +import java.util.concurrent.TimeUnit; public class TableModelTimePredicate implements TimePredicate { @@ -44,8 +46,8 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public Filter convertPredicateToTimeFilter() { - return PredicateUtils.convertPredicateToTimeFilter(timePredicate); + public Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision) { + return PredicateUtils.convertPredicateToTimeFilter(timePredicate, zoneId, currPrecision); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java index 440c34a65aab4..d424aa82d09a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java @@ -27,12 +27,14 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; public interface TimePredicate { void serialize(DataOutputStream stream) throws IOException; - Filter convertPredicateToTimeFilter(); + Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision); static TimePredicate deserialize(ByteBuffer byteBuffer) { // 0 for tree model, 1 for table model diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java index 5cc00125aa794..132077d801b61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java @@ -27,7 +27,9 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.time.ZoneId; import java.util.Objects; +import java.util.concurrent.TimeUnit; public class TreeModelTimePredicate implements TimePredicate { @@ -44,7 +46,7 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public Filter convertPredicateToTimeFilter() { + public Filter convertPredicateToTimeFilter(ZoneId zoneId, TimeUnit currPrecision) { return PredicateUtils.convertPredicateToTimeFilter(timePredicate); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java index 4e3e12f723ab0..93f7847c5faaf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationAnalyzer.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; @@ -212,6 +213,11 @@ protected Boolean visitNullIfExpression(NullIfExpression node, Void context) { return process(node.getFirst(), context) && process(node.getSecond(), context); } + @Override + protected Boolean visitExtract(Extract node, Void context) { + return process(node.getExpression(), context); + } + @Override protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { return process(node.getMin(), context) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 13923444e0c5c..6157612b42382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -61,6 +61,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -106,6 +107,7 @@ import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; import javax.annotation.Nullable; @@ -1555,6 +1557,22 @@ protected Type visitParameter(Parameter node, StackableAstVisitorContext context) { + if (node.getExpression() instanceof LongLiteral) { + // Don't visit child here to avoid setting its Type to INT32 + setExpressionType(node.getExpression(), INT64); + } else { + Type type = process(node.getExpression(), context); + + if (!(type instanceof TimestampType)) { + throw new SemanticException(String.format("Cannot extract from %s", type)); + } + } + + return setExpressionType(node, INT64); + } + @Override protected Type visitBetweenPredicate( BetweenPredicate node, StackableAstVisitorContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index a0afd0453afb8..2ec3676acaee1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -95,6 +95,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; @@ -1996,6 +1997,22 @@ protected List visitRow(Row node, Scope context) { String.format("%s are not supported now", node.getClass().getSimpleName())); } + @Override + protected List visitExtract(Extract node, Scope context) { + List childResult = process(node.getExpression(), context); + if (expandedExpressions == null) { + // no Columns need to be expanded + return Collections.singletonList(node); + } + + ImmutableList.Builder resultBuilder = new ImmutableList.Builder<>(); + for (Expression expression : childResult) { + resultBuilder.add(new Extract(expression, node.getField())); + } + + return resultBuilder.build(); + } + @Override protected List visitSearchedCaseExpression( SearchedCaseExpression node, Scope context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index 1704ccab7e0a9..0c9f42b3e63e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -58,11 +58,13 @@ import javax.annotation.Nullable; +import java.time.ZoneId; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -78,9 +80,10 @@ public class ConvertPredicateToFilterVisitor @Nullable private final String timeColumnName; private final ConvertPredicateToTimeFilterVisitor timeFilterVisitor; - public ConvertPredicateToFilterVisitor(@Nullable String timeColumnName) { + public ConvertPredicateToFilterVisitor( + @Nullable String timeColumnName, ZoneId zoneId, TimeUnit currPrecision) { this.timeColumnName = timeColumnName; - this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(); + this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java index b9ab2109a15cd..7368c5d28c2c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; @@ -36,20 +37,31 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; +import java.time.ZoneId; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; /** The caller must make sure that the Expression only contains valid time predicate */ public class ConvertPredicateToTimeFilterVisitor extends PredicateVisitor { + private final ZoneId zoneId; + private final TimeUnit currPrecision; + + public ConvertPredicateToTimeFilterVisitor(ZoneId zoneId, TimeUnit currPrecision) { + this.zoneId = zoneId; + this.currPrecision = currPrecision; + } @Override protected Filter visitInPredicate(InPredicate node, Void context) { @@ -109,7 +121,7 @@ protected Filter visitNotExpression(NotExpression node, Void context) { @Override protected Filter visitComparisonExpression(ComparisonExpression node, Void context) { long value; - if (node.getLeft() instanceof LongLiteral) { + if (node.getRight() instanceof SymbolReference) { value = getLongValue(node.getLeft()); switch (node.getOperator()) { case EQUAL: @@ -127,7 +139,7 @@ protected Filter visitComparisonExpression(ComparisonExpression node, Void conte default: throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); } - } else if (node.getRight() instanceof LongLiteral) { + } else if (node.getLeft() instanceof SymbolReference) { value = getLongValue(node.getRight()); switch (node.getOperator()) { case EQUAL: @@ -145,9 +157,51 @@ protected Filter visitComparisonExpression(ComparisonExpression node, Void conte default: throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); } + } else if (node.getRight() instanceof Extract) { + Extract extract = (Extract) node.getRight(); + value = getLongValue(node.getLeft()); + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + switch (node.getOperator()) { + case EQUAL: + return TimeFilterApi.extractTimeEq(value, field, zoneId, currPrecision); + case NOT_EQUAL: + return TimeFilterApi.extractTimeNotEq(value, field, zoneId, currPrecision); + case GREATER_THAN: + return TimeFilterApi.extractTimeLt(value, field, zoneId, currPrecision); + case GREATER_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeLtEq(value, field, zoneId, currPrecision); + case LESS_THAN: + return TimeFilterApi.extractTimeGt(value, field, zoneId, currPrecision); + case LESS_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeGtEq(value, field, zoneId, currPrecision); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } + } else if (node.getLeft() instanceof Extract) { + Extract extract = (Extract) node.getLeft(); + value = getLongValue(node.getRight()); + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + switch (node.getOperator()) { + case EQUAL: + return TimeFilterApi.extractTimeEq(value, field, zoneId, currPrecision); + case NOT_EQUAL: + return TimeFilterApi.extractTimeNotEq(value, field, zoneId, currPrecision); + case GREATER_THAN: + return TimeFilterApi.extractTimeGt(value, field, zoneId, currPrecision); + case GREATER_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeGtEq(value, field, zoneId, currPrecision); + case LESS_THAN: + return TimeFilterApi.extractTimeLt(value, field, zoneId, currPrecision); + case LESS_THAN_OR_EQUAL: + return TimeFilterApi.extractTimeLtEq(value, field, zoneId, currPrecision); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } } else { throw new IllegalStateException( - "Either left or right operand of Time ComparisonExpression should be LongLiteral"); + "Either left or right operand of ComparisonExpression should have time column."); } } @@ -218,6 +272,26 @@ protected Filter visitBetweenPredicate(BetweenPredicate node, Void context) { value >= minValue, String.format("Predicate [%s] should be simplified in previous step", node)); return TimeFilterApi.gtEq(value); + } else if (firstExpression instanceof Extract) { + long minValue = getLongValue(secondExpression); + long maxValue = getLongValue(thirdExpression); + Extract extract = (Extract) firstExpression; + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + + if (minValue == maxValue) { + return TimeFilterApi.extractTimeEq(minValue, field, zoneId, currPrecision); + } + return FilterFactory.and( + ImmutableList.of( + TimeFilterApi.extractTimeGtEq(minValue, field, zoneId, currPrecision), + TimeFilterApi.extractTimeLtEq(maxValue, field, zoneId, currPrecision))); + } else if (secondExpression instanceof Extract) { + throw new IllegalStateException( + "Should not reach here before GlobalTimePredicateExtractVisitor support Extract push-down in second child"); + } else if (thirdExpression instanceof Extract) { + throw new IllegalStateException( + "Should not reach here before GlobalTimePredicateExtractVisitor support Extract push-down in third child"); } else { throw new IllegalStateException( "Three operand of between expression should have time column."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java index 579f15628765d..b0764213dfe57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java @@ -41,17 +41,23 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; public class PredicateCombineIntoTableScanChecker extends PredicateVisitor { private final Set measurementColumns; + private final String timeColumnName; - public static boolean check(Set measurementColumns, Expression expression) { - return new PredicateCombineIntoTableScanChecker(measurementColumns).process(expression); + public static boolean check( + Set measurementColumns, String timeColumnName, Expression expression) { + return new PredicateCombineIntoTableScanChecker(measurementColumns, timeColumnName) + .process(expression); } - public PredicateCombineIntoTableScanChecker(Set measurementColumns) { + public PredicateCombineIntoTableScanChecker( + Set measurementColumns, String timeColumnName) { this.measurementColumns = measurementColumns; + this.timeColumnName = timeColumnName; } @Override @@ -115,14 +121,19 @@ protected Boolean visitLogicalExpression(LogicalExpression node, Void context) { @Override protected Boolean visitComparisonExpression(ComparisonExpression node, Void context) { return (isMeasurementColumn(node.getLeft()) && isLiteral(node.getRight())) - || (isMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())); + || (isMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())) + || (isExtractTimeColumn(node.getLeft(), timeColumnName) && isLiteral(node.getRight())) + || (isExtractTimeColumn(node.getRight(), timeColumnName) && isLiteral(node.getLeft())); } @Override protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { return (isMeasurementColumn(node.getValue()) - && isLiteral(node.getMin()) - && isLiteral(node.getMax())); + && isLiteral(node.getMin()) + && isLiteral(node.getMax())) + || (isExtractTimeColumn(node.getValue(), timeColumnName) + && isLiteral(node.getMin()) + && isLiteral(node.getMax())); // TODO After Constant-Folding introduced /*|| (isLiteral(node.getValue()) && isMeasurementColumn(node.getMin()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java index 208da696a0c13..ae17e5eab7bf3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; @@ -80,6 +81,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression.Sign.MINUS; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression.Sign.PLUS; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; +import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ExtractTransformer.constructEvaluateFunction; public class IrExpressionInterpreter { @@ -916,6 +918,23 @@ public Object visitCast(Cast node, Object context) { } } + @Override + protected Object visitExtract(Extract node, Object context) { + Object value = processWithExceptionHandling(node.getExpression(), context); + if (value == null) { + return null; + } + + // if is Extract from constant, the constant must be INT64 type, so it will be Long after the + // process + if (value instanceof Long) { + return constructEvaluateFunction(node.getField(), session.getZoneId()).apply((Long) value); + } + + checkState(value instanceof Expression, "Value reach here must be Expression"); + return new Extract((Expression) value, node.getField()); + } + @Override protected Object visitExpression(Expression node, Object context) { throw new SemanticException("not yet implemented: " + node.getClass().getName()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index 1ecf2dad77dc7..b92e50041e3b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentUser; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; @@ -306,6 +307,12 @@ protected Type visitArithmeticUnary(ArithmeticUnaryExpression node, Context cont return setExpressionType(node, process(node.getValue(), context)); } + @Override + protected Type visitExtract(Extract node, Context context) { + process(node.getExpression(), context); + return setExpressionType(node, INT64); + } + @Override protected Type visitArithmeticBinary(ArithmeticBinaryExpression node, Context context) { ImmutableList.Builder argumentTypes = ImmutableList.builder(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java index d232101d6037c..159655d4255b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionRewriter.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -174,6 +175,11 @@ public Expression rewriteDereferenceExpression( return rewriteExpression(node, context, treeRewriter); } + public Expression rewriteExtract( + Extract node, C context, ExpressionTreeRewriter treeRewriter) { + return rewriteExpression(node, context, treeRewriter); + } + // public Expression rewriteBindExpression(BindExpression node, C context, // ExpressionTreeRewriter treeRewriter) { // return rewriteExpression(node, context, treeRewriter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index 585829e25401d..d84ac593306fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -625,6 +626,25 @@ public Expression visitDereferenceExpression(DereferenceExpression node, Context return node; } + @Override + protected Expression visitExtract(Extract node, Context context) { + if (!context.isDefaultRewrite()) { + Expression result = + rewriter.rewriteExtract(node, context.get(), ExpressionTreeRewriter.this); + if (result != null) { + return result; + } + } + + Expression expression = rewrite(node.getExpression(), context.get()); + + if (node.getExpression() != expression) { + return new Extract(expression, node.getField()); + } + + return node; + } + @Override public Expression visitCast(Cast node, Context context) { if (!context.isDefaultRewrite()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java index f168eb50d8309..09d040b12f344 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; @@ -197,7 +198,8 @@ protected Pair visitBetweenPredicate( Expression thirdExpression = node.getMax(); boolean isTimeFilter = false; - if (isTimeColumn(firstExpression, context.timeColumnName)) { + if (isTimeColumn(firstExpression, context.timeColumnName) + || isExtractTimeColumn(firstExpression, context.timeColumnName)) { isTimeFilter = checkBetweenConstantSatisfy(secondExpression, thirdExpression); } // TODO After Constant-Folding introduced @@ -269,9 +271,19 @@ public static boolean isTimeColumn(Expression e, String timeColumnName) { && ((SymbolReference) e).getName().equalsIgnoreCase(timeColumnName); } + public static boolean isExtractTimeColumn(Expression e, String timeColumnName) { + return e instanceof Extract + && ((Extract) e).getExpression() instanceof SymbolReference + && ((SymbolReference) ((Extract) e).getExpression()) + .getName() + .equalsIgnoreCase(timeColumnName); + } + private static boolean checkIsTimeFilter( Expression timeExpression, String timeColumnName, Expression valueExpression) { - return isTimeColumn(timeExpression, timeColumnName) && valueExpression instanceof LongLiteral; + return (isTimeColumn(timeExpression, timeColumnName) + || isExtractTimeColumn(timeExpression, timeColumnName)) + && valueExpression instanceof LongLiteral; } private static boolean checkBetweenConstantSatisfy(Expression e1, Expression e2) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index f6be45cd2aca2..4284c37a03a07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -68,6 +68,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -530,7 +531,7 @@ private SplitExpression splitPredicate(DeviceTableScanNode node, Expression pred if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, expression)) { metadataExpressions.add(expression); } else if (PredicateCombineIntoTableScanChecker.check( - measurementColumnNames, expression)) { + measurementColumnNames, timeColumnName, expression)) { expressionsCanPushDown.add(expression); } else { expressionsCannotPushDown.add(expression); @@ -543,7 +544,8 @@ private SplitExpression splitPredicate(DeviceTableScanNode node, Expression pred if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, predicate)) { metadataExpressions.add(predicate); - } else if (PredicateCombineIntoTableScanChecker.check(measurementColumnNames, predicate)) { + } else if (PredicateCombineIntoTableScanChecker.check( + measurementColumnNames, timeColumnName, predicate)) { expressionsCanPushDown.add(predicate); } else { expressionsCannotPushDown.add(predicate); @@ -616,7 +618,12 @@ private void getDeviceEntriesWithDataPartitions( final Filter timeFilter = tableScanNode .getTimePredicate() - .map(value -> value.accept(new ConvertPredicateToTimeFilterVisitor(), null)) + .map( + value -> + value.accept( + new ConvertPredicateToTimeFilterVisitor( + queryContext.getZoneId(), TimestampPrecisionUtils.currPrecision), + null)) .orElse(null); tableScanNode.setTimeFilter(timeFilter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 3259e6fbc025b..80b5642fd6e0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -229,6 +229,10 @@ protected R visitNotExpression(NotExpression node, C context) { return visitExpression(node, context); } + protected R visitExtract(Extract node, C context) { + return visitExpression(node, context); + } + protected R visitWindowDefinition(WindowDefinition node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java index 164c86d425ca1..f22af4b020e1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java @@ -20,6 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; public abstract class DefaultTraversalVisitor extends AstVisitor { + @Override + protected Void visitExtract(Extract node, C context) { + process(node.getExpression(), context); + return null; + } @Override protected Void visitArithmeticBinary(ArithmeticBinaryExpression node, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java index 53eadfb149cc9..f4bc64e25523e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java @@ -161,6 +161,9 @@ public static Expression deserialize(ByteBuffer byteBuffer) { case 31: expression = new Row(byteBuffer); break; + case 32: + expression = new Extract(byteBuffer); + break; default: throw new IllegalArgumentException("Invalid expression type: " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java new file mode 100644 index 0000000000000..e9e04073ad26c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Extract.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.Immutable; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +@Immutable +public class Extract extends Expression { + private final Expression expression; + private final Field field; + + public enum Field { + YEAR, + QUARTER, + MONTH, + WEEK, + DAY, + DAY_OF_MONTH, + DAY_OF_WEEK, + DOW, + DAY_OF_YEAR, + DOY, + HOUR, + MINUTE, + SECOND, + MS, + US, + NS + } + + public Extract(Expression expression, Field field) { + this(null, expression, field); + } + + public Extract(NodeLocation location, Expression expression, Field field) { + super(location); + requireNonNull(expression, "expression is null"); + requireNonNull(field, "field is null"); + + this.expression = expression; + this.field = field; + } + + public Extract(ByteBuffer byteBuffer) { + super(null); + expression = deserialize(byteBuffer); + field = Field.values()[ReadWriteIOUtils.readInt(byteBuffer)]; + } + + @Override + protected void serialize(ByteBuffer byteBuffer) { + serialize(expression, byteBuffer); + ReadWriteIOUtils.write(field.ordinal(), byteBuffer); + } + + @Override + protected void serialize(DataOutputStream stream) throws IOException { + serialize(expression, stream); + ReadWriteIOUtils.write(field.ordinal(), stream); + } + + @Override + public TableExpressionType getExpressionType() { + return TableExpressionType.EXTRACT; + } + + public Expression getExpression() { + return expression; + } + + public Field getField() { + return field; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitExtract(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(expression); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Extract that = (Extract) o; + return Objects.equals(expression, that.expression) && (field == that.field); + } + + @Override + public int hashCode() { + return Objects.hash(expression, field); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + Extract otherExtract = (Extract) other; + return field.equals(otherExtract.field); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java index 78ef4feb09e75..9d0b7ae0b4aa1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java @@ -50,7 +50,8 @@ public enum TableExpressionType { WHEN_CLAUSE((short) 28), CURRENT_DATABASE((short) 29), CURRENT_USER((short) 30), - ROW((short) 31); + ROW((short) 31), + EXTRACT((short) 32); TableExpressionType(short type) { this.type = type; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index df0d86bf69cff..a00ad658c7635 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -93,6 +93,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; @@ -285,6 +286,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Long.parseLong; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.iotdb.commons.schema.table.TsTable.TIME_COLUMN_NAME; @@ -3098,6 +3100,18 @@ public Node visitCurrentUser(RelationalSqlParser.CurrentUserContext ctx) { return new CurrentUser(getLocation(ctx)); } + @Override + public Node visitExtract(RelationalSqlParser.ExtractContext context) { + String fieldString = context.identifier().getText(); + Extract.Field field; + try { + field = Extract.Field.valueOf(fieldString.toUpperCase(ENGLISH)); + } catch (IllegalArgumentException e) { + throw parseError("Invalid EXTRACT field: " + fieldString, context); + } + return new Extract(getLocation(context), (Expression) visit(context.valueExpression()), field); + } + @Override public Node visitSubqueryExpression(RelationalSqlParser.SubqueryExpressionContext ctx) { return new SubqueryExpression(getLocation(ctx), (Query) visit(ctx.query())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index 47e00fb0fb69a..9f1afa6a5f38b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -184,6 +185,11 @@ protected String visitCurrentTime(CurrentTime node, Void context) { return builder.toString(); } + @Override + protected String visitExtract(Extract node, Void context) { + return "EXTRACT(" + node.getField() + " FROM " + process(node.getExpression(), context) + ")"; + } + @Override protected String visitBooleanLiteral(BooleanLiteral node, Void context) { return literalFormatter diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java new file mode 100644 index 0000000000000..76d4c1c7e10e1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ExtractTransformer.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.time.ZoneId; +import java.util.function.Function; + +import static java.time.temporal.ChronoField.ALIGNED_WEEK_OF_YEAR; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_MS_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_NS_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.EXTRACT_TIMESTAMP_US_PART; +import static org.apache.iotdb.db.utils.DateTimeUtils.convertToZonedDateTime; + +public class ExtractTransformer extends UnaryColumnTransformer { + private final Function evaluateFunction; + + public ExtractTransformer( + Type type, ColumnTransformer child, Extract.Field field, ZoneId zoneId) { + super(type, child); + this.evaluateFunction = constructEvaluateFunction(field, zoneId); + } + + public static Function constructEvaluateFunction(Extract.Field field, ZoneId zoneId) { + switch (field) { + case YEAR: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getYear(); + case QUARTER: + return timestamp -> (convertToZonedDateTime(timestamp, zoneId).getMonthValue() + 2L) / 3L; + case MONTH: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getMonthValue(); + case WEEK: + return timestamp -> convertToZonedDateTime(timestamp, zoneId).getLong(ALIGNED_WEEK_OF_YEAR); + case DAY: + case DAY_OF_MONTH: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getDayOfMonth(); + case DAY_OF_WEEK: + case DOW: + return timestamp -> + (long) convertToZonedDateTime(timestamp, zoneId).getDayOfWeek().getValue(); + case DAY_OF_YEAR: + case DOY: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getDayOfYear(); + case HOUR: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getHour(); + case MINUTE: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getMinute(); + case SECOND: + return timestamp -> (long) convertToZonedDateTime(timestamp, zoneId).getSecond(); + case MS: + return EXTRACT_TIMESTAMP_MS_PART; + case US: + return EXTRACT_TIMESTAMP_US_PART; + case NS: + return EXTRACT_TIMESTAMP_NS_PART; + default: + throw new UnsupportedOperationException("Unexpected extract field: " + field); + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (!column.isNull(i)) { + columnBuilder.writeLong(evaluateFunction.apply(column.getLong(i))); + } else { + columnBuilder.appendNull(); + } + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (selection[i] && !column.isNull(i)) { + columnBuilder.writeLong(evaluateFunction.apply(column.getLong(i))); + } else { + columnBuilder.appendNull(); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index a94bda7303387..b413ae5ff372f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -86,22 +86,34 @@ public static long correctPrecision(long millis) { } } - private static Function CAST_TIMESTAMP_TO_MS; + public static final Function CAST_TIMESTAMP_TO_MS; + public static final Function EXTRACT_TIMESTAMP_MS_PART; + public static final Function EXTRACT_TIMESTAMP_US_PART; + public static final Function EXTRACT_TIMESTAMP_NS_PART; static { switch (CommonDescriptor.getInstance().getConfig().getTimestampPrecision()) { case "us": case "microsecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000_000L) / 1000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> Math.floorMod(timestamp, 1000L); + EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; case "ns": case "nanosecond": CAST_TIMESTAMP_TO_MS = timestamp -> timestamp / 1000000; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000_000_000L) / 1000_000; + EXTRACT_TIMESTAMP_US_PART = timestamp -> Math.floorMod(timestamp, 1000_000L) / 1000; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> Math.floorMod(timestamp, 1000L); break; case "ms": case "millisecond": default: CAST_TIMESTAMP_TO_MS = timestamp -> timestamp; + EXTRACT_TIMESTAMP_MS_PART = timestamp -> Math.floorMod(timestamp, 1000L); + EXTRACT_TIMESTAMP_US_PART = timestamp -> 0L; + EXTRACT_TIMESTAMP_NS_PART = timestamp -> 0L; break; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java new file mode 100644 index 0000000000000..ce432bf42c4a0 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; + +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.assertAnalyzeSemanticException; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.expression; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; + +public class ExtractExpressionTest { + @Test + public void pushDownTest() { + PlanTester planTester = new PlanTester(); + + assertPlan( + planTester.createPlan("select * from table1 where extract(ms from time) > 5"), + output(tableScan("testdb.table1"))); + } + + @Test + public void constantFoldTest() { + PlanTester planTester = new PlanTester(); + assertPlan( + planTester.createPlan("select extract(hour from 2025/07/08 01:18:51) from table1"), + output( + project( + ImmutableMap.of("expr", expression(new LongLiteral("1"))), + tableScan("testdb.table1")))); + } + + @Test + public void exceptionTest() { + String errMsg = "Invalid EXTRACT field: s"; + // Wrong extract field + assertAnalyzeSemanticException("select * from table1 where extract(s from time) > 5", errMsg); + + errMsg = "Cannot extract from INT64"; + // Wrong extract input type + assertAnalyzeSemanticException("select * from table1 where extract(ms from s1) > 5", errMsg); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 1cb017af4ac01..8aa8ac380d261 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1126,6 +1126,7 @@ primaryExpression trimSource=valueExpression ')' #trim | TRIM '(' trimSource=valueExpression ',' trimChar=valueExpression ')' #trim | SUBSTRING '(' valueExpression FROM valueExpression (FOR valueExpression)? ')' #substring + | EXTRACT '(' identifier FROM valueExpression ')' #extract | DATE_BIN '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBin | DATE_BIN_GAPFILL '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBinGapFill | '(' expression ')' #parenthesizedExpression diff --git a/pom.xml b/pom.xml index 51320ee015893..21a7316c1ec83 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ 0.14.1 1.9 1.5.6-3 - 2.1.0-250709-SNAPSHOT + 2.2.0-250717-SNAPSHOT