diff --git a/example/session/src/main/java/org/apache/iotdb/ObjectExample.java b/example/session/src/main/java/org/apache/iotdb/ObjectExample.java new file mode 100644 index 0000000000000..3339d3aae5a4c --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/ObjectExample.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class ObjectExample { + private static final String LOCAL_URL = "127.0.0.1:6667"; + + public static void main(String[] args) { + + // don't specify database in constructor + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .username("root") + .password("root") + .build()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test1"); + session.executeNonQueryStatement("use test1"); + + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("tsfile_table", columnNameList, dataTypeList, columnTypeList, 1); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue( + rowIndex, + 4, + true, + 0, + Files.readAllBytes( + Paths.get( + "/Users/jackietien/Downloads/2_1746622362350_fa24aa15233f4e76bcda789a5771f43f"))); + session.insert(tablet); + tablet.reset(); + + tablet = new Tablet("tsfile_table", columnNameList, dataTypeList, columnTypeList, 1); + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 2); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue( + rowIndex, + 4, + true, + 0, + Files.readAllBytes( + Paths.get( + "/Users/jackietien/Downloads/2_1746622367063_8fb5ac8e21724140874195b60b878664"))); + session.insert(tablet); + tablet.reset(); + + tablet = new Tablet("tiff_table", columnNameList, dataTypeList, columnTypeList, 1); + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue( + rowIndex, + 4, + true, + 0, + Files.readAllBytes(Paths.get("/Users/jackietien/Downloads/1751891240130.tiff"))); + session.insert(tablet); + tablet.reset(); + + tablet = new Tablet("tiff_table", columnNameList, dataTypeList, columnTypeList, 1); + rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 2); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "4"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue( + rowIndex, + 4, + true, + 0, + Files.readAllBytes(Paths.get("/Users/jackietien/Downloads/1751891242743.tiff"))); + session.insert(tablet); + tablet.reset(); + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java b/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java new file mode 100644 index 0000000000000..344939175db78 --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.tsfile.utils.Binary; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +public class ObjectReadExample { + private static final String LOCAL_URL = "127.0.0.1:6667"; + + public static void main(String[] args) { + + // don't specify database in constructor + try (ITableSession session = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(LOCAL_URL)) + .username("root") + .password("root") + .database("test1") + .thriftMaxFrameSize(256 * 1024 * 1024) + .build()) { + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from tsfile_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + System.out.println(DigestUtils.md5Hex(binary.getValues())); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from tsfile_table where time = 2")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + System.out.println(DigestUtils.md5Hex(binary.getValues())); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement("select READ_OBJECT(file) from tsfile_table")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + System.out.println(DigestUtils.md5Hex(binary.getValues())); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select geo_penetrate(file, '0,3,7501,7504') from tsfile_table")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + ByteBuffer byteBuffer = ByteBuffer.wrap(binary.getValues()); + float[] res = new float[byteBuffer.limit() / Float.BYTES]; + for (int i = 0; i < res.length; i++) { + res[i] = byteBuffer.getFloat(); + } + System.out.println(Arrays.toString(res)); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select geo_penetrate(file, '0,3,7501,7504', 'UNCOMPRESSED_TIFF') from tiff_table")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + ByteBuffer byteBuffer = ByteBuffer.wrap(binary.getValues()); + float[] res = new float[byteBuffer.limit() / Float.BYTES]; + for (int i = 0; i < res.length; i++) { + res[i] = byteBuffer.getFloat(); + } + System.out.println(Arrays.toString(res)); + } + } + + } catch (IoTDBConnectionException e) { + e.printStackTrace(); + } catch (StatementExecutionException e) { + e.printStackTrace(); + } + } +} diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java index a8d3c8b382b3b..03e2a14c04693 100644 --- a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java +++ b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java @@ -119,7 +119,7 @@ public static void main(String[] args) { int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("region_id", rowIndex, "1"); - tablet.addValue("plant_id", rowIndex, "5"); + tablet.addValue("plant_id", rowIndex, null); tablet.addValue("device_id", rowIndex, "3"); tablet.addValue("model", rowIndex, "A"); tablet.addValue("temperature", rowIndex, 37.6F); @@ -134,6 +134,26 @@ public static void main(String[] args) { tablet.reset(); } + // query device leader + List isSetTag = Arrays.asList(true, true, false, true); + String correctURL = + session.getDeviceLeaderURL("test2", Arrays.asList("test1", "1", "3"), isSetTag, 66); + System.out.println("Correct device leader URL: " + correctURL); + String errorDbURL = + session.getDeviceLeaderURL("test3", Arrays.asList("test1", "1", "3"), isSetTag, 66); + System.out.println("Error dbName device leader URL: " + errorDbURL); + String errorDeviceURL = + session.getDeviceLeaderURL("test2", Arrays.asList("test1", "3", "1"), isSetTag, 66); + System.out.println("Error deviceId device leader URL: " + errorDeviceURL); + List falseTagList = Arrays.asList(false, true, true, true); + String errorTagURL = + session.getDeviceLeaderURL("test2", Arrays.asList("test1", "1", "3"), falseTagList, 66); + System.out.println("Error tag device leader URL: " + errorTagURL); + String errorTimeURL = + session.getDeviceLeaderURL( + "test2", Arrays.asList("test1", "1", "3"), isSetTag, 6666666666666666L); + System.out.println("Error time device leader URL: " + errorTimeURL); + // query table data try (SessionDataSet dataSet = session.executeQueryStatement( diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java index 9093133631a0e..59819ee418e6f 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java @@ -49,7 +49,10 @@ public enum Type { BLOB((byte) 10), /* STRING */ - STRING((byte) 11); + STRING((byte) 11), + + /* OBJECT */ + OBJECT((byte) 12); private final byte dataType; diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java index a5c2852ac1b95..c4c38c285dce9 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java @@ -56,7 +56,11 @@ public enum Type { BLOB((byte) 10), /* STRING */ - STRING((byte) 11); + STRING((byte) 11), + + /* OBJECT */ + OBJECT((byte) 12); + private final byte dataType; Type(byte type) { @@ -92,6 +96,7 @@ public boolean checkObjectType(Object o) { case DATE: return o instanceof LocalDate; case BLOB: + case OBJECT: return o instanceof Binary; case STRING: case TEXT: @@ -102,7 +107,8 @@ public boolean checkObjectType(Object o) { } public static List allTypes() { - return Arrays.asList(BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING); + return Arrays.asList( + BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING, OBJECT); } public static List numericTypes() { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index 50be4ea5d5ac4..582704bf8ad33 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -755,6 +755,7 @@ private static String getStringByColumnIndex( case DOUBLE: case TEXT: case STRING: + case OBJECT: return resultSet.getString(columnIndex); case BLOB: byte[] v = resultSet.getBytes(columnIndex); diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java index 75a37f8384719..bda14441f4d33 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ITableSession.java @@ -24,6 +24,8 @@ import org.apache.tsfile.write.record.Tablet; +import java.util.List; + /** * This interface defines a session for interacting with IoTDB tables. It supports operations such * as data insertion, executing queries, and closing the session. Implementations of this interface @@ -46,6 +48,18 @@ public interface ITableSession extends AutoCloseable { */ void insert(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException; + /** + * Retrieves the DataNode URL of the device leader for a given database and a deviceID. + * + * @param dbName the name of the database. + * @param deviceId a list of string for constructing the specified deviceID. + * @param isSetTag a true indicating the deviceID is set, false otherwise. + * @param time the time at which partition the device leader is queried. + * @return the DataNode URL of the device leader as a String. + */ + String getDeviceLeaderURL(String dbName, List deviceId, List isSetTag, long time) + throws IoTDBConnectionException, StatementExecutionException; + /** * Executes a non-query SQL statement, such as a DDL or DML command. * diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index fa67708fed0c7..bb113680d8e5b 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -189,6 +189,7 @@ private RowRecord constructRowRecordFromValueArray() throws StatementExecutionEx case TEXT: case BLOB: case STRING: + case OBJECT: field.setBinaryV(ioTDBRpcDataSet.getBinary(columnName)); break; default: diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java index 18069bf6faa1a..a66a39e1330ef 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java @@ -396,6 +396,7 @@ public static ByteBuffer convertTsBlock( case TEXT: case STRING: case BLOB: + case OBJECT: tsBlockBuilder .getColumnBuilder(j) .writeBinary( diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml index d7efd73817cb7..a675709da36ea 100644 --- a/iotdb-client/service-rpc/pom.xml +++ b/iotdb-client/service-rpc/pom.xml @@ -84,6 +84,16 @@ org.slf4j slf4j-api + + net.imagej + ij + 1.54g + + + org.gdal + gdal + 3.11.0 + diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index 5bba6d0ea718d..43064ecdaa9fb 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -184,6 +184,7 @@ public IoTDBJDBCDataSet( case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = null; break; default: @@ -307,6 +308,7 @@ public IoTDBJDBCDataSet( case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = null; break; default: @@ -418,6 +420,7 @@ public void constructOneRow() { case TEXT: case BLOB: case STRING: + case OBJECT: int length = valueBuffer.getInt(); values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length); break; @@ -596,6 +599,8 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { case TEXT: case STRING: return new String(values[index], StandardCharsets.UTF_8); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString(values[index]); case BLOB: return BytesUtils.parseBlobByteArrayToString(values[index]); case DATE: @@ -634,6 +639,7 @@ public Object getObject(int index, TSDataType tsDataType, byte[][] values) { case TEXT: case STRING: return new String(values[index], StandardCharsets.UTF_8); + case OBJECT: case BLOB: return new Binary(values[index]); case TIMESTAMP: diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index a40197fac7988..940b3460d927b 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -494,6 +494,9 @@ private Object getObjectByTsBlockIndex(int tsBlockColumnIndex) .getColumn(tsBlockColumnIndex) .getBinary(tsBlockIndex) .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString( + curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); @@ -554,6 +557,9 @@ private String getString(int index, TSDataType tsDataType) { .getColumn(index) .getBinary(tsBlockIndex) .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString( + curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index d64d761631c65..a04a50115b3ec 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -145,6 +145,9 @@ public enum TSStatusCode { PLAN_FAILED_NETWORK_PARTITION(721), CANNOT_FETCH_FI_STATE(722), + // OBJECT + OBJECT_NOT_EXISTS(740), + // Arithmetic NUMERIC_VALUE_OUT_OF_RANGE(750), DIVISION_BY_ZERO(751), diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java new file mode 100644 index 0000000000000..ed5a44a87f91c --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.model; + +import org.gdal.gdal.Band; +import org.gdal.gdal.Dataset; +import org.gdal.gdal.Driver; +import org.gdal.gdal.gdal; +import org.gdal.gdalconst.gdalconstConstants; + +public class CompressedTiffModelProcessor extends ModelProcessor { + private static final Driver DRIVER; + private static final String VIRTUAL_FILE_PATH_PREFIX = "/vsimem"; + private static final String VIRTUAL_FILE_PATH_SUFFIX = ".tif"; + // Specifying compression options + private static String compressOption = "COMPRESS=LZW"; + // Specifying block x size options + private static String blockXSize = "BLOCKXSIZE=256"; + // Specifying block y size options + private static String blockYSize = "BLOCKYSIZE=256"; + + static { + gdal.AllRegister(); + DRIVER = gdal.GetDriverByName("GTiff"); + if (DRIVER == null) { + throw new RuntimeException("Failed to get GTiff driver: " + gdal.GetLastErrorMsg()); + } + } + + @Override + public byte[] write(float[] values, int width, int height) { + String virtualFilePath = getVirtualFilePath(); + try { + return write(virtualFilePath, values, width, height); + } finally { + gdal.Unlink(virtualFilePath); + } + } + + private byte[] write(String filePath, float[] values, int width, int height) { + // floating point data should use predictor 2 (for difference prediction), and use block storage + // (recommended for LZW) + String[] options = + new String[] {compressOption, "PREDICTOR=2", "TILED=YES", blockXSize, blockYSize}; + + Dataset dataset = null; + try { + // Create dataset with specified options + dataset = DRIVER.Create(filePath, width, height, 1, gdalconstConstants.GDT_Float32, options); + + if (dataset == null) { + throw new RuntimeException("Failed to create dataset: " + gdal.GetLastErrorMsg()); + } + + Band band = dataset.GetRasterBand(1); + int result = band.WriteRaster(0, 0, width, height, values); + if (result != gdalconstConstants.CE_None) { + throw new RuntimeException("Failed to write data to tiff file: " + gdal.GetLastErrorMsg()); + } + band.FlushCache(); + dataset.FlushCache(); + return gdal.GetMemFileBuffer(filePath); + } finally { + if (dataset != null) { + dataset.delete(); + } + } + } + + @Override + public float[] readAll(byte[] fileBytes) { + String virtualFilePath = getVirtualFilePath(); + try { + gdal.FileFromMemBuffer(virtualFilePath, fileBytes); + return readAll(virtualFilePath); + } finally { + gdal.Unlink(virtualFilePath); + } + } + + @Override + public float[] readAll(String filePath) { + Dataset dataset = gdal.Open(filePath, gdalconstConstants.GA_ReadOnly); + if (dataset == null) { + throw new RuntimeException("Failed to open tiff file: " + gdal.GetLastErrorMsg()); + } + try { + Band band = dataset.GetRasterBand(1); + if (band == null) { + throw new RuntimeException( + "Failed to get raster band from dataset" + gdal.GetLastErrorMsg()); + } + int width = band.getXSize(); + int height = band.getYSize(); + float[] result = new float[width * height]; + band.ReadRaster(0, 0, width, height, gdalconstConstants.GDT_Float32, result); + return result; + } finally { + dataset.delete(); + } + } + + private String getVirtualFilePath() { + long tid = Thread.currentThread().getId(); + long timestamp = System.currentTimeMillis(); + return String.format( + "%s/%d_%d%s", VIRTUAL_FILE_PATH_PREFIX, tid, timestamp, VIRTUAL_FILE_PATH_SUFFIX); + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTsFileModelProcessor.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTsFileModelProcessor.java new file mode 100644 index 0000000000000..678d86cd07abf --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTsFileModelProcessor.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.model; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.FloatStatistics; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.reader.TsFileInput; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.v4.DeviceTableModelWriter; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.writer.TsFileOutput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Collections; +import java.util.stream.Collectors; + +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; + +public class CompressedTsFileModelProcessor extends ModelProcessor { + private static final Logger LOGGER = + LoggerFactory.getLogger(CompressedTsFileModelProcessor.class); + private static final int DEFAULT_CHUNK_NUMBER = 128 * 128; + + @Override + public byte[] write(float[] values, int width, int height) { + try { + + TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(1); + String tableName = "t"; + MemoryTsFileOutput tsFileOutput = new MemoryTsFileOutput(60 * 1024 * 1024); + + TableSchema tableSchema = + new TableSchema( + tableName, + Collections.singletonList( + new ColumnSchemaBuilder() + .name("v") + .dataType(TSDataType.FLOAT) + .category(ColumnCategory.FIELD) + .build())); + try (ITsFileWriter writer = new DeviceTableModelWriter(tsFileOutput, tableSchema, 1)) { + Tablet tablet = + new Tablet( + Collections.singletonList("v"), + Collections.singletonList(TSDataType.FLOAT), + DEFAULT_CHUNK_NUMBER); + + for (int i = 0; i < values.length; i++) { + int row = tablet.getRowSize(); + tablet.addTimestamp(row, i); + tablet.addValue(row, "v", values[i]); + // write + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writer.write(tablet); + tablet.reset(); + } + } + // write + if (tablet.getRowSize() != 0) { + writer.write(tablet); + tablet.reset(); + } + } + return tsFileOutput.getByteBuffer().array(); + } catch (Exception e) { + LOGGER.error("write tsfile failed", e); + } + return new byte[0]; + } + + @Override + public float[] readAll(byte[] fileBytes) { + try { + float[] values = null; + int i = 0; + IDeviceID deviceID = new StringArrayDeviceID("t"); + TsFileInput tsFileInput = new ByteBufferTsFileInput(ByteBuffer.wrap(fileBytes)); + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFileInput)) { + TimeseriesMetadata timeseriesMetadata = + reader.getAllTimeseriesMetadata(false).get(deviceID).stream() + .filter(t -> t.getMeasurementId().equals("v")) + .collect(Collectors.toList()) + .get(0); + values = new float[timeseriesMetadata.getStatistics().getCount()]; + + byte marker; + ChunkHeader chunkHeader; + reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); + while ((marker = reader.readMarker()) != MetaMarker.SEPARATOR) { + switch (marker) { + case MetaMarker.CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: + case MetaMarker.TIME_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: + chunkHeader = reader.readChunkHeader(marker); + reader.position(reader.position() + chunkHeader.getDataSize()); + break; + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + chunkHeader = reader.readChunkHeader(marker); + ByteBuffer chunkDataBuffer = reader.readChunk(-1, chunkHeader.getDataSize()); + + while (chunkDataBuffer.hasRemaining()) { + PageHeader pageHeader = null; + if (((byte) (chunkHeader.getChunkType() & 0x3F)) + == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { + pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, new FloatStatistics()); + } else { + pageHeader = + PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); + } + + ByteBuffer pageData = readCompressedPageData(pageHeader, chunkDataBuffer); + IDecryptor decryptor = IDecryptor.getDecryptor(EncryptUtils.getEncryptParameter()); + ByteBuffer uncompressedPageData = + decryptAndUncompressPageData( + pageHeader, + IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), + pageData, + decryptor); + Decoder decoder = + Decoder.getDecoderByType( + chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + byte[] bitmap = null; + if (uncompressedPageData.hasRemaining()) { + int size = ReadWriteIOUtils.readInt(uncompressedPageData); + bitmap = new byte[(size + 7) / 8]; + uncompressedPageData.get(bitmap); + } + while (decoder.hasNext(uncompressedPageData)) { + values[i++] = decoder.readFloat(uncompressedPageData); + } + } + break; + case MetaMarker.CHUNK_GROUP_HEADER: + reader.readChunkGroupHeader(); + break; + default: + return values; + } + } + } + return values; + } catch (Exception e) { + LOGGER.error("Read TS file failed", e); + return new float[] {0}; + } + } + + @Override + public float[] readAll(String filePath) { + try { + byte[] fileBytes = Files.readAllBytes(new File(filePath).toPath()); + return readAll(fileBytes); + } catch (Exception e) { + LOGGER.error("Read TS file failed", e); + return new float[] {0}; + } + } + + private static class MemoryTsFileOutput implements TsFileOutput { + + private ByteArrayOutputStream baos; + + public MemoryTsFileOutput(int initialSize) { + this.baos = new PublicBAOS(initialSize); + } + + public ByteBuffer getByteBuffer() { + return ByteBuffer.wrap(baos.toByteArray()); + } + + @Override + public void write(byte[] b) throws IOException { + baos.write(b); + } + + @Override + public void write(byte b) throws IOException { + baos.write(b); + } + + @Override + public void write(ByteBuffer b) throws IOException { + baos.write(b.array()); + } + + @Override + public long getPosition() throws IOException { + return baos.size(); + } + + @Override + public void close() throws IOException { + baos.close(); + } + + @Override + public OutputStream wrapAsStream() throws IOException { + return baos; + } + + @Override + public void flush() throws IOException { + baos.flush(); + } + + @Override + public void truncate(long size) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void force() throws IOException {} + } + + public static class ByteBufferTsFileInput implements TsFileInput { + + public ByteBuffer buffer; + + public ByteBufferTsFileInput(ByteBuffer buffer) { + this.buffer = buffer; + } + + @Override + public long size() throws IOException { + return buffer.limit(); + } + + @Override + public long position() throws IOException { + return buffer.position(); + } + + @Override + public TsFileInput position(long newPosition) throws IOException { + buffer.position((int) newPosition); + return this; + } + + @Override + public int read(ByteBuffer dst) { + int bytesToRead = Math.min(dst.remaining(), buffer.remaining()); + if (bytesToRead == 0) { + return 0; + } + ByteBuffer slice = buffer.slice(); + slice.limit(bytesToRead); + dst.put(slice); + buffer.position(buffer.position() + bytesToRead); + return bytesToRead; + } + + @Override + public int read(ByteBuffer dst, long position) { + ByteBuffer readBuffer = buffer.slice(); + readBuffer.position((int) position); + int bytesToRead = Math.min(dst.remaining(), readBuffer.remaining()); + if (bytesToRead > 0) { + readBuffer.limit(readBuffer.position() + bytesToRead); + dst.put(readBuffer); + } + return bytesToRead; + } + + @Override + public InputStream wrapAsInputStream() throws IOException { + return new InputStream() { + @Override + public int read() throws IOException { + if (!buffer.hasRemaining()) { + return -1; + } + return buffer.get() & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (!buffer.hasRemaining()) { + return -1; + } + int toRead = Math.min(len, buffer.remaining()); + buffer.get(b, off, toRead); + return toRead; + } + + @Override + public int available() throws IOException { + return buffer.remaining(); + } + }; + } + + @Override + public void close() throws IOException {} + + @Override + public String getFilePath() { + return "memory tsfile data buffer"; + } + } + + public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffer chunkBuffer) + throws IOException { + int compressedPageBodyLength = pageHeader.getCompressedSize(); + if (compressedPageBodyLength > chunkBuffer.remaining()) { + throw new IOException( + "do not have a complete page body. Expected:" + + compressedPageBodyLength + + ". Actual:" + + chunkBuffer.remaining()); + } + ByteBuffer pageBodyBuffer = chunkBuffer.slice(); + pageBodyBuffer.limit(compressedPageBodyLength); + chunkBuffer.position(chunkBuffer.position() + compressedPageBodyLength); + return pageBodyBuffer; + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java new file mode 100644 index 0000000000000..a7f9ef5080516 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.model; + +public abstract class ModelProcessor { + /** + * Write float[] to specific type of file, eg. tsfile, tiff + * + * @param values the float values of image + * @param width the width of image + * @param height the height of image + * @return the byte array of the file + */ + public abstract byte[] write(float[] values, int width, int height); + + /** + * Read all float values from a file, eg. tsfile, tiff + * + * @param fileBytes the byte array of the file to read + * @return the float array of the image + */ + public abstract float[] readAll(byte[] fileBytes); + + /** + * Read all float values from a file, eg. tsfile, tiff + * + * @param filePath the path of the file to read + * @return the float array of the image + */ + public abstract float[] readAll(String filePath); + + public static ModelProcessor getInstance(String modelFileType) { + if (modelFileType.equalsIgnoreCase("COMPRESSED_TSFILE")) { + return new CompressedTsFileModelProcessor(); + } else if (modelFileType.equalsIgnoreCase("UNCOMPRESSED_TIFF")) { + return new UnCompressedTiffModelProcessor(); + } else if (modelFileType.equalsIgnoreCase("COMPRESSED_TIFF")) { + return new CompressedTiffModelProcessor(); + } else { + throw new IllegalArgumentException("Unsupported model file type: " + modelFileType); + } + } +} diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/UnCompressedTiffModelProcessor.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/UnCompressedTiffModelProcessor.java new file mode 100644 index 0000000000000..17a286da4d7a5 --- /dev/null +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/UnCompressedTiffModelProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc.model; + +import ij.ImagePlus; +import ij.io.FileSaver; +import ij.io.Opener; +import ij.process.FloatProcessor; +import ij.process.ImageProcessor; + +public class UnCompressedTiffModelProcessor extends ModelProcessor { + @Override + public byte[] write(float[] values, int width, int height) { + FloatProcessor floatProcessor = new FloatProcessor(width, height, values); + ImagePlus imp = new ImagePlus("first level", floatProcessor); + FileSaver fs = new FileSaver(imp); + return fs.serialize(); + } + + @Override + public float[] readAll(String filePath) { + Opener opener = new Opener(); + ImagePlus imagePlus = opener.openImage(filePath); + ImageProcessor processor = imagePlus.getProcessor(); + if (processor instanceof FloatProcessor) { + return (float[]) processor.getPixels(); + } else { + return new float[0]; + } + } + + @Override + public float[] readAll(byte[] fileBytes) { + Opener opener = new Opener(); + ImagePlus imagePlus = opener.deserialize(fileBytes); + ImageProcessor processor = imagePlus.getProcessor(); + if (processor instanceof FloatProcessor) { + return (float[]) processor.getPixels(); + } else { + return new float[0]; + } + } +} diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index da3185e51f87a..e933ab6e078e5 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -56,6 +56,8 @@ import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp; import org.apache.iotdb.service.rpc.thrift.TSSetSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderReq; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderResp; import org.apache.iotdb.session.template.MeasurementNode; import org.apache.iotdb.session.template.TemplateQueryType; import org.apache.iotdb.session.util.SessionUtils; @@ -2778,6 +2780,13 @@ public void insertRelationalTablet(Tablet tablet) } } + public TTableDeviceLeaderResp fetchDeviceLeader( + String dbName, List deviceId, List isSetTag, long time) + throws IoTDBConnectionException, StatementExecutionException { + TTableDeviceLeaderReq req = new TTableDeviceLeaderReq(dbName, deviceId, isSetTag, time); + return getDefaultSessionConnection().fetchDeviceLeader(req); + } + private void insertRelationalTabletWithLeaderCache(Tablet tablet) throws IoTDBConnectionException, StatementExecutionException { Map relationalTabletGroup = new HashMap<>(); @@ -3649,6 +3658,7 @@ private Object sortList(Object valueList, TSDataType dataType, Integer[] index) case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) valueList; Binary[] sortedBinaryValues = new Binary[binaryValues.length]; for (int i = 0; i < index.length; i++) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 477eba234eb35..eae7150b2e89f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -65,6 +65,8 @@ import org.apache.iotdb.service.rpc.thrift.TSSetSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderReq; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderResp; import org.apache.iotdb.session.util.SessionUtils; import org.apache.thrift.TException; @@ -1225,6 +1227,16 @@ protected TSBackupConfigurationResp getBackupConfiguration() return execResp; } + protected TTableDeviceLeaderResp fetchDeviceLeader(TTableDeviceLeaderReq req) + throws StatementExecutionException { + final TTableDeviceLeaderResp resp = + callWithRetryAndReconnect( + () -> client.fetchDeviceLeader(req), TTableDeviceLeaderResp::getStatus) + .getResult(); + RpcUtils.verifySuccess(resp.getStatus()); + return resp; + } + private RetryResult callWithReconnect(TFunction supplier) throws IoTDBConnectionException { T ret; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java index 059fdea346948..2a83b533efa40 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSession.java @@ -23,9 +23,12 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderResp; import org.apache.tsfile.write.record.Tablet; +import java.util.List; + public class TableSession implements ITableSession { private final Session session; @@ -39,6 +42,14 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne session.insertRelationalTablet(tablet); } + @Override + public String getDeviceLeaderURL( + String dbName, List deviceId, List isSetTag, long time) + throws IoTDBConnectionException, StatementExecutionException { + TTableDeviceLeaderResp resp = session.fetchDeviceLeader(dbName, deviceId, isSetTag, time); + return resp.getIp() + ":" + resp.getPort(); + } + @Override public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 761bb3a40d3b1..559040b375243 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -23,12 +23,14 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderResp; import org.apache.iotdb.session.Session; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; @@ -67,6 +69,14 @@ public void insert(Tablet tablet) throws StatementExecutionException, IoTDBConne } } + @Override + public String getDeviceLeaderURL( + String dbName, List deviceId, List isSetTag, long time) + throws IoTDBConnectionException, StatementExecutionException { + TTableDeviceLeaderResp resp = session.fetchDeviceLeader(dbName, deviceId, isSetTag, time); + return resp.getIp() + ":" + resp.getPort(); + } + @Override public SessionDataSet executeQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java index a3f7f8e3f26d1..a3f01b3e11e12 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java @@ -242,6 +242,7 @@ private static Field generateFieldFromTabletValue( case TEXT: case STRING: case BLOB: + case OBJECT: final Binary binaryValue = new Binary((((Binary[]) value)[index]).getValues()); field.setBinaryV(binaryValue); break; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index 7a43578b97535..de622d259f265 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -132,6 +132,7 @@ private static int calOccupationOfOneColumn( case TEXT: case BLOB: case STRING: + case OBJECT: valueOccupation += rowSize * 4; Binary[] binaries = (Binary[]) values[columnIndex]; for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { @@ -182,6 +183,7 @@ private static int calculateLength(List types, List values) break; case TEXT: case STRING: + case OBJECT: res += Integer.BYTES; if (values.get(i) instanceof Binary) { res += ((Binary) values.get(i)).getValues().length; @@ -330,6 +332,7 @@ private static void getValueBufferOfDataType( case TEXT: case STRING: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) tablet.getValues()[i]; for (int index = 0; index < tablet.getRowSize(); index++) { if (!tablet.isNull(index, i) && binaryValues[index] != null) { diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index f1cd4a4fe46f5..c40553cde8117 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -392,6 +392,11 @@ 1.3.0 test + + org.gdal + gdal + 3.11.0 + diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java index efb6758066f10..551d8a9031d63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java @@ -60,8 +60,7 @@ public class DataNodeMemoryConfig { private int queryThreadCount = Runtime.getRuntime().availableProcessors(); /** Max bytes of each FragmentInstance for DataExchange */ - private long maxBytesPerFragmentInstance = - Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001 / queryThreadCount; + private long maxBytesPerFragmentInstance = Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001; /** The memory manager of on heap */ private MemoryManager onHeapMemoryManager; @@ -483,7 +482,7 @@ private void initQueryEngineMemoryAllocate( operatorsMemorySize += partForOperators; } // set max bytes per fragment instance - setMaxBytesPerFragmentInstance(dataExchangeMemorySize / getQueryThreadCount()); + setMaxBytesPerFragmentInstance(dataExchangeMemorySize); bloomFilterCacheMemoryManager = queryEngineMemoryManager.getOrCreateMemoryManager( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 8137959a3917a..afd957cbf6b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1162,6 +1162,8 @@ public class IoTDBConfig { private long cacheLastValuesMemoryBudgetInByte = 4 * 1024 * 1024; + private long maxObjectSizeInByte = 4 * 1024 * 1024 * 1024L; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -4116,4 +4118,12 @@ public long getCacheLastValuesMemoryBudgetInByte() { public void setCacheLastValuesMemoryBudgetInByte(long cacheLastValuesMemoryBudgetInByte) { this.cacheLastValuesMemoryBudgetInByte = cacheLastValuesMemoryBudgetInByte; } + + public long getMaxObjectSizeInByte() { + return maxObjectSizeInByte; + } + + public void setMaxObjectSizeInByte(long maxObjectSizeInByte) { + this.maxObjectSizeInByte = maxObjectSizeInByte; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index a4d199f662ddf..0b6ebb0813ae6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -897,6 +897,11 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir())); + conf.setMaxObjectSizeInByte( + Long.parseLong( + properties.getProperty( + "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte())))); + // At the same time, set TSFileConfig List fsTypes = new ArrayList<>(); fsTypes.add(FSType.LOCAL); @@ -2056,6 +2061,11 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) properties.getProperty( "tvlist_sort_threshold", ConfigurationFileUtils.getConfigurationDefaultValue("tvlist_sort_threshold")))); + + conf.setMaxObjectSizeInByte( + Long.parseLong( + properties.getProperty( + "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte())))); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -2569,6 +2579,7 @@ public TSEncoding getDefaultEncodingByType(TSDataType dataType) { return conf.getDefaultDoubleEncoding(); case STRING: case BLOB: + case OBJECT: case TEXT: default: return conf.getDefaultTextEncoding(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index 7431aa9a79d82..91c76f0f9b10d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -292,4 +293,16 @@ public TSStatus visitPipeEnrichedDeleteDataNode( node.getDeleteDataNode().markAsGeneratedByPipe(); return node.getDeleteDataNode().accept(this, context); } + + @Override + public TSStatus visitWriteObjectFile(ObjectNode node, DataRegion dataRegion) { + try { + dataRegion.writeObject(node); + dataRegion.insertSeparatorToWAL(); + return StatusUtils.OK; + } catch (final Exception e) { + LOGGER.error("Error in executing plan node: {}", node, e); + return new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaServerHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaServerHandle.java index 48d7c415ab3ff..91b3378c8ab1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaServerHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcda/OpcDaServerHandle.java @@ -318,6 +318,7 @@ private short convertTsDataType2VariantType(final TSDataType dataType) { // Note that "Variant" does not support "VT_BLOB" data, and not all the DA server // support this, thus we use "VT_BSTR" to substitute case BLOB: + case OBJECT: return Variant.VT_BSTR; default: throw new UnSupportedDataTypeException("UnSupported dataType " + dataType); @@ -354,6 +355,7 @@ private Variant.VARIANT getTabletObjectValue4Opc( case TEXT: case STRING: case BLOB: + case OBJECT: bstr = OleAuto.INSTANCE.SysAllocString(((Binary[]) column)[rowIndex].toString()); value.setValue(Variant.VT_BSTR, bstr); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java index 3ea470de38a96..0f8d8ee78412b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java @@ -136,6 +136,7 @@ protected Object reorderValueListAndBitMap( return deDuplicatedDoubleValues; case TEXT: case BLOB: + case OBJECT: case STRING: final Binary[] binaryValues = (Binary[]) valueList; final Binary[] deDuplicatedBinaryValues = new Binary[binaryValues.length]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index 1119deaf71287..3a0b45fc57e26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -497,6 +497,7 @@ private Map> processRow( timestamp, row.getString(index), outputMinReportIntervalMilliseconds); break; case BLOB: + case OBJECT: result = state.updateWindows( timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds); @@ -684,6 +685,7 @@ public void collectWindowOutputs( break; case TEXT: case BLOB: + case OBJECT: case STRING: valueColumns[columnIndex] = new Binary[distinctOutputs.size()]; break; @@ -733,6 +735,7 @@ public void collectWindowOutputs( TSFileConfig.STRING_CHARSET); break; case BLOB: + case OBJECT: ((Binary[]) valueColumns[columnIndex])[rowIndex] = (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index ed34f0de31039..38e8fac057f8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -630,6 +630,7 @@ public static long sizeOfColumns( case STRING: case TEXT: case BLOB: + case OBJECT: { size += getBinarySize((Binary[]) columns[i]); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index d149d9dc4c95b..7e05bc2f2125c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -181,6 +181,8 @@ import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq; import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo; import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderReq; +import org.apache.iotdb.service.rpc.thrift.TTableDeviceLeaderResp; import io.airlift.units.Duration; import io.jsonwebtoken.lang.Strings; @@ -844,7 +846,7 @@ private List executeGroupByQueryInternal( IMeasurementSchema measurementSchema = new MeasurementSchema(measurement, dataType); AbstractSeriesAggregationScanOperator operator; boolean canUseStatistics = - !TSDataType.BLOB.equals(dataType) + (!TSDataType.BLOB.equals(dataType) && !TSDataType.OBJECT.equals(dataType)) || (!TAggregationType.LAST_VALUE.equals(aggregationType) && !TAggregationType.FIRST_VALUE.equals(aggregationType)); IFullPath path; @@ -3092,6 +3094,38 @@ public TSStatus testConnectionEmptyRPC() throws TException { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } + @Override + public TTableDeviceLeaderResp fetchDeviceLeader(TTableDeviceLeaderReq req) throws TException { + int start_index = 0; + String[] trueSegments = new String[req.getIsSetTagSize()]; + for (int i = 0; i < req.getIsSetTagSize(); i++) { + trueSegments[i] = req.getIsSetTag().get(i) ? req.getDeviceId().get(start_index++) : null; + } + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create(trueSegments); + TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(req.getTime()); + DataPartitionQueryParam queryParam = + new DataPartitionQueryParam(deviceID, Collections.singletonList(timePartitionSlot)); + DataPartition dataPartition = + partitionFetcher.getDataPartition( + Collections.singletonMap(req.getDbName(), Collections.singletonList(queryParam))); + TRegionReplicaSet targetRegionReplicaSet = + dataPartition.getAllReplicaSets().stream().findFirst().orElse(null); + TEndPoint targetEndPoint = + targetRegionReplicaSet != null + ? targetRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint() + : null; + TTableDeviceLeaderResp resp = new TTableDeviceLeaderResp(); + resp.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + if (targetEndPoint != null) { + resp.setIp(targetEndPoint.getIp()); + resp.setPort(String.valueOf(targetEndPoint.getPort())); + } else { + resp.setIp(""); + resp.setPort(""); + } + return resp; + } + @Override public TSStatus insertStringRecord(final TSInsertStringRecordReq req) { final long t1 = System.nanoTime(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java index 24a998f54a917..b4025a23de767 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java @@ -160,6 +160,7 @@ private static Accumulator createModeAccumulator(TSDataType tsDataType) { case DOUBLE: return new DoubleModeAccumulator(); case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java index c6d1baa33830a..6549a7a6748e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java @@ -57,6 +57,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java index 76a42b41c7180..ff54a48ce5e79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java @@ -58,6 +58,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -90,6 +91,7 @@ public void addIntermediate(Column[] partialResult) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -124,6 +126,7 @@ public void addStatistics(Statistics statistics) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -155,6 +158,7 @@ public void setFinal(Column finalResult) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -188,6 +192,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -219,6 +224,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java index 3bc2e0f9c8961..640008f0a5fda 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java @@ -63,6 +63,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(columns, bitMap); return; case BOOLEAN: @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryFirstValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); break; @@ -132,6 +134,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime()); break; @@ -167,6 +170,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case STRING: firstValue.setBinary(finalResult.getBinary(0)); break; @@ -206,6 +210,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilders[0].writeBinary(firstValue.getBinary()); break; @@ -242,6 +247,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(firstValue.getBinary()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java index 603f3427860ad..36c6ed5f88df7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java @@ -63,6 +63,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(columns, bitMap); return; case BOOLEAN: @@ -99,6 +100,7 @@ public void addIntermediate(Column[] partialResult) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryLastValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); break; case BOOLEAN: @@ -133,6 +135,7 @@ public void addStatistics(Statistics statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime()); break; case BOOLEAN: @@ -168,6 +171,7 @@ public void setFinal(Column finalResult) { case TEXT: case BLOB: case STRING: + case OBJECT: lastValue.setBinary(finalResult.getBinary(0)); break; case BOOLEAN: @@ -207,6 +211,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilders[0].writeBinary(lastValue.getBinary()); break; case BOOLEAN: @@ -243,6 +248,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(lastValue.getBinary()); break; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java index a0e4f80120cb0..ebbd1aa6dc149 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java @@ -88,6 +88,7 @@ public void addInput(Column[] column, BitMap bitMap) { case TEXT: case BLOB: case BOOLEAN: + case OBJECT: default: throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); } @@ -302,6 +303,7 @@ private void writeX(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -335,6 +337,7 @@ private void updateX(Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xResult.setBinary(xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -385,6 +388,7 @@ private void writeIntermediateToStream( case TEXT: case STRING: case BLOB: + case OBJECT: String content = value.getBinary().toString(); dataOutputStream.writeInt(content.length()); dataOutputStream.writeBytes(content); @@ -441,6 +445,7 @@ private void updateFromBytesIntermediateInput(byte[] bytes) { case TEXT: case BLOB: case BOOLEAN: + case OBJECT: default: throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); } @@ -471,6 +476,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java index 0d58de8064f26..d02678bd6225a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java @@ -64,6 +64,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -130,6 +132,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -164,6 +167,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -199,6 +203,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -232,6 +237,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java index 1d9cc59aa17de..fd55ea0f1bc74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java @@ -64,6 +64,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -130,6 +132,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -164,6 +167,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -199,6 +203,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -232,6 +237,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java index 37daf1a84b1e0..066d8cd4fc5d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java @@ -57,6 +57,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case TIMESTAMP: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java index 3242518c3dc2d..8d27f1497e06d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java @@ -70,6 +70,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index 60dda19757818..e3c2d3b4842e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; @@ -370,6 +371,19 @@ public RegionExecutionResult visitDeleteData( } } + @Override + public RegionExecutionResult visitWriteObjectFile( + final ObjectNode node, final WritePlanNodeExecutionContext context) { + // data deletion don't need to block data insertion, but there are some creation operation + // require write lock on data region. + context.getRegionWriteValidationRWLock().writeLock().lock(); + try { + return super.visitWriteObjectFile(node, context); + } finally { + context.getRegionWriteValidationRWLock().writeLock().unlock(); + } + } + @Override public RegionExecutionResult visitDeleteTimeseries( final DeleteTimeSeriesNode node, final WritePlanNodeExecutionContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java index 03766ab52762d..b6ce8b521100a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java @@ -30,7 +30,6 @@ public class LocalMemoryManager { private final MemoryPool queryPool; public LocalMemoryManager() { - // TODO @spricoder: why this pool is only used for query data exchange queryPool = new MemoryPool( "read", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java index cba7c3defa6b4..ea5e1f6b5ecdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java @@ -262,6 +262,7 @@ public static long getOutputColumnSizePerLine(TSDataType tsDataType) { return BooleanColumn.SIZE_IN_BYTES_PER_POSITION; case TEXT: case BLOB: + case OBJECT: case STRING: return StatisticsManager.getInstance().getMaxBinarySizeInBytes(); default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java index 4668da6413ed9..bf021e5b4f409 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java @@ -306,6 +306,7 @@ private void initResultTsBlock() { case TEXT: case STRING: case BLOB: + case OBJECT: columns[i] = new BinaryColumn( positionCount, @@ -382,6 +383,7 @@ private long getMemoryUsageOfOneMergeSortKey() { case TEXT: case STRING: case BLOB: + case OBJECT: memory += 16; break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java index 6dd24e3795a9c..28a80462ea771 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java @@ -344,6 +344,7 @@ protected YieldableState collectDataPoint(ColumnBuilder writer, long currentTime break; case TEXT: case BLOB: + case OBJECT: case STRING: writer.writeBinary(valueColumn.getBinary(currentIndex)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java index e332e6b7b0b82..8b1323246c551 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java @@ -113,6 +113,7 @@ public static Comparator getComparator(TSDataType dataType, int index, break; case TEXT: case BLOB: + case OBJECT: case STRING: comparator = Comparator.comparing( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index 90b519d52cd97..91e39753344d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -105,6 +105,7 @@ private void writeDefaultValue( case TEXT: case STRING: case BLOB: + case OBJECT: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java index e21ff13a861bb..c51eac1563596 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java @@ -106,6 +106,7 @@ private void writeDefaultValue( case TEXT: case STRING: case BLOB: + case OBJECT: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index ac6f225880931..73ff0fe60e64c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -99,6 +99,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse case STRING: case TEXT: case BLOB: + case OBJECT: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -178,6 +179,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o case TEXT: case STRING: case BLOB: + case OBJECT: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -242,6 +244,7 @@ public boolean equal(List columns1, int offset1, List columns2, case TEXT: case STRING: case BLOB: + case OBJECT: Binary bin1 = column1.getBinary(offset1); Binary bin2 = column2.getBinary(offset2); if (!bin1.equals(bin2)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 817649869d945..b4234c557721b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -983,6 +983,7 @@ private void addTimeValuePairToResult(TimeValuePair timeValuePair, TsBlockBuilde break; case TEXT: case BLOB: + case OBJECT: case STRING: builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java index df487f06c602f..53f5ab8807d63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java @@ -401,6 +401,7 @@ private TsPrimitiveType cloneTsPrimitiveType(TsPrimitiveType originalValue) { return new TsPrimitiveType.TsDouble(originalValue.getDouble()); case TEXT: case BLOB: + case OBJECT: case STRING: return new TsPrimitiveType.TsBinary(originalValue.getBinary()); case VECTOR: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 3ff20974168be..ca7853495159b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -348,6 +348,7 @@ public static GroupedAccumulator getGroupedApproxMostFrequentAccumulator(TSDataT case STRING: return new BinaryGroupedApproxMostFrequentAccumulator(); case BLOB: + case OBJECT: return new BlobGroupedApproxMostFrequentAccumulator(); default: throw new UnSupportedDataTypeException( @@ -373,6 +374,7 @@ public static TableAccumulator getApproxMostFrequentAccumulator(TSDataType type) case STRING: return new BinaryApproxMostFrequentAccumulator(); case BLOB: + case OBJECT: return new BlobApproxMostFrequentAccumulator(); default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java index b78aa1b0177d5..ef31759ee677b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java @@ -78,6 +78,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], mask, hll); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java index 464820e22f4dc..7bb03faadeb7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java @@ -73,6 +73,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: @@ -101,6 +102,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java index db50a3cb7d704..cc9d3e3354cb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java @@ -68,6 +68,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -100,6 +101,7 @@ public void addIntermediate(Column argument) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -136,6 +138,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -168,6 +171,7 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -200,6 +204,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java index 4bfc981c5a300..322079c999ac1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java @@ -80,6 +80,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -130,6 +131,7 @@ public void addIntermediate(Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -182,6 +184,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(firstValue.getBinary()); break; case BOOLEAN: @@ -225,6 +228,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryFirstValue( (Binary) statistics[0].getFirstValue(), statistics[0].getStartTime()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java index 3691a13e5f610..fe6c4a5db302a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java @@ -97,6 +97,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -157,6 +158,7 @@ public void addIntermediate(Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -212,6 +214,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xResult.getBinary()); break; @@ -276,6 +279,7 @@ public void addStatistics(Statistics[] statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xResult.setBinary((Binary) statistics[0].getFirstValue()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java index 2745ee1dff9f9..27a4b94283e28 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java @@ -98,6 +98,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -147,6 +148,7 @@ public void addIntermediate(Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -199,6 +201,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(lastValue.getBinary()); break; @@ -242,6 +245,7 @@ public void addStatistics(Statistics[] statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryLastValue((Binary) statistics[0].getLastValue(), statistics[0].getEndTime()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java index 1e457c19afad6..403e8ac340f39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java @@ -121,6 +121,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -182,6 +183,7 @@ public void addIntermediate(Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -237,6 +239,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -301,6 +304,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: xResult.setBinary((Binary) statistics[0].getLastValue()); break; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java index aaab07aa90b82..6f42e1a9995f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java @@ -67,6 +67,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: @@ -95,6 +96,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java index c33fd5f703a98..c6002304b1070 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java @@ -85,6 +85,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case STRING: case TEXT: case BLOB: + case OBJECT: addBinaryInput(arguments, mask); return; case BOOLEAN: @@ -350,6 +351,7 @@ private void writeX(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -383,6 +385,7 @@ private void updateX(Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xResult.setBinary(xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -451,6 +454,7 @@ private void updateFromBytesIntermediateInput(byte[] bytes) { case STRING: case TEXT: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -494,6 +498,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java index 355acba8aa6f7..92ea803f2d0ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java @@ -79,6 +79,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: @@ -106,6 +107,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java index 37f2f5d912dc5..99d5fef12089f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java @@ -54,6 +54,7 @@ public static void serializeValue( case TEXT: case STRING: case BLOB: + case OBJECT: BytesUtils.intToBytes(value.getBinary().getValues().length, valueBytes, offset); offset += 4; System.arraycopy( @@ -116,6 +117,7 @@ public static int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { return 8; case TEXT: case BLOB: + case OBJECT: case STRING: return 4 + value.getBinary().getValues().length; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java index 8ae7faa2c4600..f078ddc65a0c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java @@ -78,6 +78,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask, hlls, maxStandardError); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java index 2947c258e79ed..4e177c352b7de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java @@ -77,6 +77,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java index 8d46ab274f4c3..f53ea59314231 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java @@ -62,6 +62,7 @@ public GroupedExtremeAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -92,6 +93,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: break; case BOOLEAN: break; @@ -124,6 +126,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -150,6 +153,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -183,6 +187,7 @@ public void addIntermediate(int[] groupIds, Column argument) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -214,6 +219,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -245,6 +251,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -277,6 +284,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java index a1884162c4948..b8fe8bbe1b224 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java @@ -78,6 +78,7 @@ public GroupedFirstAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -110,6 +111,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -144,6 +146,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -176,6 +179,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -225,6 +229,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -276,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -313,6 +319,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: @@ -356,6 +363,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = binaryValues.get(groupId).getValues(); length += Integer.BYTES + values.length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java index 33ee0f9087395..d56ba59ef946b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java @@ -87,6 +87,7 @@ public GroupedFirstByAccumulator(TSDataType xDataType, TSDataType yDataType) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -120,6 +121,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -156,6 +158,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -192,6 +195,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -227,6 +231,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -288,6 +293,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -343,6 +349,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, Long.BYTES + 1); @@ -373,6 +380,7 @@ private int calculateValueLength(int groupId) { return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; case BOOLEAN: @@ -407,6 +415,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java index 75cc674ef967f..b21a683ab0a1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java @@ -78,6 +78,7 @@ public GroupedLastAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -110,6 +111,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -144,6 +146,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -176,6 +179,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -225,6 +229,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -276,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -313,6 +319,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: @@ -356,6 +363,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = binaryValues.get(groupId).getValues(); length += Integer.BYTES + values.length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java index 386d4c2f34f03..db48d221f18b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java @@ -87,6 +87,7 @@ public GroupedLastByAccumulator(TSDataType xDataType, TSDataType yDataType) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -120,6 +121,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -156,6 +158,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -192,6 +195,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -227,6 +231,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -288,6 +293,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -343,6 +349,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, Long.BYTES + 1); @@ -373,6 +380,7 @@ private int calculateValueLength(int groupId) { return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; case BOOLEAN: @@ -407,6 +415,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java index 3b538f0244bae..ed47c96d25137 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java @@ -69,6 +69,7 @@ public GroupedMaxAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -101,6 +102,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -135,6 +137,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -167,6 +170,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); return; case BOOLEAN: @@ -204,6 +208,7 @@ public void addIntermediate(int[] groupIds, Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryValue(groupIds[i], argument.getBinary(i)); break; case BOOLEAN: @@ -240,6 +245,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -275,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -311,6 +318,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java index 7cfe93629eaea..b9b25f6271552 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java @@ -96,6 +96,7 @@ protected GroupedMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -124,6 +125,7 @@ protected GroupedMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: yBinaryValues = new BinaryBigArray(); break; @@ -157,6 +159,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -185,6 +188,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += yBinaryValues.sizeOf(); break; case BOOLEAN: @@ -220,6 +224,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); break; case BOOLEAN: @@ -247,6 +252,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: yBinaryValues.ensureCapacity(groupCount); break; case BOOLEAN: @@ -282,6 +288,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -310,6 +317,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: yBinaryValues.reset(); break; @@ -341,6 +349,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: addBinaryInput(groupIds, arguments, mask); return; @@ -596,6 +605,7 @@ private void writeX(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; case BOOLEAN: @@ -630,6 +640,7 @@ private void updateX(int groupId, Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.set(groupId, xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -692,6 +703,7 @@ private void writeIntermediate( case TEXT: case STRING: case BLOB: + case OBJECT: byte[] values = isX ? xBinaryValues.get(groupId).getValues() : yBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, offset); @@ -725,6 +737,7 @@ private int calculateValueLength(int groupId, TSDataType dataType, boolean isX) return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + (isX @@ -774,6 +787,7 @@ private void updateFromBytesIntermediateInput(int groupId, byte[] bytes) { case STRING: case TEXT: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -818,6 +832,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java index 1c7206928ff79..8fa9b4d8fa018 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java @@ -69,6 +69,7 @@ public GroupedMinAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -101,6 +102,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -135,6 +137,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -167,6 +170,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); return; case BOOLEAN: @@ -203,6 +207,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryValue(groupIds[i], argument.getBinary(i)); break; @@ -240,6 +245,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case STRING: case TEXT: case BLOB: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -274,6 +280,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -311,6 +318,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java index 697a7d3b44a4a..a5bf82c5f409b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java @@ -95,6 +95,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); break; default: @@ -166,6 +167,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(maxEntry.getKey().getBinary()); break; default: @@ -270,6 +272,7 @@ private byte[] serializeCountMap(int groupId) { case TEXT: case STRING: case BLOB: + case OBJECT: bytes = new byte [offset @@ -366,6 +369,7 @@ private void deserializeAndMergeCountMap(int groupId, byte[] bytes) { case TEXT: case STRING: case BLOB: + case OBJECT: for (int i = 0; i < size; i++) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java index 4734e7c36718f..0c0325ebd0b30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java @@ -70,6 +70,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java index ea8dead07bdd6..cb6123b636f33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java @@ -83,6 +83,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java index 3c3fc6e616aaa..ae4975dff9ad9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java @@ -70,6 +70,7 @@ private static VariationWindowManager genEqualEventWindowManager( case BOOLEAN: return new EqualBooleanWindowManager(eventWindowParameter, ascending); case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -97,6 +98,7 @@ private static VariationWindowManager genVariationEventWindowManager( case STRING: case BOOLEAN: case BLOB: + case OBJECT: case TEXT: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index d0b7e35e94ce4..d766570e903b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; import org.apache.iotdb.db.queryengine.plan.relational.function.arithmetic.AdditionResolver; @@ -133,6 +134,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrim2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrimColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RadiansColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLike2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLikeColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Replace2ColumnTransformer; @@ -177,6 +179,8 @@ import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.utils.Binary; +import javax.annotation.Nullable; + import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -199,6 +203,7 @@ import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.apache.tsfile.read.common.type.ObjectType.OBJECT; import static org.apache.tsfile.read.common.type.StringType.STRING; public class ColumnTransformerBuilder @@ -999,6 +1004,29 @@ private ColumnTransformer getFunctionColumnTransformer( Type returnType = columnTransformers.get(0).getType(); return AbstractGreatestLeastColumnTransformer.getLeastColumnTransformer( returnType, columnTransformers); + } else if (TableBuiltinScalarFunction.READ_OBJECT + .getFunctionName() + .equalsIgnoreCase(functionName)) { + ColumnTransformer first = this.process(children.get(0), context); + if (children.size() == 1) { + return new ReadObjectColumnTransformer(OBJECT, first, context.fragmentInstanceContext); + } else if (children.size() == 2) { + return new ReadObjectColumnTransformer( + OBJECT, + ((LongLiteral) children.get(1)).getParsedValue(), + first, + context.fragmentInstanceContext); + } else { + long offset = ((LongLiteral) children.get(1)).getParsedValue(); + long length = ((LongLiteral) children.get(2)).getParsedValue(); + checkArgument(offset >= 0 && length >= 0); + return new ReadObjectColumnTransformer( + OBJECT, + ((LongLiteral) children.get(1)).getParsedValue(), + ((LongLiteral) children.get(2)).getParsedValue(), + first, + context.fragmentInstanceContext); + } } else { // user defined function if (TableUDFUtils.isScalarFunction(functionName)) { @@ -1463,6 +1491,8 @@ public static class Context { private final Metadata metadata; + private final Optional fragmentInstanceContext; + public Context( SessionInfo sessionInfo, List leafList, @@ -1473,7 +1503,8 @@ public Context( List inputDataTypes, int originSize, TypeProvider typeProvider, - Metadata metadata) { + Metadata metadata, + @Nullable FragmentInstanceContext fragmentInstanceContext) { this.sessionInfo = sessionInfo; this.leafList = leafList; this.inputLocations = inputLocations; @@ -1484,6 +1515,7 @@ public Context( this.originSize = originSize; this.typeProvider = typeProvider; this.metadata = metadata; + this.fragmentInstanceContext = Optional.ofNullable(fragmentInstanceContext); } public Type getType(SymbolReference symbolReference) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 132f5ebcc0087..811af011a9089 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.operator.EmptyDataOperator; import org.apache.iotdb.db.queryengine.execution.operator.ExplainAnalyzeOperator; import org.apache.iotdb.db.queryengine.execution.operator.Operator; @@ -1311,13 +1312,15 @@ private Operator constructFilterAndProjectOperator( ColumnTransformerBuilder visitor = new ColumnTransformerBuilder(); + FragmentInstanceContext fragmentInstanceContext = + context.getDriverContext().getFragmentInstanceContext(); ColumnTransformer filterOutputTransformer = predicate .map( p -> { ColumnTransformerBuilder.Context filterColumnTransformerContext = new ColumnTransformerBuilder.Context( - context.getDriverContext().getFragmentInstanceContext().getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), filterLeafColumnTransformerList, inputLocations, filterExpressionColumnTransformerMap, @@ -1326,7 +1329,8 @@ private Operator constructFilterAndProjectOperator( ImmutableList.of(), 0, context.getTypeProvider(), - metadata); + metadata, + fragmentInstanceContext); return visitor.process(p, filterColumnTransformerContext); }) @@ -1344,7 +1348,7 @@ private Operator constructFilterAndProjectOperator( ColumnTransformerBuilder.Context projectColumnTransformerContext = new ColumnTransformerBuilder.Context( - context.getDriverContext().getFragmentInstanceContext().getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), projectLeafColumnTransformerList, inputLocations, projectExpressionColumnTransformerMap, @@ -1353,7 +1357,8 @@ private Operator constructFilterAndProjectOperator( filterOutputDataTypes, inputLocations.size(), context.getTypeProvider(), - metadata); + metadata, + fragmentInstanceContext); for (Expression expression : projectExpressions) { projectOutputTransformerList.add( @@ -2426,6 +2431,8 @@ public Operator visitTableDeviceQueryCount( // In "count" we have to reuse filter operator per "next" final List filterLeafColumnTransformerList = new ArrayList<>(); + FragmentInstanceContext fragmentInstanceContext = + context.getDriverContext().getFragmentInstanceContext(); return new SchemaCountOperator<>( node.getPlanNodeId(), context @@ -2447,10 +2454,7 @@ public Operator visitTableDeviceQueryCount( .process( node.getTagFuzzyPredicate(), new ColumnTransformerBuilder.Context( - context - .getDriverContext() - .getFragmentInstanceContext() - .getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), filterLeafColumnTransformerList, makeLayout(Collections.singletonList(node)), new HashMap<>(), @@ -2459,7 +2463,8 @@ public Operator visitTableDeviceQueryCount( ImmutableList.of(), 0, context.getTypeProvider(), - metadata)), + metadata, + fragmentInstanceContext)), columnSchemaList, database, table) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 2f157a9086cff..98936da12bb29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -111,6 +111,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -312,6 +313,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROW((short) 2001), RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), + OBJECT_FILE_NODE((short) 2004), ; public static final int BYTES = Short.BYTES; @@ -355,6 +357,8 @@ public static PlanNode deserializeFromWAL(DataInputStream stream) throws IOExcep return RelationalInsertRowsNode.deserializeFromWAL(stream); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(stream); + case 2004: + return ObjectNode.deserializeFromWAL(stream); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -381,6 +385,8 @@ public static PlanNode deserializeFromWAL(ByteBuffer buffer) { return RelationalInsertRowsNode.deserializeFromWAL(buffer); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(buffer); + case 2004: + return ObjectNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -697,6 +703,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return RelationalInsertRowsNode.deserialize(buffer); case 2003: return RelationalDeleteDataNode.deserialize(buffer); + case 2004: + return ObjectNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 4ff7f9f62cbd5..609aacf866257 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -115,6 +115,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -620,6 +621,10 @@ public R visitDeleteData(RelationalDeleteDataNode node, C context) { return visitPlan(node, context); } + public R visitWriteObjectFile(ObjectNode node, C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Pipe Related Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index c8b840cc6bba1..3d24ffcbc1193 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -95,10 +95,26 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue { // proper positions. protected List range; + private Boolean shouldCheckTTL; + public InsertTabletNode(PlanNodeId id) { super(id); } + public boolean shouldCheckTTL() { + if (shouldCheckTTL != null) { + return shouldCheckTTL; + } + shouldCheckTTL = true; + for (MeasurementSchema measurementSchema : measurementSchemas) { + if (measurementSchema.getType() == TSDataType.OBJECT) { + shouldCheckTTL = false; + break; + } + } + return shouldCheckTTL; + } + @Override public InsertNode mergeInsertNode(List insertNodes) { List index = new ArrayList<>(); @@ -220,7 +236,6 @@ public List splitByPartition(IAnalysis analysis) { final Map deviceIDSplitInfoMap = collectSplitRanges(); final Map> splitMap = splitByReplicaSet(deviceIDSplitInfoMap, analysis); - return doSplit(splitMap); } @@ -290,7 +305,7 @@ protected Map> splitByReplicaSet( return splitMap; } - private List doSplit(Map> splitMap) { + protected List doSplit(Map> splitMap) { List result = new ArrayList<>(); if (splitMap.size() == 1) { @@ -327,7 +342,7 @@ protected InsertTabletNode getEmptySplit(int count) { subTimes.length); } - private WritePlanNode generateOneSplit(Map.Entry> entry) { + protected WritePlanNode generateOneSplit(Map.Entry> entry) { List locs; // generate a new times and values locs = entry.getValue(); @@ -388,6 +403,7 @@ protected Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] da case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = new Binary[rowSize]; break; case FLOAT: @@ -641,6 +657,7 @@ private void serializeColumn(TSDataType dataType, Object column, ByteBuffer buff case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = 0; j < rowCount; j++) { if (binaryValues[j] != null) { @@ -693,6 +710,7 @@ private void serializeColumn(TSDataType dataType, Object column, DataOutputStrea case STRING: case TEXT: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = 0; j < rowCount; j++) { if (binaryValues[j] != null) { @@ -833,6 +851,7 @@ private int getColumnSize(TSDataType dataType, Object column, int start, int end case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = start; j < end; j++) { size += ReadWriteIOUtils.sizeToWrite(binaryValues[j]); @@ -964,6 +983,7 @@ private void serializeColumn( case STRING: case TEXT: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = start; j < end; j++) { if (binaryValues[j] != null) { @@ -1127,6 +1147,7 @@ private boolean equals(Object[] columns) { case TEXT: case BLOB: case STRING: + case OBJECT: if (!Arrays.equals((Binary[]) this.columns[i], (Binary[]) columns[i])) { return false; } @@ -1199,6 +1220,8 @@ public TimeValuePair composeLastTimeValuePair( Binary[] binaryValues = (Binary[]) columns[measurementIndex]; value = new TsPrimitiveType.TsBinary(binaryValues[lastIdx]); break; + case OBJECT: + return null; default: throw new UnSupportedDataTypeException( String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex])); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java new file mode 100644 index 0000000000000..ebe839bbc0c6d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.exception.ObjectFileNotExist; +import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Optional; + +public class ObjectNode extends SearchNode implements WALEntryValue { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectNode.class); + + private final boolean isEOF; + + private final long offset; + + private byte[] content; + + private String filePath; + + private final int contentLength; + + private TRegionReplicaSet dataRegionReplicaSet; + + private boolean isGeneratedByRemoteConsensusLeader; + + private Long time = null; + + private String table = null; + + public ObjectNode(boolean isEOF, long offset, byte[] content, String filePath) { + super(new PlanNodeId("")); + this.isEOF = isEOF; + this.offset = offset; + this.filePath = filePath; + this.content = content; + this.contentLength = content.length; + } + + public ObjectNode(boolean isEOF, long offset, int contentLength, String filePath) { + super(new PlanNodeId("")); + this.isEOF = isEOF; + this.offset = offset; + this.filePath = filePath; + this.contentLength = contentLength; + } + + public long getTimestamp() { + calculateTimeAndTableName(); + return time; + } + + public String getTable() { + calculateTimeAndTableName(); + return table; + } + + private void calculateTimeAndTableName() { + if (time != null && table != null) { + return; + } + File file = new File(filePath); + String fileName = new File(filePath).getName(); + String timeStr = fileName.substring(0, fileName.length() - ".bin".length()); + time = Long.parseLong(timeStr); + table = file.getParentFile().getParentFile().getParentFile().getParentFile().getName(); + } + + public boolean isEOF() { + return isEOF; + } + + public byte[] getContent() { + return content; + } + + public long getOffset() { + return offset; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public String getFilePath() { + return filePath; + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer) { + buffer.putShort(getType().getNodeType()); + buffer.putLong(searchIndex); + buffer.put((byte) (isEOF ? 1 : 0)); + buffer.putLong(offset); + WALWriteUtils.write(filePath, buffer); + buffer.putInt(content.length); + } + + @Override + public int serializedSize() { + return Short.BYTES + + Long.BYTES + + Byte.BYTES + + Long.BYTES + + Integer.BYTES + + ReadWriteIOUtils.sizeToWrite(filePath); + } + + public static ObjectNode deserializeFromWAL(DataInputStream stream) throws IOException { + long searchIndex = stream.readLong(); + boolean isEOF = stream.readByte() == 1; + long offset = stream.readLong(); + String filePath = ReadWriteIOUtils.readString(stream); + int contentLength = stream.readInt(); + ObjectNode objectNode = new ObjectNode(isEOF, offset, contentLength, filePath); + objectNode.setSearchIndex(searchIndex); + return objectNode; + } + + public static ObjectNode deserializeFromWAL(ByteBuffer buffer) { + long searchIndex = buffer.getLong(); + boolean isEOF = buffer.get() == 1; + long offset = buffer.getLong(); + String filePath = ReadWriteIOUtils.readString(buffer); + Optional objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath); + int contentLength = buffer.getInt(); + byte[] contents = new byte[contentLength]; + if (objectFile.isPresent()) { + try (RandomAccessFile raf = new RandomAccessFile(objectFile.get(), "r")) { + raf.seek(offset); + raf.read(contents); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new ObjectFileNotExist(filePath); + } + + ObjectNode objectNode = new ObjectNode(isEOF, offset, contents, filePath); + objectNode.setSearchIndex(searchIndex); + return objectNode; + } + + public static ObjectNode deserialize(ByteBuffer byteBuffer) { + boolean isEoF = ReadWriteIOUtils.readBool(byteBuffer); + long offset = ReadWriteIOUtils.readLong(byteBuffer); + String filePath = ReadWriteIOUtils.readString(byteBuffer); + int contentLength = ReadWriteIOUtils.readInt(byteBuffer); + byte[] content = ReadWriteIOUtils.readBytes(byteBuffer, contentLength); + return new ObjectNode(isEoF, offset, content, filePath); + } + + @Override + public SearchNode merge(List searchNodes) { + if (searchNodes.size() == 1) { + return searchNodes.get(0); + } + throw new UnsupportedOperationException("Merge is not supported"); + } + + @Override + public ProgressIndex getProgressIndex() { + return null; + } + + @Override + public void setProgressIndex(ProgressIndex progressIndex) {} + + @Override + public List splitByPartition(IAnalysis analysis) { + return null; + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return dataRegionReplicaSet; + } + + public void setDataRegionReplicaSet(TRegionReplicaSet dataRegionReplicaSet) { + this.dataRegionReplicaSet = dataRegionReplicaSet; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return null; + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + getType().serialize(byteBuffer); + ReadWriteIOUtils.write(isEOF, byteBuffer); + ReadWriteIOUtils.write(offset, byteBuffer); + ReadWriteIOUtils.write(filePath, byteBuffer); + ReadWriteIOUtils.write(contentLength, byteBuffer); + byteBuffer.put(content); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + getType().serialize(stream); + ReadWriteIOUtils.write(isEOF, stream); + ReadWriteIOUtils.write(offset, stream); + ReadWriteIOUtils.write(filePath, stream); + ReadWriteIOUtils.write(contentLength, stream); + stream.write(content); + } + + public ByteBuffer serialize() { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream stream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(WALEntryType.OBJECT_FILE_NODE.getCode(), stream); + ReadWriteIOUtils.write((long) TsFileProcessor.MEMTABLE_NOT_EXIST, stream); + ReadWriteIOUtils.write(getType().getNodeType(), stream); + byte[] contents = new byte[contentLength]; + boolean readSuccess = false; + for (int i = 0; i < 2; i++) { + Optional objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath); + if (objectFile.isPresent()) { + try { + readContentFromFile(objectFile.get(), contents); + readSuccess = true; + } catch (IOException e) { + LOGGER.error("Error when read object file {}.", objectFile.get(), e); + } + if (readSuccess) { + break; + } + } + Optional objectTmpFile = + TierManager.getInstance().getAbsoluteObjectFilePath(filePath + ".tmp"); + if (objectTmpFile.isPresent()) { + try { + readContentFromFile(objectTmpFile.get(), contents); + readSuccess = true; + } catch (IOException e) { + LOGGER.error("Error when read tmp object file {}.", objectTmpFile.get(), e); + } + if (readSuccess) { + break; + } + } + } + ReadWriteIOUtils.write(readSuccess && isEOF, stream); + ReadWriteIOUtils.write(offset, stream); + ReadWriteIOUtils.write(filePath, stream); + ReadWriteIOUtils.write(contentLength, stream); + stream.write(contents); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } catch (IOException e) { + throw new SerializationRunTimeException(e); + } + } + + private void readContentFromFile(File file, byte[] contents) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(file, "r")) { + raf.seek(offset); + raf.read(contents); + } + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.OBJECT_FILE_NODE; + } + + @Override + public long getMemorySize() { + return contentLength; + } + + @Override + public void markAsGeneratedByRemoteConsensusLeader() { + isGeneratedByRemoteConsensusLeader = true; + } + + public boolean isGeneratedByRemoteConsensusLeader() { + return isGeneratedByRemoteConsensusLeader; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitWriteObjectFile(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 0d4b698108ee1..58047ca56646d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -30,14 +30,18 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -46,10 +50,12 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; public class RelationalInsertTabletNode extends InsertTabletNode { @@ -58,6 +64,8 @@ public class RelationalInsertTabletNode extends InsertTabletNode { private boolean singleDevice; + private Object[] convertedColumns; + public RelationalInsertTabletNode( PlanNodeId id, PartialPath devicePath, @@ -108,6 +116,16 @@ public void setSingleDevice() { this.singleDevice = true; } + public List getObjectColumns() { + List objectColumns = new ArrayList<>(); + for (int i = 0; i < columns.length; i++) { + if (dataTypes[i] == TSDataType.OBJECT) { + objectColumns.add((Binary[]) columns[i]); + } + } + return objectColumns; + } + @Override public IDeviceID getDeviceID(int rowIdx) { if (singleDevice) { @@ -372,4 +390,60 @@ public void updateLastCache(final String databaseName) { startOffset = endOffset; } } + + @Override + protected List doSplit(Map> splitMap) { + List result = new ArrayList<>(); + + if (splitMap.size() == 1) { + final Entry> entry = splitMap.entrySet().iterator().next(); + if (entry.getValue().size() == 2) { + // Avoid using system arraycopy when there is no need to split + setRange(entry.getValue()); + setDataRegionReplicaSet(entry.getKey()); + for (int i = 0; i < columns.length; i++) { + if (dataTypes[i] == TSDataType.OBJECT) { + for (int j = 0; j < times.length; j++) { + byte[] binary = ((Binary[]) columns[i])[j].getValues(); + ByteBuffer buffer = ByteBuffer.wrap(binary); + boolean isEoF = buffer.get() == 1; + long offset = buffer.getLong(); + byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining()); + String relativePath = + TsFileNameGenerator.generateObjectFilePath( + dataRegionReplicaSet.regionId.getId(), times[j], getDeviceID(j)); + ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath); + objectNode.setDataRegionReplicaSet(entry.getKey()); + result.add(objectNode); + if (isEoF) { + byte[] filePathBytes = relativePath.getBytes(StandardCharsets.UTF_8); + byte[] valueBytes = new byte[filePathBytes.length + Long.BYTES]; + System.arraycopy( + BytesUtils.longToBytes(offset + content.length), 0, valueBytes, 0, Long.BYTES); + System.arraycopy(filePathBytes, 0, valueBytes, Long.BYTES, filePathBytes.length); + ((Binary[]) columns[i])[j] = new Binary(valueBytes); + } else { + ((Binary[]) columns[i])[j] = null; + if (bitMaps == null) { + bitMaps = new BitMap[columns.length]; + } + if (bitMaps[i] == null) { + bitMaps[i] = new BitMap(rowCount); + } + bitMaps[i].mark(j); + } + } + } + } + result.add(this); + return result; + } + } + + for (Map.Entry> entry : splitMap.entrySet()) { + // TODO: add ObjectNode for split + result.add(generateOneSplit(entry)); + } + return result; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/scalar/GeoPenetrate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/scalar/GeoPenetrate.java new file mode 100644 index 0000000000000..45e512ebd2739 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/scalar/GeoPenetrate.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.function.scalar; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.utils.model.ModelReader; +import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; +import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; +import org.apache.iotdb.udf.api.exception.UDFArgumentNotValidException; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.relational.ScalarFunction; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.iotdb.db.utils.ObjectTypeUtils.getObjectPathFromBinary; + +public class GeoPenetrate implements ScalarFunction { + + private ModelReader reader; + private List> startAndEndTimeArray; + + @Override + public ScalarFunctionAnalysis analyze(FunctionArguments arguments) + throws UDFArgumentNotValidException { + if (arguments.getArgumentsSize() != 2 && arguments.getArgumentsSize() != 3) { + throw new UDFArgumentNotValidException( + "function requires 2 or 3 arguments, current parameters' size is " + + arguments.getArgumentsSize()); + } + if (arguments.getDataType(0) != Type.OBJECT) { + throw new UDFArgumentNotValidException("function's first parameter type should be OBJECT"); + } + + if (arguments.getDataType(1) != Type.STRING) { + throw new UDFArgumentNotValidException("function's second parameter type should be STRING"); + } + + if (arguments.getArgumentsSize() == 3 && arguments.getDataType(1) != Type.STRING) { + throw new UDFArgumentNotValidException("function's third parameter type should be STRING"); + } + + return new ScalarFunctionAnalysis.Builder().outputDataType(Type.BLOB).build(); + } + + @Override + public void beforeStart(FunctionArguments arguments) throws UDFException { + if (arguments.getArgumentsSize() == 2) { + reader = ModelReader.getDefaultInstance(); + } + } + + @Override + public Object evaluate(Record input) throws UDFException { + if (reader == null) { + reader = ModelReader.getInstance(input.getString(2)); + } + if (startAndEndTimeArray == null) { + startAndEndTimeArray = getStartAndEndTimeArray(input.getString(1)); + } + if (input.isNull(0)) { + return null; + } else { + File objectPath = getObjectPathFromBinary(input.getBinary(0)); + List res = reader.penetrate(objectPath.getAbsolutePath(), startAndEndTimeArray); + int count = 0; + for (float[] array : res) { + count += array.length; + } + ByteBuffer byteBuffer = ByteBuffer.allocate(count * Float.BYTES); + for (float[] array : res) { + for (float value : array) { + ReadWriteIOUtils.write(value, byteBuffer); + } + } + return new Binary(byteBuffer.array()); + } + } + + // argument like: 1,2;200,201;300,401 + private List> getStartAndEndTimeArray(String argument) { + String[] arguments = argument.split(","); + if (arguments.length % 2 != 0) { + throw new SemanticException("second argument is wrong: " + argument); + } + List> res = new ArrayList<>(arguments.length); + for (int index = 0; index < arguments.length; index += 2) { + List startAndEndTime = new ArrayList<>(2); + try { + startAndEndTime.add(Integer.parseInt(arguments[index])); + startAndEndTime.add(Integer.parseInt(arguments[index + 1])); + } catch (NumberFormatException e) { + throw new SemanticException("second argument is wrong: " + argument); + } + res.add(startAndEndTime); + } + return res; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 8934de172e9c5..40ef4c4bb5464 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -65,7 +65,7 @@ import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.read.common.type.BlobType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeFactory; @@ -79,6 +79,7 @@ import static org.apache.iotdb.db.queryengine.transformation.dag.column.FailFunctionColumnTransformer.FAIL_FUNCTION_NAME; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; +import static org.apache.tsfile.read.common.type.BlobType.BLOB; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DateType.DATE; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -570,6 +571,20 @@ && isIntegerNumber(argumentTypes.get(2)))) { + " must have at least two arguments, and all type must be the same."); } return argumentTypes.get(0); + } else if (TableBuiltinScalarFunction.READ_OBJECT + .getFunctionName() + .equalsIgnoreCase(functionName)) { + if (argumentTypes.isEmpty() + || argumentTypes.size() > 3 + || !isObjectType(argumentTypes.get(0)) + || (argumentTypes.size() >= 2 && !isIntegerNumber(argumentTypes.get(1))) + || (argumentTypes.size() >= 3 && !isIntegerNumber(argumentTypes.get(2)))) { + throw new SemanticException( + "Scalar function " + + functionName.toLowerCase(Locale.ENGLISH) + + " must have at 1~3 arguments, and first argument must be OBJECT type, other arguments must be int32 or int64 type"); + } + return BLOB; } // builtin aggregation function @@ -962,8 +977,12 @@ public static boolean isCharType(Type type) { return TEXT.equals(type) || StringType.STRING.equals(type); } + public static boolean isObjectType(Type type) { + return ObjectType.OBJECT.equals(type); + } + public static boolean isBlobType(Type type) { - return BlobType.BLOB.equals(type); + return BLOB.equals(type); } public static boolean isBool(Type type) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java index 10935e97d4eb2..4a73f4b792fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java @@ -227,7 +227,8 @@ public static DeviceBlackListConstructor constructDevicePredicateUpdater( ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; return new DeviceBlackListConstructor( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index 6f2aaef76ab14..5719a4555ff12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -35,6 +35,7 @@ import static org.apache.tsfile.read.common.type.FloatType.FLOAT; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.apache.tsfile.read.common.type.ObjectType.OBJECT; import static org.apache.tsfile.read.common.type.StringType.STRING; import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -52,6 +53,7 @@ public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.TEXT.name().toLowerCase(Locale.ENGLISH)), TEXT); types.put(new TypeSignature(TypeEnum.STRING.name().toLowerCase(Locale.ENGLISH)), STRING); types.put(new TypeSignature(TypeEnum.BLOB.name().toLowerCase(Locale.ENGLISH)), BLOB); + types.put(new TypeSignature(TypeEnum.OBJECT.name().toLowerCase(Locale.ENGLISH)), OBJECT); types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE); types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP); types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN); @@ -105,6 +107,8 @@ public static TSDataType getTSDataType(Type type) { return TSDataType.BLOB; case STRING: return TSDataType.STRING; + case OBJECT: + return TSDataType.OBJECT; default: throw new IllegalArgumentException(); } @@ -132,6 +136,8 @@ public static Type fromTSDataType(TSDataType dataType) { return TIMESTAMP; case BLOB: return BLOB; + case OBJECT: + return OBJECT; case STRING: return STRING; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java index 0cc31381072f3..fe951d1e4ce10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java @@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.DoubleType; import org.apache.tsfile.read.common.type.FloatType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; @@ -116,6 +117,8 @@ public static Type getType(TypeEnum typeEnum, List subTypes) { return TimestampType.TIMESTAMP; case DATE: return DATE; + case OBJECT: + return ObjectType.OBJECT; case ROW: return RowType.anonymous(subTypes); default: @@ -123,7 +126,6 @@ public static Type getType(TypeEnum typeEnum, List subTypes) { } } - // TODO move these methods into each Type to avoid branch miss public static boolean isFlatVariableWidth(Type type) { switch (type.getTypeEnum()) { case BOOLEAN: @@ -137,6 +139,7 @@ public static boolean isFlatVariableWidth(Type type) { case TEXT: case STRING: case BLOB: + case OBJECT: return true; default: throw new UnsupportedOperationException(); @@ -160,6 +163,7 @@ public static int getFlatFixedSize(Type type) { case TEXT: case STRING: case BLOB: + case OBJECT: return 16; default: throw new UnsupportedOperationException(); @@ -179,6 +183,7 @@ public static int getFlatVariableWidthSize(Type type, Column column, int positio case TEXT: case STRING: case BLOB: + case OBJECT: return column.isNull(position) ? 0 : column.getBinary(position).getLength(); default: throw new UnsupportedOperationException(); @@ -198,6 +203,7 @@ public static int getFlatVariableWidthSize(Type type, Column column, int[] posit case TEXT: case STRING: case BLOB: + case OBJECT: int result = 0; for (int i = 0; i < position.length; i++) { if (!column.isNull(i)) { @@ -237,6 +243,7 @@ public static void readFlat( case TEXT: case STRING: case BLOB: + case OBJECT: int length = bytesToInt(fixedChunk, fixedOffset); byte[] result = new byte[length]; if (length <= 12) { @@ -282,6 +289,7 @@ public static void writeFlat( case TEXT: case STRING: case BLOB: + case OBJECT: byte[] value = column.getBinary(position).getValues(); intToBytes(value.length, fixedChunk, fixedOffset); if (value.length <= 12) { @@ -320,6 +328,7 @@ public static boolean notDistinctFrom( case TEXT: case STRING: case BLOB: + case OBJECT: int leftLength = bytesToInt(fixedChunk, fixedOffset); byte[] leftValue = new byte[leftLength]; byte[] rightValue = column.getBinary(position).getValues(); @@ -414,6 +423,7 @@ public static long hash(Type type, byte[] fixedChunk, int fixedOffset, byte[] va case TEXT: case STRING: case BLOB: + case OBJECT: int length = bytesToInt(fixedChunk, fixedOffset); byte[] values = new byte[length]; @@ -490,6 +500,7 @@ public static long hash(Type type, Column column, int position) { case TEXT: case STRING: case BLOB: + case OBJECT: return XxHash64.hash(column.getBinary(position).getValues()); default: throw new UnsupportedOperationException(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java new file mode 100644 index 0000000000000..84a094e39fe69 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; +import org.apache.iotdb.db.utils.ObjectTypeUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.util.Optional; + +public class ReadObjectColumnTransformer extends UnaryColumnTransformer { + + private final Optional fragmentInstanceContext; + private long offset = 0; + private long length = -1; + + public ReadObjectColumnTransformer( + Type type, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.fragmentInstanceContext = fragmentInstanceContext; + } + + public ReadObjectColumnTransformer( + Type type, + long offset, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.offset = offset; + this.fragmentInstanceContext = fragmentInstanceContext; + } + + public ReadObjectColumnTransformer( + Type type, + long offset, + long length, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.offset = offset; + this.length = length; + this.fragmentInstanceContext = fragmentInstanceContext; + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (!column.isNull(i)) { + transform(column, columnBuilder, i); + } else { + columnBuilder.appendNull(); + } + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (selection[i] && !column.isNull(i)) { + transform(column, columnBuilder, i); + } else { + columnBuilder.appendNull(); + } + } + } + + private void transform(Column column, ColumnBuilder columnBuilder, int i) { + // BinaryColumn.getDataType() returns TSDataType.TEXT + if (TSDataType.TEXT == column.getDataType()) { + Binary binary = column.getBinary(i); + columnBuilder.writeBinary(readObject(binary)); + } + } + + private Binary readObject(Binary binary) { + File file = ObjectTypeUtils.getObjectPathFromBinary(binary); + long fileSize = file.length(); + if (offset >= fileSize) { + throw new UnsupportedOperationException("offset is greater than object size"); + } + long actualReadSize = Math.min(length < 0 ? fileSize : length, fileSize - offset); + if (actualReadSize > Integer.MAX_VALUE) { + throw new UnsupportedOperationException("Read object size is too large (size > 2G)"); + } + fragmentInstanceContext.ifPresent( + context -> context.getMemoryReservationContext().reserveMemoryCumulatively(actualReadSize)); + byte[] bytes = new byte[(int) actualReadSize]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + fileChannel.read(buffer); + } catch (IOException e) { + throw new IoTDBRuntimeException(e, TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()); + } + return new Binary(bytes); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java index 32125c110ef57..14643a7b6ac50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java @@ -92,6 +92,7 @@ public ConstantInputReader(ConstantOperand expression) throws QueryProcessExcept cachedColumns[0] = new RunLengthEncodedColumn(booleanColumn, count); break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java index fcbb30b7ca139..59b3dbbc8b636 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java @@ -59,6 +59,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) case TEXT: case TIMESTAMP: case BLOB: + case OBJECT: case BOOLEAN: case STRING: default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java index 5bdfa8f96a7f7..09cc6e76ad380 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java @@ -88,6 +88,7 @@ private void initTypedSet(Set values) { stringSet = values; break; case BLOB: + case OBJECT: default: throw new UnsupportedOperationException("unsupported data type: " + layerReaderDataType); } @@ -124,6 +125,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) transformBinary(columns, builder); return; case BLOB: + case OBJECT: default: throw new QueryProcessException("unsupported data type: " + layerReaderDataType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java index 11cf952e0b435..195acb6de8527 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java @@ -68,6 +68,7 @@ protected Column[] transform(Column[] columns) throws QueryProcessException, IOE case TEXT: return castBinaries(columns); case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -139,6 +140,7 @@ private Column[] castInts(Column[] columns) { break; case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case DATE: case INT32: @@ -213,6 +215,7 @@ private Column[] castLongs(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case DATE: case TIMESTAMP: @@ -288,6 +291,7 @@ private Column[] castFloats(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -363,6 +367,7 @@ private Column[] castDoubles(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -439,6 +444,7 @@ private Column[] castBooleans(Column[] columns) { break; case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: case BOOLEAN: @@ -521,6 +527,7 @@ private Column[] castBinaries(Column[] columns) { case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java index ee7ba10e60e49..03081f9206f1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java @@ -62,6 +62,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws QueryProce transformDouble(columns, builder); return; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java index a4ddff58bab46..47e4261807730 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java @@ -67,6 +67,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) case STRING: case TEXT: case BLOB: + case OBJECT: default: throw new UnsupportedOperationException( String.format("Unsupported source dataType: %s", layerReaderDataType)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java index c05d7adfecc24..015904399c201 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java @@ -78,6 +78,7 @@ public static Column transformConstantOperandToColumn(ConstantOperand constantOp return new BooleanColumn(1, Optional.empty(), new boolean[] {(boolean) value}); case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: default: @@ -158,6 +159,7 @@ public static boolean splitWindowForStateWindow( case TIMESTAMP: case DATE: case BLOB: + case OBJECT: case STRING: default: throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java index 0eedb86a31a9e..e8c679e2bb55f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java @@ -50,6 +50,7 @@ public static ColumnBuilder initColumnBuilder(TSDataType type, int count) { case TEXT: case BLOB: case STRING: + case OBJECT: return new BinaryColumnBuilder(null, count); default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java index 6ec6537de362f..3bacd1ac5e79a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java @@ -107,6 +107,7 @@ protected static int calculateCapacity( case TEXT: case BLOB: case STRING: + case OBJECT: rowLength += MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl; break; @@ -214,6 +215,7 @@ private Object[] getRowSkipPrefixNulls(int index) { case TEXT: case BLOB: case STRING: + case OBJECT: row[i] = block[i].getBinary(offset); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java index d29926947fe50..de5c6b61a4e84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java @@ -93,6 +93,7 @@ protected static int calculateCapacity(TSDataType dataType, float memoryLimitInM case TEXT: case STRING: case BLOB: + case OBJECT: rowLength += MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 3f49dfec93d9d..d09f2204db89f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1523,7 +1523,8 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; final List filterOutputDataTypes = @@ -1550,7 +1551,8 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( filterOutputDataTypes, inputLocations.size(), mockTypeProvider, - metadata); + metadata, + null); final List attributeNames = updateNode.getAssignments().stream() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 2c2281dcf66f5..20111516d7cfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -76,12 +76,15 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager; import org.apache.iotdb.db.storageengine.load.LoadTsFileManager; import org.apache.iotdb.db.storageengine.load.limiter.LoadTsFileRateLimiter; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.ThreadUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.commons.io.FileUtils; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.tsfile.utils.FilePathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +92,8 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -105,6 +110,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -118,6 +124,8 @@ public class StorageEngine implements IService { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); + private final FSFactory fsFactory = FSFactoryProducer.getFSFactory(); + /** * a folder (system/databases/ by default) that persist system info. Each database will have a * subfolder under the systemDir. @@ -155,6 +163,8 @@ public class StorageEngine implements IService { private final LoadTsFileManager loadTsFileManager = new LoadTsFileManager(); + public final AtomicLong objectFileId = new AtomicLong(0); + private StorageEngine() {} public static StorageEngine getInstance() { @@ -231,6 +241,8 @@ private void asyncRecoverDataRegion() throws StartupException { } private void asyncRecover(List> futures) { + checkObjectFiles(); + Map> localDataRegionInfo = getLocalDataRegionInfo(); localDataRegionInfo.values().forEach(list -> recoverDataRegionNum += list.size()); readyDataRegionNum = new AtomicInteger(0); @@ -1065,6 +1077,32 @@ public static File getDataRegionSystemDir(String dataBaseName, String dataRegion systemDir + File.separator + dataBaseName, dataRegionId); } + private void checkObjectFiles() { + List folders = TierManager.getInstance().getAllObjectFileFolders(); + for (String baseDir : folders) { + File fileFolder = fsFactory.getFile(baseDir); + try (Stream paths = Files.walk(fileFolder.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin.back"); + }) + .forEach( + path -> { + try { + Files.delete(path); + } catch (IOException e) { + LOGGER.error("Failed to delete: {} -> {}", path, e.getMessage()); + } + }); + } catch (IOException e) { + LOGGER.error("Failed to check Object Files: {}", e.getMessage()); + } + } + } + public Runnable executeCompactFileTimeIndexCache() { return () -> { if (!isReadyForNonReadWriteFunctions()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 6a3b09174226f..166901363f6a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -76,6 +76,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy; @@ -150,6 +151,7 @@ import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectWriter; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -174,6 +176,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1301,7 +1304,10 @@ private boolean executeInsertTablet( InsertTabletNode insertTabletNode, TSStatus[] results, long[] infoForMetrics) throws OutOfTTLException { boolean noFailure; - int loc = insertTabletNode.checkTTL(results, getTTL(insertTabletNode)); + int loc = + insertTabletNode.shouldCheckTTL() + ? insertTabletNode.checkTTL(results, getTTL(insertTabletNode)) + : 0; noFailure = loc == 0; List> deviceEndOffsetPairs = insertTabletNode.splitByDevice(loc, insertTabletNode.getRowCount()); @@ -1835,6 +1841,7 @@ public void syncDeleteDataFiles() throws TsFileProcessorException { } }); deleteAllSGFolders(TierManager.getInstance().getAllFilesFolders()); + deleteAllObjectFiles(TierManager.getInstance().getAllObjectFileFolders()); this.workSequenceTsFileProcessors.clear(); this.workUnsequenceTsFileProcessors.clear(); this.tsFileManager.clear(); @@ -1872,6 +1879,23 @@ private void deleteAllSGFolders(List folder) { } } + private void deleteAllObjectFiles(List folders) { + for (String objectFolder : folders) { + File dataRegionObjectFolder = fsFactory.getFile(objectFolder, dataRegionId); + if (FSUtils.getFSType(dataRegionObjectFolder) != FSType.LOCAL) { + try { + fsFactory.deleteDirectory(dataRegionObjectFolder.getPath()); + } catch (IOException e) { + logger.error("Fail to delete data region object folder {}", dataRegionObjectFolder); + } + } else { + if (dataRegionObjectFolder.exists()) { + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(dataRegionObjectFolder); + } + } + } + } + public void timedFlushSeqMemTable() { int count = 0; writeLock("timedFlushSeqMemTable"); @@ -3026,6 +3050,43 @@ public int compact() { } } + public void writeObject(ObjectNode objectNode) throws Exception { + writeLock("writeObject"); + try { + String relativeTmpPathString = objectNode.getFilePath() + ".tmp"; + String objectFileDir = TierManager.getInstance().getNextFolderForObjectFile(); + File objectTmpFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeTmpPathString); + try (ObjectWriter writer = new ObjectWriter(objectTmpFile)) { + writer.write( + objectNode.isGeneratedByRemoteConsensusLeader(), + objectNode.getOffset(), + objectNode.getContent()); + } + if (objectNode.isEOF()) { + File objectFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, objectNode.getFilePath()); + if (objectFile.exists()) { + String relativeBackPathString = objectNode.getFilePath() + ".back"; + File objectBackFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeBackPathString); + Files.move( + objectFile.toPath(), objectBackFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move( + objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.delete(objectBackFile.toPath()); + } else { + Files.move( + objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + getWALNode() + .ifPresent(walNode -> walNode.log(TsFileProcessor.MEMTABLE_NOT_EXIST, objectNode)); + } finally { + writeUnlock(); + } + } + /** * Load a new tsfile to unsequence dir. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index e970b07f2f4a2..ce79e0a833496 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -142,8 +142,7 @@ public void perform() throws Exception { sortedSourceFiles.addAll(unseqFiles); boolean isTreeModel = !isAligned || device.getTableName().startsWith("root."); long ttl = deviceIterator.getTTLForCurrentDevice(); - sortedSourceFiles.removeIf( - x -> x.definitelyNotContains(device) || !x.isDeviceAlive(device, ttl)); + sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device)); // checked above //noinspection OptionalGetWithoutIsPresent sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device).get())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java index 41d859924f073..290466ffeb6b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICrossCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; @@ -192,6 +193,11 @@ public boolean doCompaction() { performer.setTargetFiles(targetTsfileResourceList); performer.setSummary(summary); performer.perform(); + if (performer instanceof ReadPointCompactionPerformer) { + for (TsFileResource resource : getAllSourceTsFiles()) { + CompactionUtils.removeDeletedObjectFiles(resource); + } + } CompactionUtils.updateProgressIndexAndMark( targetTsfileResourceList, selectedSequenceFiles, selectedUnsequenceFiles); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index 3cd5ff532af23..9c7fcc5424fdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; @@ -389,6 +390,11 @@ protected void compact(SimpleCompactionLogger compactionLogger) throws Exception performer.setTargetFiles(filesView.targetFilesInPerformer); performer.setSummary(summary); performer.perform(); + if (performer instanceof ReadPointCompactionPerformer) { + for (TsFileResource resource : filesView.sourceFilesInCompactionPerformer) { + CompactionUtils.removeDeletedObjectFiles(resource); + } + } prepareTargetFiles(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java index e00703e5e7521..79d6a41025bcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java @@ -18,10 +18,12 @@ */ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.SimpleCompactionLogger; @@ -204,7 +206,7 @@ protected boolean doCompaction() { return isSuccess; } - public boolean settleWithFullyDirtyFiles() { + public boolean settleWithFullyDirtyFiles() throws IllegalPathException, IOException { if (fullyDirtyFiles.isEmpty()) { return true; } @@ -213,6 +215,8 @@ public boolean settleWithFullyDirtyFiles() { if (recoverMemoryStatus) { tsFileManager.remove(resource, resource.isSeq()); } + CompactionUtils.removeDeletedObjectFiles(resource); + boolean res = deleteTsFileOnDisk(resource); if (res) { fullyDeletedSuccessNum++; @@ -288,7 +292,7 @@ public void recover() { } } - public void recoverFullyDirtyFiles() { + public void recoverFullyDirtyFiles() throws IllegalPathException, IOException { if (!settleWithFullyDirtyFiles()) { throw new CompactionRecoverException("Failed to delete fully_dirty source file."); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 44905afc461d3..584ff0fe44ae7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -29,6 +29,8 @@ import org.apache.iotdb.db.service.metrics.CompactionMetrics; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionAlignedChunkReader; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; @@ -39,21 +41,36 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; +import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.SystemMetric; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -448,4 +465,158 @@ public static ModEntry convertTtlToDeletion(IDeviceID deviceID, long timeLowerBo new TimeRange(Long.MIN_VALUE, timeLowerBound)); } } + + public static void removeDeletedObjectFiles(TsFileResource resource) + throws IOException, IllegalPathException { + try (MultiTsFileDeviceIterator deviceIterator = + new MultiTsFileDeviceIterator(Collections.singletonList(resource))) { + while (deviceIterator.hasNextDevice()) { + deviceIterator.nextDevice(); + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + } + } + } + + public static void removeDeletedObjectFiles( + TsFileSequenceReader reader, + List alignedChunkMetadataList, + List timeMods, + List> valueMods) + throws IOException { + if (alignedChunkMetadataList.isEmpty()) { + return; + } + List objectColumnIndexList = new ArrayList<>(); + List> objectDeletionIntervalList = new ArrayList<>(); + boolean objectColumnHasDeletion = false; + + TSDataType[] dataTypes = new TSDataType[valueMods.size()]; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + boolean hasNull = false; + for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { + if (dataTypes[i] != null) { + continue; + } + IChunkMetadata chunkMetadata = alignedChunkMetadata.getValueChunkMetadataList().get(i); + if (chunkMetadata == null) { + hasNull = true; + continue; + } + dataTypes[i] = chunkMetadata.getDataType(); + if (dataTypes[i] == TSDataType.OBJECT) { + objectColumnIndexList.add(i); + List deletionInterval = ModificationUtils.sortAndMerge(valueMods.get(i)); + objectColumnHasDeletion |= (!deletionInterval.isEmpty() || !timeMods.isEmpty()); + objectDeletionIntervalList.add(deletionInterval); + } + } + if (!hasNull) { + break; + } + } + if (!objectColumnHasDeletion) { + return; + } + int[] deletionCursors = new int[objectColumnIndexList.size() + 1]; + List timeDeletionIntervalList = ModificationUtils.sortAndMerge(timeMods); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + CompactionUtils.removeDeletedObjectFiles( + reader, + alignedChunkMetadata, + objectColumnIndexList, + timeDeletionIntervalList, + objectDeletionIntervalList, + deletionCursors); + } + } + + private static void removeDeletedObjectFiles( + TsFileSequenceReader reader, + AbstractAlignedChunkMetadata alignedChunkMetadata, + List objectColumnIndexList, + List timeDeletions, + List> objectDeletions, + int[] deletionCursors) + throws IOException { + Chunk timeChunk = + reader.readMemChunk((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); + CompactionChunkReader compactionChunkReader = new CompactionChunkReader(timeChunk); + List> timePages = + compactionChunkReader.readPageDataWithoutUncompressing(); + + List valueChunks = new ArrayList<>(); + List>> valuePages = new ArrayList<>(); + + for (int i = 0; i < objectColumnIndexList.size(); i++) { + int idxInAlignedChunkMetadata = objectColumnIndexList.get(i); + if (timeDeletions.isEmpty() && objectDeletions.get(i).isEmpty()) { + continue; + } + ChunkMetadata valueChunkMetadata = + (ChunkMetadata) + alignedChunkMetadata.getValueChunkMetadataList().get(idxInAlignedChunkMetadata); + if (valueChunkMetadata == null) { + continue; + } + Chunk chunk = reader.readMemChunk(valueChunkMetadata); + valueChunks.add(chunk); + valuePages.add( + chunk == null + ? null + : new CompactionChunkReader(chunk).readPageDataWithoutUncompressing()); + } + + CompactionAlignedChunkReader alignedChunkReader = + new CompactionAlignedChunkReader(timeChunk, valueChunks, true); + for (int i = 0; i < timePages.size(); i++) { + Pair timePage = timePages.get(i); + List valuePageHeaders = new ArrayList<>(valuePages.size()); + List compressedValuePages = new ArrayList<>(valuePages.size()); + for (int j = 0; j < valuePages.size(); j++) { + Pair valuePage = valuePages.get(j).get(i); + valuePageHeaders.add(valuePage.getLeft()); + compressedValuePages.add(valuePage.getRight()); + } + IPointReader pagePointReader = + alignedChunkReader.getPagePointReader( + timePage.getLeft(), valuePageHeaders, timePage.getRight(), compressedValuePages); + + while (pagePointReader.hasNextTimeValuePair()) { + TimeValuePair timeValuePair = pagePointReader.nextTimeValuePair(); + removeDeletedObjectFiles(timeValuePair, deletionCursors, timeDeletions, objectDeletions); + } + } + } + + private static void removeDeletedObjectFiles( + TimeValuePair timeValuePair, + int[] cursors, + List timeDeletions, + List> objectDeletions) { + long timestamp = timeValuePair.getTimestamp(); + boolean timeDeleted = isDeleted(timestamp, timeDeletions, cursors, 0); + for (int i = 0; i < timeValuePair.getValues().length; i++) { + Binary value = (Binary) timeValuePair.getValues()[i]; + if (value == null) { + continue; + } + if (timeDeleted || isDeleted(timestamp, objectDeletions.get(i), cursors, i + 1)) { + ObjectTypeUtils.deleteObjectPathFromBinary(value); + } + } + } + + private static boolean isDeleted( + long timestamp, List deleteIntervalList, int[] deleteCursors, int idx) { + while (deleteIntervalList != null && deleteCursors[idx] < deleteIntervalList.size()) { + if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().contains(timestamp)) { + return true; + } else if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().getMax() < timestamp) { + deleteCursors[idx]++; + } else { + return false; + } + } + return false; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index ea886aafd786d..0ce8649d5740e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -424,7 +424,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx */ private void applyModificationForAlignedChunkMetadataList( TsFileResource tsFileResource, List alignedChunkMetadataList) - throws IllegalPathException { + throws IllegalPathException, IOException { if (alignedChunkMetadataList.isEmpty()) { // all the value chunks is empty chunk return; @@ -476,6 +476,15 @@ private void applyModificationForAlignedChunkMetadataList( modificationList.isEmpty() ? Collections.emptyList() : modificationList); } + if (ttlDeletion != null) { + List emptyList = Collections.emptyList(); + CompactionUtils.removeDeletedObjectFiles( + readerMap.get(tsFileResource), + alignedChunkMetadataList, + Collections.singletonList(ttlDeletion), + modificationForValueColumns.stream().map(v -> emptyList).collect(Collectors.toList())); + } + ModificationUtils.modifyAlignedChunkMetaData( alignedChunkMetadataList, modificationForTimeColumn, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 7ad78814c5fbb..4e28306ebec20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -22,8 +22,11 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.exception.WriteProcessException; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.AlignedSeriesBatchCompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.AlignedPageElement; @@ -58,9 +61,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecutor { @@ -171,7 +176,8 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) new ChunkMetadataElement( alignedChunkMetadataList.get(i), i == alignedChunkMetadataList.size() - 1, - fileElement)); + fileElement, + measurementSchemas)); } } } @@ -260,6 +266,23 @@ protected List getAlignedChunkMetadataList(TsFileR } }); + long ttlForTable = + deviceId.isTableModel() + ? DataNodeTTLCache.getInstance() + .getTTLForTable(resource.getDatabaseName(), deviceId.getTableName()) + : Long.MAX_VALUE; + if (ttlForTable != Long.MAX_VALUE) { + ModEntry ttlDeletion = + CompactionUtils.convertTtlToDeletion( + deviceId, CommonDateTimeUtils.currentTime() - ttlForTable); + List emptyList = Collections.emptyList(); + CompactionUtils.removeDeletedObjectFiles( + readerCacheMap.get(resource), + alignedChunkMetadataList, + Collections.singletonList(ttlDeletion), + valueModifications.stream().map(v -> emptyList).collect(Collectors.toList())); + } + // modify aligned chunk metadatas ModificationUtils.modifyAlignedChunkMetaData( alignedChunkMetadataList, timeModifications, valueModifications, ignoreAllNullRows); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index e2be4be0cc204..402f06994b9ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -51,6 +51,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -159,7 +160,10 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) // add into queue chunkMetadataQueue.add( new ChunkMetadataElement( - chunkMetadata, i == iChunkMetadataList.size() - 1, fileElement)); + chunkMetadata, + i == iChunkMetadataList.size() - 1, + fileElement, + Collections.emptyList())); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java index 6828841f1bda1..2d29e3944d858 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java @@ -23,6 +23,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.util.List; @@ -42,12 +43,19 @@ public class ChunkMetadataElement { public boolean needForceDecodingPage; + // for aligned series + public List measurementSchemasOfValueChunks; + public ChunkMetadataElement( - IChunkMetadata chunkMetadata, boolean isLastChunk, FileElement fileElement) { + IChunkMetadata chunkMetadata, + boolean isLastChunk, + FileElement fileElement, + List measurementSchemasOfValueChunks) { this.chunkMetadata = chunkMetadata; this.startTime = chunkMetadata.getStartTime(); this.isLastChunk = isLastChunk; this.fileElement = fileElement; + this.measurementSchemasOfValueChunks = measurementSchemasOfValueChunks; } public void clearChunks() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index e6891199bb4db..4bca1c83ab049 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -563,6 +563,7 @@ private long estimateMemorySizeAsPageWriter(PageLoader pageLoader) { case TEXT: case STRING: case BLOB: + case OBJECT: size = pageLoader.getHeader().getUncompressedSize(); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index 69ce89d5af37b..e7c27be8e0a4d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -317,6 +317,7 @@ private void writeTimeAndValueToChunkWriter(TimeValuePair timeValuePair) { case TEXT: case BLOB: case STRING: + case OBJECT: chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary()); break; case FLOAT: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 364e21e22737d..849c3a9031c81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -135,6 +135,7 @@ protected void writeDataPoint(long timestamp, TsPrimitiveType value, IChunkWrite case TEXT: case STRING: case BLOB: + case OBJECT: chunkWriterImpl.write(timestamp, value.getBinary()); break; case DOUBLE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 5ad706df1217d..a225b1489e4f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -233,6 +233,7 @@ public void initChunkMetaFromTvLists() { case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < tsBlock.getPositionCount(); i++) { if (tsBlock.getColumn(column).isNull(i)) { continue; @@ -365,6 +366,7 @@ private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOExcept case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(values[columnIndex].getBinary()); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index d41ef855d368a..84f2a7e5d0b9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -592,6 +592,7 @@ private void handleEncoding( case TEXT: case STRING: case BLOB: + case OBJECT: alignedChunkWriter.writeByColumn( time, isNull ? null : list.getBinaryByValueIndex(originRowIndex, columnIndex), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 1f4078dfc7d96..450dd01d85d71 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -150,6 +150,7 @@ public void writeNonAlignedTablet( case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) valueList; putBinaries(times, binaryValues, bitMap, start, end); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 3586aface40ba..10e9fb3539f2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -290,6 +290,7 @@ private void updatePageStatisticsFromTsBlock( break; case TEXT: case BLOB: + case OBJECT: case STRING: for (int i = 0; i < tsBlock.getPositionCount(); i++) { valueStatistics[column].update( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java index 90231d1bedfb5..687a3d7a5299e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java @@ -105,6 +105,7 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { case TEXT: case STRING: case BLOB: + case OBJECT: batchData.putBinary( tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getBinary(i)); break; @@ -266,6 +267,7 @@ private void updatePageStatisticsFromTsBlock(Statistics statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < tsBlock.getPositionCount(); i++) { statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getBinary(i)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java index ca0088479e98b..0ecf9c34198c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.slf4j.Logger; @@ -167,6 +168,20 @@ public static TsFileName getTsFileName(String fileName) throws IOException { } } + public static String generateObjectFilePath(int regionId, long time, IDeviceID iDeviceID) { + String objectFileName = time + ".bin"; + Object[] segments = iDeviceID.getSegments(); + StringBuilder relativePathString = + new StringBuilder(String.valueOf(regionId)).append(File.separator); + for (Object segment : segments) { + relativePathString + .append(segment == null ? "null" : segment.toString().toLowerCase()) + .append(File.separator); + } + relativePathString.append(objectFileName); + return relativePathString.toString(); + } + @TestOnly public static TsFileResource increaseCrossCompactionCnt(TsFileResource tsFileResource) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 16be3f8ad089e..951df0437500c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; @@ -332,6 +333,8 @@ private void handleInfoEntry(WALEntry walEntry) { searchIndex = ((DeleteDataNode) walEntry.getValue()).getSearchIndex(); } else if (walEntry.getType() == WALEntryType.RELATIONAL_DELETE_DATA_NODE) { searchIndex = ((RelationalDeleteDataNode) walEntry.getValue()).getSearchIndex(); + } else if (walEntry.getType() == WALEntryType.OBJECT_FILE_NODE) { + searchIndex = ((ObjectNode) walEntry.getValue()).getSearchIndex(); } else { searchIndex = ((InsertNode) walEntry.getValue()).getSearchIndex(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java index 88cfb8b1564eb..059d363f53997 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; @@ -77,17 +78,27 @@ protected WALEntry(long memTableId, WALEntryValue value, boolean wait) { this.type = WALEntryType.CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE; } else if (value instanceof RelationalDeleteDataNode) { this.type = WALEntryType.RELATIONAL_DELETE_DATA_NODE; + } else if (value instanceof ObjectNode) { + this.type = WALEntryType.OBJECT_FILE_NODE; } else { throw new RuntimeException("Unknown WALEntry type"); } - walFlushListener = new WALFlushListener(wait, value); + if (type == WALEntryType.OBJECT_FILE_NODE) { + this.walFlushListener = new WALFlushListener(wait, null); + } else { + this.walFlushListener = new WALFlushListener(wait, value); + } } protected WALEntry(WALEntryType type, long memTableId, WALEntryValue value, boolean wait) { this.type = type; this.memTableId = memTableId; this.value = value; - this.walFlushListener = new WALFlushListener(wait, value); + if (type == WALEntryType.OBJECT_FILE_NODE) { + this.walFlushListener = new WALFlushListener(wait, null); + } else { + this.walFlushListener = new WALFlushListener(wait, value); + } } public abstract void serialize(IWALByteBufferView buffer); @@ -134,6 +145,9 @@ public static WALEntry deserialize(DataInputStream stream) throws IOException { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: value = (ContinuousSameSearchIndexSeparatorNode) PlanNodeType.deserializeFromWAL(stream); break; + case OBJECT_FILE_NODE: + value = (ObjectNode) PlanNodeType.deserializeFromWAL(stream); + break; default: throw new RuntimeException("Unknown WALEntry type " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java index 1c3ddcbf702ad..829affd1b4e18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java @@ -45,6 +45,7 @@ public enum WALEntryType { /** {@link org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable} */ MEMORY_TABLE_SNAPSHOT((byte) 10), RELATIONAL_DELETE_DATA_NODE((byte) 11), + OBJECT_FILE_NODE((byte) 12), // endregion // region signal entry type // signal wal buffer has been closed @@ -71,7 +72,8 @@ public boolean needSearch() { || this == INSERT_ROW_NODE || this == INSERT_ROWS_NODE || this == DELETE_DATA_NODE - || this == RELATIONAL_DELETE_DATA_NODE; + || this == RELATIONAL_DELETE_DATA_NODE + || this == OBJECT_FILE_NODE; } public static WALEntryType valueOf(byte code) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java index 6da50edba4f8f..791ef63a7525d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; @@ -89,6 +90,7 @@ public void serialize(IWALByteBufferView buffer) { case RELATIONAL_DELETE_DATA_NODE: case MEMORY_TABLE_SNAPSHOT: case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: + case OBJECT_FILE_NODE: value.serializeToWAL(buffer); break; case MEMORY_TABLE_CHECKPOINT: @@ -166,6 +168,8 @@ public long getMemorySize() { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: case MEMORY_TABLE_CHECKPOINT: return RamUsageEstimator.sizeOfObject(value); + case OBJECT_FILE_NODE: + return ((ObjectNode) value).serializedSize(); default: throw new RuntimeException("Unsupported wal entry type " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java index 0d0e0a527f0f7..a8fbbee0dc4f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushListener; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; @@ -54,6 +55,8 @@ public interface IWALNode extends FlushListener, AutoCloseable, ConsensusReqRead /** Log BatchDoneNode */ WALFlushListener log(long memTableId, ContinuousSameSearchIndexSeparatorNode separatorNode); + WALFlushListener log(long memTableId, ObjectNode objectNode); + /** Callback when memTable created. */ void onMemTableCreated(IMemTable memTable, String targetTsFile); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java index c13e066415bf7..7cc9a60b4d50f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALException; @@ -82,6 +83,11 @@ public WALFlushListener log( return getResult(); } + @Override + public WALFlushListener log(long memTableId, ObjectNode objectNode) { + return getResult(); + } + private WALFlushListener getResult() { switch (status) { case SUCCESS: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 69af5d621626c..ce00a0cdd5bba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; @@ -64,6 +65,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -192,6 +195,12 @@ public WALFlushListener log( return log(walEntry); } + @Override + public WALFlushListener log(long memTableId, ObjectNode objectNode) { + WALEntry walEntry = new WALInfoEntry(memTableId, objectNode); + return log(walEntry); + } + private WALFlushListener log(WALEntry walEntry) { buffer.write(walEntry); @@ -730,6 +739,8 @@ public boolean hasNext() { AtomicBoolean notFirstFile = new AtomicBoolean(false); AtomicBoolean hasCollectedSufficientData = new AtomicBoolean(false); + long memorySize = 0; + // try to collect current tmpNodes to insertNodes, return true if successfully collect an // insert node Runnable tryToCollectInsertNodeAndBumpIndex = @@ -768,7 +779,23 @@ public boolean hasNext() { } else if (currentWalEntryIndex < nextSearchIndex) { // WAL entry is outdated, do nothing, continue to see next WAL entry } else if (currentWalEntryIndex == nextSearchIndex) { - tmpNodes.get().add(new IoTConsensusRequest(buffer)); + if (type == WALEntryType.OBJECT_FILE_NODE) { + WALEntry walEntry = + WALEntry.deserialize( + new DataInputStream(new ByteArrayInputStream(buffer.array()))); + // only be called by leader read from wal + // wal only has relativePath, offset, eof, length + // need to add WALEntryType + memtableId + relativePath, offset, eof, length + + // content + // need to add IoTConsensusRequest instead of ObjectNode + tmpNodes + .get() + .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize())); + memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize(); + } else { + tmpNodes.get().add(new IoTConsensusRequest(buffer)); + memorySize += buffer.remaining(); + } } else { // currentWalEntryIndex > targetIndex // WAL entry of targetIndex has been fully collected, put them into insertNodes @@ -780,12 +807,31 @@ public boolean hasNext() { currentWalEntryIndex); nextSearchIndex = currentWalEntryIndex; } - tmpNodes.get().add(new IoTConsensusRequest(buffer)); + if (type == WALEntryType.OBJECT_FILE_NODE) { + WALEntry walEntry = + WALEntry.deserialize( + new DataInputStream(new ByteArrayInputStream(buffer.array()))); + // only be called by leader read from wal + // wal only has relativePath, offset, eof, length + // need to add WALEntryType + memtableId + relativePath, offset, eof, length + + // content + // need to add IoTConsensusRequest instead of ObjectNode + tmpNodes + .get() + .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize())); + memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize(); + } else { + tmpNodes.get().add(new IoTConsensusRequest(buffer)); + memorySize += buffer.remaining(); + } } } else { tryToCollectInsertNodeAndBumpIndex.run(); } - if (hasCollectedSufficientData.get()) { + if (memorySize > config.getWalBufferSize()) { + tryToCollectInsertNodeAndBumpIndex.run(); + } + if (memorySize > config.getWalBufferSize() || hasCollectedSufficientData.get()) { break COLLECT_FILE_LOOP; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index c955f67c79c03..38935f9fd62f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -223,6 +223,8 @@ public void redoLog(WALEntry walEntry) { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: // The CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE doesn't need redo break; + case OBJECT_FILE_NODE: + break; default: throw new RuntimeException("Unsupported type " + walEntry.getType()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index 5a8e22aa8e2a6..8cfa6ac5e3f1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -115,6 +115,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream); break; @@ -204,6 +205,7 @@ private void buildValueChunkWriter( break; case TEXT: case BLOB: + case OBJECT: case STRING: final Binary binaryValue = isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index d929f7656077c..61bb6ffafba58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.rescon.disk; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.exception.ObjectFileNotExist; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -65,12 +67,16 @@ public class TierManager { */ private final List unSeqTiers = new ArrayList<>(); + private final List objectTiers = new ArrayList<>(); + /** seq file folder's rawFsPath path -> tier level */ private final Map seqDir2TierLevel = new HashMap<>(); /** unSeq file folder's rawFsPath path -> tier level */ private final Map unSeqDir2TierLevel = new HashMap<>(); + private List objectDirs; + /** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */ private long[] tierDiskTotalSpace; @@ -151,6 +157,22 @@ public synchronized void initFolders() { for (String dir : unSeqDirs) { unSeqDir2TierLevel.put(dir, tierLevel); } + + objectDirs = + Arrays.stream(tierDirs[tierLevel]) + .filter(Objects::nonNull) + .map( + v -> + FSFactoryProducer.getFSFactory() + .getFile(v, IoTDBConstant.OBJECT_FOLDER_NAME) + .getPath()) + .collect(Collectors.toList()); + + try { + objectTiers.add(new FolderManager(objectDirs, directoryStrategyType)); + } catch (DiskSpaceInsufficientException e) { + logger.error("All disks of tier {} are full.", tierLevel, e); + } } tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL); @@ -160,6 +182,7 @@ public synchronized void resetFolders() { long startTime = System.currentTimeMillis(); seqTiers.clear(); unSeqTiers.clear(); + objectTiers.clear(); seqDir2TierLevel.clear(); unSeqDir2TierLevel.clear(); @@ -190,6 +213,10 @@ public String getNextFolderForTsFile(int tierLevel, boolean sequence) : unSeqTiers.get(tierLevel).getNextFolder(); } + public String getNextFolderForObjectFile() throws DiskSpaceInsufficientException { + return objectTiers.get(0).getNextFolder(); + } + public FolderManager getFolderManager(int tierLevel, boolean sequence) { return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel); } @@ -222,6 +249,20 @@ public List getAllLocalUnSequenceFileFolders() { .collect(Collectors.toList()); } + public List getAllObjectFileFolders() { + return objectDirs; + } + + public Optional getAbsoluteObjectFilePath(String filePath) { + for (String objectDir : objectDirs) { + File objectFile = FSFactoryProducer.getFSFactory().getFile(objectDir, filePath); + if (objectFile.exists()) { + return Optional.of(objectFile); + } + } + return Optional.empty(); + } + public int getTiersNum() { return seqTiers.size(); } @@ -294,6 +335,16 @@ private long[] getTierDiskSpace(DiskSpaceType type) { return tierDiskSpace; } + public File getObjectFile(String relativePath) { + for (String folder : objectDirs) { + File file = new File(folder, relativePath); + if (file.exists()) { + return file; + } + } + throw new ObjectFileNotExist(relativePath); + } + private enum DiskSpaceType { TOTAL, USABLE, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java index e8a2a9bb83741..dbf3b989800eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java @@ -249,6 +249,7 @@ private static Object createPrimitiveArray(TSDataType dataType) { case TEXT: case STRING: case BLOB: + case OBJECT: dataArray = new Binary[ARRAY_SIZE]; break; default: @@ -342,6 +343,7 @@ public static Object createDataListsByType(TSDataType dataType, int size) { case TEXT: case STRING: case BLOB: + case OBJECT: Binary[][] binaries = new Binary[arrayNumber][]; for (int i = 0; i < arrayNumber; i++) { binaries[i] = new Binary[ARRAY_SIZE]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index f50e472c414f3..38686bee04774 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -430,6 +430,7 @@ protected void rewritePageIntoFiles( case TEXT: case BLOB: case STRING: + case OBJECT: chunkWriter.write(time, (Binary) value); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index d83379bd7386d..0e58fbfa4d0a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -380,6 +380,7 @@ public static Object createValueColumnOfDataType( case TEXT: case STRING: case BLOB: + case OBJECT: valueColumn = new Binary[rowNum]; break; case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java index 402921338a494..d9d4cecc4e423 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java @@ -51,6 +51,7 @@ public static TSEncoding getDefaultEncoding(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return conf.getDefaultTextEncoding(); default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 843bcdb8937c4..1de1422b07bef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -108,7 +108,9 @@ public static long getBinaryColumnSize(Binary[] column, int start, int end, TSSt if (results == null || results[i] == null || results[i].code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - memSize += RamUsageEstimator.sizeOf(column[i].getValues()); + if (column[i] != null) { + memSize += RamUsageEstimator.sizeOf(column[i].getValues()); + } } } return memSize; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java new file mode 100644 index 0000000000000..fc1bb40b93fb7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.commons.exception.ObjectFileNotExist; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.utils.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Optional; + +public class ObjectTypeUtils { + + private static final Logger logger = LoggerFactory.getLogger(ObjectTypeUtils.class); + private static final TierManager TIER_MANAGER = TierManager.getInstance(); + + private ObjectTypeUtils() {} + + public static File getObjectPathFromBinary(Binary binary) { + byte[] bytes = binary.getValues(); + String relativeObjectFilePath = + new String(bytes, 8, bytes.length - 8, TSFileConfig.STRING_CHARSET); + Optional file = TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath); + if (!file.isPresent()) { + throw new ObjectFileNotExist(relativeObjectFilePath); + } + return file.get(); + } + + public static Optional getNullableObjectPathFromBinary(Binary binary) { + byte[] bytes = binary.getValues(); + String relativeObjectFilePath = + new String(bytes, 8, bytes.length - 8, TSFileConfig.STRING_CHARSET); + return TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath); + } + + public static void deleteObjectPathFromBinary(Binary binary) { + Optional file = ObjectTypeUtils.getNullableObjectPathFromBinary(binary); + if (!file.isPresent()) { + return; + } + File parentFile = file.get().getParentFile(); + File tmpFile = new File(parentFile, file.get().getName().replace(".bin", ".tmp")); + File bakFile = new File(parentFile, file.get().getName().replace(".bin", ".bak")); + logger.info("Remove object file {}", file.get().getAbsolutePath()); + for (int i = 0; i < 2; i++) { + try { + if (Files.deleteIfExists(file.get().toPath())) { + return; + } + if (Files.deleteIfExists(tmpFile.toPath())) { + return; + } + if (Files.deleteIfExists(bakFile.toPath())) { + return; + } + } catch (IOException e) { + logger.error("Failed to remove object file {}", file.get().getAbsolutePath(), e); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java new file mode 100644 index 0000000000000..9ba84df94ecb3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; + +public class ObjectWriter implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectWriter.class); + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final FileOutputStream fos; + + private final File file; + + public ObjectWriter(File filePath) throws FileNotFoundException { + try { + FileUtils.forceMkdir(filePath.getParentFile()); + } catch (final IOException e) { + throw new FileNotFoundException("Error occurred during creating directory " + filePath); + } + if (!Files.exists(filePath.toPath())) { + try { + Files.createFile(filePath.toPath()); + } catch (IOException e) { + throw new FileNotFoundException(e.getMessage()); + } + } + file = filePath; + fos = new FileOutputStream(filePath, true); + } + + public void write(boolean isGeneratedByConsensus, long offset, byte[] content) + throws IOException { + if (file.length() != offset) { + if (isGeneratedByConsensus || offset == 0) { + fos.getChannel().truncate(offset); + } else { + throw new IOException( + "The file length " + file.length() + " is not equal to the offset " + offset); + } + } + if (file.length() + content.length > config.getMaxObjectSizeInByte()) { + throw new IOException("The file length is larger than max_object_file_size_in_bytes"); + } + fos.write(content); + } + + @Override + public void close() throws Exception { + fos.close(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index cd4ca79ac4387..fed80815f692a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -236,6 +236,7 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < currentCount; i++) { rowCount++; if (column.isNull(i)) { @@ -379,6 +380,7 @@ private static void serializeTsBlock( case TEXT: case BLOB: case STRING: + case OBJECT: doWithTextColumn( rowCount, column, @@ -753,6 +755,7 @@ public static Object[] readTabletValuesFromBuffer( case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = new Binary[size]; for (int index = 0; index < size; index++) { int binarySize = buffer.getInt(); @@ -795,6 +798,7 @@ public static Object[] readTabletValuesFromStream( case TEXT: case BLOB: case STRING: + case OBJECT: parseTextColumn(size, stream, values, i); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java index e11b586e7e427..a66e7d7b856bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java @@ -62,6 +62,7 @@ public static void setTimeValuePair(TimeValuePair from, TimeValuePair to) { case TEXT: case BLOB: case STRING: + case OBJECT: to.getValue().setBinary(from.getValue().getBinary()); break; case BOOLEAN: @@ -89,6 +90,7 @@ public static TimeValuePair getEmptyTimeValuePair(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return new TimeValuePair(0, new TsBinary(new Binary("", TSFileConfig.STRING_CHARSET))); default: throw new UnsupportedOperationException("Unrecognized datatype: " + dataType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java index 8fc1d647dc36e..41cf1e12ddba4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java @@ -325,6 +325,7 @@ public static boolean canAutoCast(TSDataType fromType, TSDataType toType) { case DATE: case TIMESTAMP: case BLOB: + case OBJECT: case STRING: return false; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 80649eb182d15..7c0acb166a3f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -201,6 +201,7 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { case TEXT: case BLOB: case STRING: + case OBJECT: ((Binary[]) columnValues.get(arrayIndex))[elementIndex] = columnValue != null ? (Binary) columnValue : Binary.EMPTY_VALUE; memoryBinaryChunkSize[i] += @@ -300,6 +301,7 @@ private TsPrimitiveType getAlignedValueByValueIndex( case TEXT: case BLOB: case STRING: + case OBJECT: Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.TEXT, valueT); break; @@ -361,6 +363,7 @@ public void extendColumn(TSDataType dataType) { case TEXT: case STRING: case BLOB: + case OBJECT: columnValue.add(getPrimitiveArraysByType(TSDataType.TEXT)); break; case FLOAT: @@ -640,6 +643,7 @@ protected Object cloneValue(TSDataType type, Object value) { case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] valueT = (Binary[]) value; Binary[] cloneT = new Binary[valueT.length]; System.arraycopy(valueT, 0, cloneT, 0, valueT.length); @@ -834,6 +838,7 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] arrayT = ((Binary[]) columnValues.get(arrayIndex)); System.arraycopy(value[i], idx, arrayT, elementIndex, remaining); @@ -1122,6 +1127,7 @@ public TsBlock buildTsBlock( case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, columnIndex)); break; default: @@ -1199,6 +1205,7 @@ public int serializedSize() { case TEXT: case BLOB: case STRING: + case OBJECT: for (int rowIdx = 0; rowIdx < rowCount; ++rowIdx) { size += ReadWriteIOUtils.sizeToWrite(getBinaryByValueIndex(rowIdx, columnIndex)); } @@ -1259,6 +1266,7 @@ public void serializeToWAL(IWALByteBufferView buffer) { case TEXT: case BLOB: case STRING: + case OBJECT: Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; // In some scenario, the Binary in AlignedTVList will be null if this field is empty in // current row. We need to handle this scenario to get rid of NPE. See the similar issue @@ -1338,6 +1346,7 @@ public static AlignedTVList deserialize(DataInputStream stream) throws IOExcepti case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = new Binary[rowCount]; for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { binaryValues[rowIndex] = ReadWriteIOUtils.readBinary(stream); @@ -1725,6 +1734,7 @@ public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) { case TEXT: case BLOB: case STRING: + case OBJECT: return TsPrimitiveType.getByType( TSDataType.TEXT, getBinaryByValueIndex(valueIndex, validColumnIndex)); default: @@ -1889,6 +1899,7 @@ public TsBlock nextBatch() { case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, validColumnIndex)); break; default: @@ -2060,6 +2071,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo case TEXT: case BLOB: case STRING: + case OBJECT: valueChunkWriter.write( time, isNull ? null : getBinaryByValueIndex(originRowIndex, validColumnIndex), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java index 24654e0501264..0ec099f522d61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java @@ -229,6 +229,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo break; case TEXT: case BLOB: + case OBJECT: case STRING: valueChunkWriter.write( currentTime, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java index 616835082bceb..cca4db65cd525 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java @@ -131,6 +131,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo break; case TEXT: case BLOB: + case OBJECT: case STRING: Binary value = currIterator.getTVList().getBinary(row); chunkWriterImpl.write(time, value); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java index 5c4b730af80b7..1cdca27e220ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java @@ -193,6 +193,7 @@ public TsBlock nextBatch() { break; case TEXT: case BLOB: + case OBJECT: case STRING: valueBuilder.writeBinary( alignedTVList.getBinaryByValueIndex(valueIndex, validColumnIndex)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java index aa9ec257311af..c04bcc8341afc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java @@ -137,6 +137,7 @@ public TsBlock nextBatch() { case TEXT: case BLOB: case STRING: + case OBJECT: builder.getColumnBuilder(0).writeBinary(iterator.getTVList().getBinary(rowIndex)); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 5e50efd25c06d..16da916f40a1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -112,6 +112,7 @@ public static TVList newList(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return BinaryTVList.newList(); case FLOAT: return FloatTVList.newList(); @@ -575,6 +576,7 @@ public static TVList deserialize(DataInputStream stream) throws IOException { switch (dataType) { case TEXT: case BLOB: + case OBJECT: case STRING: return BinaryTVList.deserialize(stream); case FLOAT: @@ -601,6 +603,7 @@ public static TVList deserializeWithoutBitMap(DataInputStream stream) throws IOE case TEXT: case BLOB: case STRING: + case OBJECT: return BinaryTVList.deserializeWithoutBitMap(stream); case FLOAT: return FloatTVList.deserializeWithoutBitMap(stream); @@ -821,6 +824,7 @@ public TsBlock nextBatch() { case TEXT: case BLOB: case STRING: + case OBJECT: while (index < rows && builder.getPositionCount() < maxNumberOfPointsInPage) { long time = getTime(index); if (!isNullValue(getValueIndex(index)) @@ -891,6 +895,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo case TEXT: case BLOB: case STRING: + case OBJECT: Binary value = getBinary(index); chunkWriterImpl.write(time, value); encodeInfo.dataSizeInChunk += 8L + getBinarySize(value); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java new file mode 100644 index 0000000000000..d78caa4e467e3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.model; + +import org.gdal.gdal.Band; +import org.gdal.gdal.Dataset; +import org.gdal.gdal.gdal; +import org.gdal.gdalconst.gdalconstConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class CompressedTiffModelReader extends ModelReader { + private static final Logger LOGGER = LoggerFactory.getLogger(CompressedTiffModelReader.class); + + static { + gdal.AllRegister(); + } + + @Override + public List penetrate(String filePath, List> startAndEndTimeArray) { + Dataset dataset = gdal.Open(filePath, gdalconstConstants.GA_ReadOnly); + if (dataset == null) { + LOGGER.error("Failed to open tiff file: {}", gdal.GetLastErrorMsg()); + throw new RuntimeException("Failed to open tiff file: " + gdal.GetLastErrorMsg()); + } + try { + Band band = dataset.GetRasterBand(1); + if (band == null) { + LOGGER.error("Failed to get raster band from dataset: {}", gdal.GetLastErrorMsg()); + throw new RuntimeException( + "Failed to get raster band from dataset" + gdal.GetLastErrorMsg()); + } + int width = band.getXSize(); + List result = new ArrayList<>(); + for (List startAndEndTime : startAndEndTimeArray) { + int xIndex = startAndEndTime.get(0); + int yIndex = startAndEndTime.get(1); + float[] tmp = new float[yIndex - xIndex + 1]; + int xOff = xIndex % width; + int yOff = xIndex / width; + int xSize = yIndex - xIndex + 1; + int ySize = 1; + band.ReadRaster(xOff, yOff, xSize, ySize, gdalconstConstants.GDT_Float32, tmp); + result.add(tmp); + } + return result; + } finally { + dataset.delete(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTsFileModelReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTsFileModelReader.java new file mode 100644 index 0000000000000..d30982219996c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTsFileModelReader.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.model; + +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static org.apache.tsfile.read.reader.chunk.ChunkReader.decryptAndUncompressPageData; + +public class CompressedTsFileModelReader extends ModelReader { + private static final Logger LOGGER = LoggerFactory.getLogger(CompressedTsFileModelReader.class); + + @Override + public List penetrate(String filePath, List> startAndEndTimeArray) { + int pairSize = startAndEndTimeArray.size(); + Integer[] indexes = new Integer[pairSize]; + for (int i = 0; i < pairSize; i++) { + indexes[i] = i; + } + Arrays.sort( + indexes, + Comparator.comparingInt((Integer i) -> startAndEndTimeArray.get(i).get(0)) + .thenComparingInt(i -> startAndEndTimeArray.get(i).get(1))); + List> sorted = new ArrayList<>(); + + for (int sortedIndex = 0; sortedIndex < pairSize; sortedIndex++) { + int originalIndex = indexes[sortedIndex]; + sorted.add(startAndEndTimeArray.get(originalIndex)); + } + + try { + List results = new ArrayList<>(pairSize); + int currentQueryIndex = 0; + int currentResultSetIndex = 0; + for (List ints : startAndEndTimeArray) { + results.add(new float[ints.get(1) - ints.get(0) + 1]); + } + IDecryptor decryptor = null; + IDeviceID deviceID = new StringArrayDeviceID("t"); + int index = 0; + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + TimeseriesMetadata timeseriesMetadata = reader.readTimeseriesMetadata(deviceID, "v", true); + for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { + TimeRange timeRange = + new TimeRange(chunkMetadata.getStartTime(), chunkMetadata.getEndTime()); + boolean overlap = false; + for (int i = currentQueryIndex; i < pairSize; i++) { + List startAndEnd = sorted.get(i); + if (timeRange.overlaps(new TimeRange(startAndEnd.get(0), startAndEnd.get(1)))) { + overlap = true; + break; + } + } + if (!overlap) { + index += chunkMetadata.getStatistics().getCount(); + continue; + } + + Chunk chunk = reader.readMemChunk((ChunkMetadata) chunkMetadata); + ChunkHeader chunkHeader = chunk.getHeader(); + ByteBuffer chunkDataBuffer = chunk.getData(); + while (chunkDataBuffer.hasRemaining()) { + PageHeader pageHeader = null; + if (((byte) (chunkHeader.getChunkType() & 0x3F)) + == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { + pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunk.getChunkStatistic()); + } else { + pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); + } + + ByteBuffer pageData = readCompressedPageData(pageHeader, chunkDataBuffer); + TimeRange pageTimeRange = + new TimeRange(pageHeader.getStartTime(), pageHeader.getEndTime()); + boolean pageOverlap = false; + for (int i = currentQueryIndex; i < pairSize; i++) { + List startAndEnd = sorted.get(i); + if (pageTimeRange.overlaps(new TimeRange(startAndEnd.get(0), startAndEnd.get(1)))) { + pageOverlap = true; + break; + } + } + if (!pageOverlap) { + index += pageHeader.getStatistics().getCount(); + continue; + } + decryptor = + decryptor == null ? IDecryptor.getDecryptor(chunk.getEncryptParam()) : decryptor; + ByteBuffer uncompressedPageData = + decryptAndUncompressPageData( + pageHeader, + IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()), + pageData, + decryptor); + Decoder decoder = + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); + + byte[] bitmap = null; + if (uncompressedPageData.hasRemaining()) { + int size = ReadWriteIOUtils.readInt(uncompressedPageData); + bitmap = new byte[(size + 7) / 8]; + uncompressedPageData.get(bitmap); + } + while (decoder.hasNext(uncompressedPageData)) { + float[] currentQueryResult = results.get(indexes[currentQueryIndex]); + if (currentResultSetIndex >= currentQueryResult.length) { + currentQueryIndex++; + currentResultSetIndex = 0; + } + if (currentQueryIndex == pairSize) { + return results; + } + currentQueryResult = results.get(indexes[currentQueryIndex]); + float v = decoder.readFloat(uncompressedPageData); + + List currentQueryStartAndEnd = sorted.get(currentQueryIndex); + if (index >= currentQueryStartAndEnd.get(0) + && index <= currentQueryStartAndEnd.get(1)) { + currentQueryResult[currentResultSetIndex++] = v; + } + index++; + } + } + } + } + return results; + } catch (Exception e) { + LOGGER.error("Penetrate TS file failed", e); + return new ArrayList<>(); + } + } + + public static ByteBuffer readCompressedPageData(PageHeader pageHeader, ByteBuffer chunkBuffer) + throws IOException { + int compressedPageBodyLength = pageHeader.getCompressedSize(); + if (compressedPageBodyLength > chunkBuffer.remaining()) { + throw new IOException( + "do not have a complete page body. Expected:" + + compressedPageBodyLength + + ". Actual:" + + chunkBuffer.remaining()); + } + ByteBuffer pageBodyBuffer = chunkBuffer.slice(); + pageBodyBuffer.limit(compressedPageBodyLength); + chunkBuffer.position(chunkBuffer.position() + compressedPageBodyLength); + return pageBodyBuffer; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java new file mode 100644 index 0000000000000..3d5c043a1257d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.model; + +import org.apache.iotdb.db.exception.sql.SemanticException; + +import java.util.List; + +public abstract class ModelReader { + /** + * Read part of float values from a file, eg. tsfile, tiff + * + * @param filePath the path of the file to read + * @param startAndEndTimeArray a list of start and end time pairs, each pair is a list of two + * integers + * @return a list of float arrays, each array corresponds to a start and end time pair + */ + public abstract List penetrate( + String filePath, List> startAndEndTimeArray); + + public static ModelReader getDefaultInstance() { + return new CompressedTsFileModelReader(); + } + + public static ModelReader getInstance(String modelFileType) { + if (modelFileType.equalsIgnoreCase("UNCOMPRESSED_TIFF")) { + return new UnCompressedTiffModelReader(); + } else if (modelFileType.equalsIgnoreCase("COMPRESSED_TSFILE")) { + return new CompressedTsFileModelReader(); + } else if (modelFileType.equalsIgnoreCase("COMPRESSED_TIFF")) { + return new CompressedTiffModelReader(); + } else { + throw new SemanticException("Unsupported model file type: " + modelFileType); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/UnCompressedTiffModelReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/UnCompressedTiffModelReader.java new file mode 100644 index 0000000000000..e763e72c0fdb5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/UnCompressedTiffModelReader.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils.model; + +import ij.ImagePlus; +import ij.io.Opener; +import ij.process.FloatProcessor; +import ij.process.ImageProcessor; + +import java.util.ArrayList; +import java.util.List; + +public class UnCompressedTiffModelReader extends ModelReader { + + @Override + public List penetrate(String filePath, List> startAndEndTimeArray) { + Opener opener = new Opener(); + ImagePlus imagePlus = opener.openImage(filePath); + ImageProcessor processor = imagePlus.getProcessor(); + List result = new ArrayList<>(); + for (List startAndEndTime : startAndEndTimeArray) { + int start = startAndEndTime.get(0); + int end = startAndEndTime.get(1); + int length = end - start + 1; + float[] pixels = new float[length]; + if (processor instanceof FloatProcessor) { + for (int i = start; i <= end; i++) { + pixels[i - start] = processor.getf(i); + } + } + result.add(pixels); + } + return result; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java index 0bc72b3205aa8..a87fb480d9d2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java @@ -87,6 +87,7 @@ private void init(EvictableBatchList list, int begin) { case TEXT: case BLOB: case STRING: + case OBJECT: binaryValues = new Binary[size]; for (int i = 0; i < size; ++i) { binaryValues[i] = list.getBinaryByIndex(begin + i); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java new file mode 100644 index 0000000000000..a4b35c01da6d8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.TimeRange; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class CompactionDeleteObjectFileTest extends AbstractCompactionTest { + @Before + public void setUp() + throws IOException, WriteProcessException, MetadataException, InterruptedException { + super.setUp(); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + } + + @Test + public void test1() throws IOException { + createTable("tsfile_table", 100); + File dir = new File("/Users/shuww/Downloads/0708/1_副本"); + List resources = new ArrayList<>(); + for (File file : dir.listFiles()) { + if (!file.getName().endsWith(".tsfile")) { + continue; + } + TsFileResource resource = new TsFileResource(file); + + try (ModificationFile modificationFile = resource.getExclusiveModFile()) { + modificationFile.write( + new TableDeletionEntry( + new DeletionPredicate( + "tsfile_table", + new IDPredicate.FullExactMatch( + new StringArrayDeviceID(new String[] {"tsfile_table", "1", "5", "3"})), + Arrays.asList("file")), + new TimeRange(-1, 0))); + modificationFile.write( + new TableDeletionEntry( + new DeletionPredicate( + "tsfile_table", + new IDPredicate.FullExactMatch( + new StringArrayDeviceID(new String[] {"tsfile_table", "1", "5", "3"})), + Arrays.asList("file")), + new TimeRange(2, 2))); + } + resource.deserialize(); + resources.add(resource); + } + + // InnerSpaceCompactionTask task = + // new InnerSpaceCompactionTask( + // 0, tsFileManager, resources, true, new ReadChunkCompactionPerformer(), 0); + SettleCompactionTask task = + new SettleCompactionTask( + 0, + tsFileManager, + resources, + Collections.emptyList(), + true, + new FastCompactionPerformer(false), + 0); + task.start(); + } + + public void createTable(String tableName, long ttl) { + TsTable tsTable = new TsTable(tableName); + tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); + tsTable.addColumnSchema( + new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); + tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); + DataNodeTableCache.getInstance().preUpdateTable("Downloads", tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable("Downloads", tableName, null); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java index 624ff6eb01243..a2877a8c3836e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java @@ -98,6 +98,7 @@ public void testUpdateLimits() { break; case TEXT: case BLOB: + case OBJECT: case STRING: Assert.assertTrue(o instanceof Binary[]); Assert.assertEquals(ARRAY_SIZE, ((Binary[]) o).length); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 28af70751ba0c..15530845cc9c3 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1246,6 +1246,11 @@ enable_tsfile_validation=false # Unit: ms tier_ttl_in_ms=-1 +# The maximum size limit for a single object file. +# effectiveMode: hot_reload +# Datatype: long +max_object_file_size_in_byte=4294967296 + #################### ### Compaction Configurations #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index ba568eae8966c..b76ab05f885b8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -248,6 +248,7 @@ private IoTDBConstant() {} public static final String DATA_FOLDER_NAME = "data"; public static final String SEQUENCE_FOLDER_NAME = "sequence"; public static final String UNSEQUENCE_FOLDER_NAME = "unsequence"; + public static final String OBJECT_FOLDER_NAME = "object"; public static final String FILE_NAME_SEPARATOR = "-"; public static final String CONSENSUS_FOLDER_NAME = "consensus"; public static final String DATA_REGION_FOLDER_NAME = "data_region"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java new file mode 100644 index 0000000000000..05add08b218f7 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.exception; + +import static org.apache.iotdb.rpc.TSStatusCode.OBJECT_NOT_EXISTS; + +public class ObjectFileNotExist extends IoTDBRuntimeException { + + private static final String ERROR_MSG = "Object file %s does not exist"; + + public ObjectFileNotExist(String relativeObjectPath) { + super(String.format(ERROR_MSG, relativeObjectPath), OBJECT_NOT_EXISTS.getStatusCode()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java index be0d16807e216..c3fe93420660c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java @@ -64,6 +64,7 @@ public void transform(Row row, PointCollector collector) collector.putDouble(time, Math.abs(row.getDouble(0))); break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case TEXT: @@ -103,6 +104,7 @@ public Object transform(Row row) throws IOException { case TIMESTAMP: case STRING: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -131,6 +133,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case STRING: case TEXT: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java index bb12f7ff4a58b..045e49b064b73 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java @@ -73,6 +73,7 @@ protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { }); break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java index 00b3fd4755016..cb5cbcc0486b8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java @@ -61,6 +61,7 @@ protected void doTransform(Row row, PointCollector collector) case TEXT: case STRING: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java index 75305f7316513..7f8d81f6b8299 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java @@ -56,6 +56,7 @@ protected void doTransform(Row row, PointCollector collector) break; case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case TEXT: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java index 209802c26cb90..f3436859f2a41 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java @@ -57,6 +57,7 @@ public class UDTFConst implements UDTF { VALID_TYPES.add(TSDataType.TEXT.name()); VALID_TYPES.add(TSDataType.STRING.name()); VALID_TYPES.add(TSDataType.BLOB.name()); + VALID_TYPES.add(TSDataType.OBJECT.name()); } private TSDataType dataType; @@ -107,6 +108,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati binaryValue = BytesUtils.valueOf(parameters.getString("value")); break; case BLOB: + case OBJECT: binaryValue = new Binary(BlobUtils.parseBlobString(parameters.getString("value"))); break; default: @@ -141,6 +143,7 @@ public void transform(Row row, PointCollector collector) throws Exception { case TEXT: case STRING: case BLOB: + case OBJECT: collector.putBinary(row.getTime(), UDFBinaryTransformer.transformToUDFBinary(binaryValue)); break; default: @@ -166,6 +169,7 @@ public Object transform(Row row) throws IOException { case TEXT: case STRING: case BLOB: + case OBJECT: return UDFBinaryTransformer.transformToUDFBinary(binaryValue); default: throw new UnsupportedOperationException(); @@ -257,6 +261,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception case TEXT: case STRING: case BLOB: + case OBJECT: for (int i = 0; i < count; i++) { boolean hasWritten = false; for (int j = 0; j < columns.length - 1; j++) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java index b76b0276e7cfd..de6232ac6db34 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java @@ -103,6 +103,7 @@ public void transform(Row row, PointCollector collector) case TEXT: case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case DATE: default: @@ -230,6 +231,7 @@ public void terminate(PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java index d016732761373..21506698605fe 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java @@ -482,6 +482,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) aggregator.aggregateDouble(rowWindow, collector); break; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java index cf80380eea2a2..fea5cb694fa73 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java @@ -64,6 +64,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) case STRING: case TEXT: case BLOB: + case OBJECT: default: // This will not happen throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java index f3b3aacba378c..85490ac534e47 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java @@ -658,6 +658,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) break; case TEXT: case BLOB: + case OBJECT: case DATE: case STRING: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java index b9c733a2df0d5..894366fdcbcfa 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java @@ -66,6 +66,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: // This will not happen diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java index f24b9d6ae7b8b..4c17a27069733 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java @@ -86,6 +86,7 @@ public void transform(Row row, PointCollector collector) collector.putBoolean(time, row.getDouble(0) >= lower && upper >= row.getDouble(0)); break; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: @@ -123,6 +124,7 @@ public Object transform(Row row) throws Exception { case STRING: case TEXT: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -151,6 +153,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java index d554cfe25720a..0c5ccd73b492e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java @@ -98,6 +98,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati case TIMESTAMP: case DATE: case BLOB: + case OBJECT: default: throw new UDFInputSeriesDataTypeNotValidException( 0, @@ -170,6 +171,7 @@ public void transform(Row row, PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case INT64: case INT32: case FLOAT: @@ -346,6 +348,7 @@ public void getValues(Row row) throws IOException, UDFInputSeriesDataTypeNotVali break; case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: default: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java index 824740b8c9b04..3de46df72a061 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java @@ -138,6 +138,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) transformDouble(rowWindow, collector); break; case BLOB: + case OBJECT: case DATE: case STRING: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java index de4f8edcec5d9..947cbdb2c0134 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java @@ -91,6 +91,7 @@ public void transform(Row row, PointCollector collector) case TIMESTAMP: case DATE: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -119,6 +120,7 @@ public Object transform(Row row) throws IOException { return transformer.transform(row.getDouble(0)); case DATE: case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case TEXT: @@ -155,6 +157,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception case STRING: case TIMESTAMP: case BLOB: + case OBJECT: case DATE: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java index 6812d2d2d8ee9..077db957f6e2e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java @@ -57,6 +57,7 @@ protected void doTransform(Row row, PointCollector collector) break; case DATE: case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java index 1183e5397fefc..fcb68817f153e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java @@ -59,6 +59,7 @@ protected void doTransform(Row row, PointCollector collector) case STRING: case TIMESTAMP: case BLOB: + case OBJECT: case DATE: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java index 66b5cd1add8e8..b0c95bd0d23b8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java @@ -145,6 +145,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case DATE: case STRING: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java index 49faa398f1a78..a888c998d8d88 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java @@ -107,6 +107,7 @@ public void transform(Row row, PointCollector collector) transformString(row.getTime(), row.getString(0)); break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. @@ -180,6 +181,7 @@ public void terminate(PointCollector collector) } break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java index 1838b85022b3a..b8d94308ed966 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java @@ -53,6 +53,7 @@ protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { break; case BOOLEAN: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java index c990021932948..d4487fdfd953a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java @@ -71,6 +71,7 @@ protected void updatePreviousValue(Row row) case STRING: case DATE: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java index 6d5b1ef4721ac..ced804e681c4c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java @@ -67,6 +67,7 @@ public enum TableBuiltinScalarFunction { FORMAT("format"), GREATEST("greatest"), LEAST("least"), + READ_OBJECT("read_object"), ; private final String functionName; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java index 83792ebf66aff..cf9ecf89f96c1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java @@ -30,6 +30,7 @@ import org.apache.tsfile.read.common.type.FloatType; import org.apache.tsfile.read.common.type.IntType; import org.apache.tsfile.read.common.type.LongType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; @@ -82,6 +83,8 @@ public static Type transformReadTypeToUDFDataType(org.apache.tsfile.read.common. return Type.BLOB; case STRING: return Type.STRING; + case OBJECT: + return Type.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } @@ -112,6 +115,8 @@ public static org.apache.tsfile.read.common.type.Type transformUDFDataTypeToRead return BlobType.BLOB; case STRING: return StringType.STRING; + case OBJECT: + return ObjectType.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } @@ -139,6 +144,8 @@ private static Type getUDFDataType(byte type) { return Type.BLOB; case 11: return Type.STRING; + case 12: + return Type.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java index 7b34111aa1fb8..a069c993abef2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java @@ -156,6 +156,7 @@ public static BatchData deserializeBatchData(ByteBuffer buffer) { } break; case BLOB: + case OBJECT: case STRING: case TEXT: for (int i = 0; i < length; i++) { @@ -200,6 +201,7 @@ public static BatchData deserializeBatchData(ByteBuffer buffer) { values[j] = new TsPrimitiveType.TsFloat(buffer.getFloat()); break; case BLOB: + case OBJECT: case STRING: case TEXT: int len = buffer.getInt(); @@ -297,6 +299,7 @@ public static void serializeTVPairs( dataOutputStream.writeInt(timeValuePairs.size()); switch (timeValuePairs.get(0).getValue().getDataType()) { case BLOB: + case OBJECT: case STRING: case TEXT: serializeTextTVPairs(timeValuePairs, dataOutputStream); @@ -337,6 +340,7 @@ public static void serializeTVPair( switch (dataType) { case STRING: case BLOB: + case OBJECT: case TEXT: dataOutputStream.writeLong(timeValuePair.getTimestamp()); if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { @@ -488,6 +492,7 @@ public static List deserializeTVPairs(ByteBuffer buffer) { deserializeBooleanTVPairs(buffer, ret, size, dataType); break; case BLOB: + case OBJECT: case STRING: case TEXT: deserializeTextTVPairs(buffer, ret, size, dataType); @@ -521,6 +526,7 @@ public static TimeValuePair deserializeTVPair(ByteBuffer buffer) { case BOOLEAN: return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.get() == 1)); case BLOB: + case OBJECT: case STRING: case TEXT: int bytesLen = buffer.getInt(); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index 7e9a0387722f0..ddcf8a002f7c9 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -548,6 +548,19 @@ struct TSConnectionInfoResp { 1: required list connectionInfoList } +struct TTableDeviceLeaderReq { + 1: required string dbName + 2: required list deviceId + 3: required list isSetTag + 4: required i64 time +} + +struct TTableDeviceLeaderResp { + 1: required common.TSStatus status + 2: required string ip + 3: required string port +} + service IClientRPCService { TSExecuteStatementResp executeQueryStatementV2(1:TSExecuteStatementReq req); @@ -683,5 +696,7 @@ service IClientRPCService { TSConnectionInfoResp fetchAllConnectionsInfo(); /** For other node's call */ - common.TSStatus testConnectionEmptyRPC() + common.TSStatus testConnectionEmptyRPC(); + + TTableDeviceLeaderResp fetchDeviceLeader(TTableDeviceLeaderReq req); } \ No newline at end of file diff --git a/pom.xml b/pom.xml index ace5230a5daad..d9bc272865c1e 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ 0.14.1 1.9 1.5.6-3 - 2.1.0-250709-SNAPSHOT + 2.1.0-object-SNAPSHOT