Skip to content

Commit b0033af

Browse files
CaideyipiCaideyipi
authored
Optimized the overall performance of IoTDB & Fixed the NPE in LimitOperatorTest (#17664) (#17819)
* Opt * Update UnclosedFileScanHandleImpl.java * Update StorageEngine.java * Update ClosedFileScanHandleImpl.java * column index * spt * Address performance review comments * fix (cherry picked from commit 89730b1)
1 parent 24be79a commit b0033af

15 files changed

Lines changed: 363 additions & 309 deletions

File tree

‎iotdb-client/client-cpp/src/main/SessionDataSet.cpp‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -211,39 +211,41 @@ const std::vector<std::string>& SessionDataSet::DataIterator::getColumnTypeList(
211211

212212
shared_ptr<RowRecord> SessionDataSet::constructRowRecordFromValueArray() {
213213
std::vector<Field> outFields;
214-
for (int i = iotdbRpcDataSet_->getValueColumnStartIndex(); i < iotdbRpcDataSet_->getColumnSize(); i++) {
214+
const int32_t valueColumnStartIndex = iotdbRpcDataSet_->getValueColumnStartIndex();
215+
const int32_t columnSize = iotdbRpcDataSet_->getColumnSize();
216+
outFields.reserve(columnSize - valueColumnStartIndex);
217+
for (int32_t columnIndex = valueColumnStartIndex + 1; columnIndex <= columnSize; ++columnIndex) {
215218
Field field;
216-
std::string columnName = iotdbRpcDataSet_->getColumnNameList().at(i);
217-
if (!iotdbRpcDataSet_->isNullByColumnName(columnName)) {
218-
TSDataType::TSDataType dataType = iotdbRpcDataSet_->getDataType(columnName);
219+
if (!iotdbRpcDataSet_->isNullByIndex(columnIndex)) {
220+
TSDataType::TSDataType dataType = iotdbRpcDataSet_->getDataTypeByIndex(columnIndex);
219221
field.dataType = dataType;
220222
switch (dataType) {
221223
case TSDataType::BOOLEAN:
222-
field.boolV = iotdbRpcDataSet_->getBoolean(columnName);
224+
field.boolV = iotdbRpcDataSet_->getBooleanByIndex(columnIndex);
223225
break;
224226
case TSDataType::INT32:
225-
field.intV = iotdbRpcDataSet_->getInt(columnName);
227+
field.intV = iotdbRpcDataSet_->getIntByIndex(columnIndex);
226228
break;
227229
case TSDataType::DATE:
228-
field.dateV = parseIntToDate(iotdbRpcDataSet_->getInt(columnName));
230+
field.dateV = iotdbRpcDataSet_->getDateByIndex(columnIndex);
229231
break;
230232
case TSDataType::INT64:
231233
case TSDataType::TIMESTAMP:
232-
field.longV = iotdbRpcDataSet_->getLong(columnName);
234+
field.longV = iotdbRpcDataSet_->getLongByIndex(columnIndex);
233235
break;
234236
case TSDataType::FLOAT:
235-
field.floatV = iotdbRpcDataSet_->getFloat(columnName);
237+
field.floatV = iotdbRpcDataSet_->getFloatByIndex(columnIndex);
236238
break;
237239
case TSDataType::DOUBLE:
238-
field.doubleV = iotdbRpcDataSet_->getDouble(columnName);
240+
field.doubleV = iotdbRpcDataSet_->getDoubleByIndex(columnIndex);
239241
break;
240242
case TSDataType::TEXT:
241243
case TSDataType::BLOB:
242244
case TSDataType::STRING:
243-
field.stringV = iotdbRpcDataSet_->getBinary(columnName)->getStringValue();
245+
field.stringV = iotdbRpcDataSet_->getBinaryByIndex(columnIndex)->getStringValue();
244246
break;
245247
default:
246-
throw new UnSupportedDataTypeException("Data type %s is not supported." + dataType);
248+
throw UnSupportedDataTypeException("Data type %s is not supported." + dataType);
247249
}
248250
}
249251
outFields.emplace_back(field);

‎iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java‎

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@
4040
import java.util.List;
4141
import java.util.Map;
4242

43-
import static org.apache.iotdb.rpc.IoTDBRpcDataSet.START_INDEX;
44-
4543
public class SessionDataSet implements ISessionDataSet {
4644

4745
private final IoTDBRpcDataSet ioTDBRpcDataSet;
@@ -176,56 +174,45 @@ public boolean hasNext() throws StatementExecutionException, IoTDBConnectionExce
176174
}
177175

178176
private RowRecord constructRowRecordFromValueArray() throws StatementExecutionException {
179-
List<Field> outFields = new ArrayList<>();
180-
for (int i = 0; i < ioTDBRpcDataSet.columnSize; i++) {
177+
int valueColumnStartIndex = ioTDBRpcDataSet.getValueColumnStartIndex();
178+
int columnSize = ioTDBRpcDataSet.getColumnSize();
179+
List<Field> outFields = new ArrayList<>(columnSize - valueColumnStartIndex);
180+
for (int columnIndex = valueColumnStartIndex + 1; columnIndex <= columnSize; columnIndex++) {
181181
Field field;
182-
183-
int index = i + 1;
184-
int datasetColumnIndex = i + START_INDEX;
185-
if (ioTDBRpcDataSet.ignoreTimeStamp) {
186-
index--;
187-
datasetColumnIndex--;
188-
}
189-
int loc =
190-
ioTDBRpcDataSet.columnOrdinalMap.get(ioTDBRpcDataSet.columnNameList.get(index))
191-
- START_INDEX;
192-
193-
if (!ioTDBRpcDataSet.isNull(datasetColumnIndex)) {
194-
TSDataType dataType = ioTDBRpcDataSet.columnTypeDeduplicatedList.get(loc);
182+
if (!ioTDBRpcDataSet.isNull(columnIndex)) {
183+
TSDataType dataType = ioTDBRpcDataSet.getDataType(columnIndex);
195184
field = new Field(dataType);
196185
switch (dataType) {
197186
case BOOLEAN:
198-
boolean booleanValue = ioTDBRpcDataSet.getBoolean(datasetColumnIndex);
187+
boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnIndex);
199188
field.setBoolV(booleanValue);
200189
break;
201190
case INT32:
202191
case DATE:
203-
int intValue = ioTDBRpcDataSet.getInt(datasetColumnIndex);
192+
int intValue = ioTDBRpcDataSet.getInt(columnIndex);
204193
field.setIntV(intValue);
205194
break;
206195
case INT64:
207196
case TIMESTAMP:
208-
long longValue = ioTDBRpcDataSet.getLong(datasetColumnIndex);
197+
long longValue = ioTDBRpcDataSet.getLong(columnIndex);
209198
field.setLongV(longValue);
210199
break;
211200
case FLOAT:
212-
float floatValue = ioTDBRpcDataSet.getFloat(datasetColumnIndex);
201+
float floatValue = ioTDBRpcDataSet.getFloat(columnIndex);
213202
field.setFloatV(floatValue);
214203
break;
215204
case DOUBLE:
216-
double doubleValue = ioTDBRpcDataSet.getDouble(datasetColumnIndex);
205+
double doubleValue = ioTDBRpcDataSet.getDouble(columnIndex);
217206
field.setDoubleV(doubleValue);
218207
break;
219208
case TEXT:
220209
case BLOB:
221210
case STRING:
222-
field.setBinaryV(ioTDBRpcDataSet.getBinary(datasetColumnIndex));
211+
field.setBinaryV(ioTDBRpcDataSet.getBinary(columnIndex));
223212
break;
224213
default:
225214
throw new UnSupportedDataTypeException(
226-
String.format(
227-
"Data type %s is not supported.",
228-
ioTDBRpcDataSet.columnTypeDeduplicatedList.get(i)));
215+
String.format("Data type %s is not supported.", dataType));
229216
}
230217
} else {
231218
field = new Field(null);

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -127,20 +127,15 @@ public IoTDBJDBCDataSet(
127127

128128
// deduplicate and map
129129
if (columnNameIndex != null) {
130-
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
131-
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
132-
for (int i = 0; i < deduplicatedColumnSize; i++) {
133-
columnTypeDeduplicatedList.add(null);
134-
}
130+
this.columnTypeDeduplicatedList =
131+
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
135132
for (int i = 0; i < columnNameList.size(); i++) {
136133
String name = columnNameList.get(i);
137134
this.columnNameList.add(name);
138135
this.columnTypeList.add(columnTypeList.get(i));
139136
if (!columnOrdinalMap.containsKey(name)) {
140137
int index = columnNameIndex.get(name);
141-
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
142-
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
143-
}
138+
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
144139
columnOrdinalMap.put(name, index + START_INDEX);
145140
}
146141
}
@@ -241,11 +236,8 @@ public IoTDBJDBCDataSet(
241236

242237
// deduplicate and map
243238
if (columnNameIndex != null) {
244-
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
245-
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
246-
for (int i = 0; i < deduplicatedColumnSize; i++) {
247-
columnTypeDeduplicatedList.add(null);
248-
}
239+
this.columnTypeDeduplicatedList =
240+
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
249241
for (int i = 0; i < columnNameList.size(); i++) {
250242
String name = "";
251243
if (sgList != null
@@ -261,9 +253,7 @@ public IoTDBJDBCDataSet(
261253
// "Time".equals(name) -> to allow the Time column appear in value columns
262254
if (!columnOrdinalMap.containsKey(name) || "Time".equals(name)) {
263255
int index = columnNameIndex.get(name);
264-
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
265-
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
266-
}
256+
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
267257
columnOrdinalMap.put(name, index + START_INDEX);
268258
}
269259
}
@@ -318,6 +308,31 @@ public IoTDBJDBCDataSet(
318308
this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining());
319309
}
320310

311+
private static int getDeduplicatedColumnSize(Map<String, Integer> columnNameIndex) {
312+
int deduplicatedColumnSize = 0;
313+
for (Integer index : columnNameIndex.values()) {
314+
if (index != null && index + 1 > deduplicatedColumnSize) {
315+
deduplicatedColumnSize = index + 1;
316+
}
317+
}
318+
return deduplicatedColumnSize;
319+
}
320+
321+
private static List<TSDataType> initDeduplicatedColumnTypes(int deduplicatedColumnSize) {
322+
List<TSDataType> columnTypes = new ArrayList<>(deduplicatedColumnSize);
323+
for (int i = 0; i < deduplicatedColumnSize; i++) {
324+
columnTypes.add(null);
325+
}
326+
return columnTypes;
327+
}
328+
329+
private static void setColumnTypeIfAbsent(
330+
List<TSDataType> columnTypeDeduplicatedList, int index, String columnType) {
331+
if (columnTypeDeduplicatedList.get(index) == null) {
332+
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnType));
333+
}
334+
}
335+
321336
public void close() throws StatementExecutionException, TException {
322337
if (isClosed) {
323338
return;

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,4 +634,12 @@ public void checkRecord() throws StatementExecutionException {
634634
throw new StatementExecutionException("No record remains");
635635
}
636636
}
637+
638+
public int getValueColumnStartIndex() {
639+
return ignoreTimeStamp ? 0 : 1;
640+
}
641+
642+
public int getColumnSize() {
643+
return columnNameList.size();
644+
}
637645
}

0 commit comments

Comments
 (0)