Skip to content

Commit 4f7b108

Browse files
CaideyipiCaideyipi
committed
Merge branch 'master' of https://github.com/apache/iotdb into optimize-kernel-log-noise
2 parents 63affe8 + 65cd197 commit 4f7b108

120 files changed

Lines changed: 3964 additions & 812 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎LICENSE-binary‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -237,17 +237,17 @@ io.dropwizard.metrics:metrics-core:4.2.19
237237
io.dropwizard.metrics:metrics-jvm:3.2.2
238238
com.librato.metrics:metrics-librato:5.1.0
239239
com.github.moquette-io.moquette:moquette-broker:0.18
240-
io.netty:netty-buffer:4.1.126.Final
241-
io.netty:netty-codec:4.1.126.Final
242-
io.netty:netty-codec-http:4.1.126.Final
243-
io.netty:netty-codec-mqtt:4.1.126.Final
244-
io.netty:netty-common:4.1.126.Final
245-
io.netty:netty-handler:4.1.126.Final
246-
io.netty:netty-resolver:4.1.126.Final
247-
io.netty:netty-transport:4.1.126.Final
248-
io.netty:netty-transport-native-epoll:4.1.126.Final:linux-aarch_64
249-
io.netty:netty-transport-native-epoll:4.1.126.Final:linux-x86_64
250-
io.netty:netty-transport-native-unix-common:4.1.126.Final
240+
io.netty:netty-buffer:4.1.134.Final
241+
io.netty:netty-codec:4.1.134.Final
242+
io.netty:netty-codec-http:4.1.134.Final
243+
io.netty:netty-codec-mqtt:4.1.134.Final
244+
io.netty:netty-common:4.1.134.Final
245+
io.netty:netty-handler:4.1.134.Final
246+
io.netty:netty-resolver:4.1.134.Final
247+
io.netty:netty-transport:4.1.134.Final
248+
io.netty:netty-transport-native-epoll:4.1.134.Final:linux-aarch_64
249+
io.netty:netty-transport-native-epoll:4.1.134.Final:linux-x86_64
250+
io.netty:netty-transport-native-unix-common:4.1.134.Final
251251
org.osgi:org.osgi.core:7.0.0
252252
org.osgi:osgi.cmpn:7.0.0
253253
org.ops4j.pax.jdbc:pax-jdbc-common:1.5.6

‎external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
3535
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert;
3636
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
37+
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
3738
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
3839
import org.apache.iotdb.db.utils.CommonUtils;
3940
import org.apache.iotdb.db.utils.SetThreadName;
@@ -139,7 +140,7 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext)
139140
.ifPresent(
140141
s ->
141142
CommonUtils.addStatementExecutionLatency(
142-
OperationType.EXECUTE_QUERY_STATEMENT, s.toString(), costTime));
143+
OperationType.EXECUTE_QUERY_STATEMENT, StatementType.QUERY.name(), costTime));
143144
}
144145
}
145146

@@ -194,7 +195,6 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex
194195
SqlParser relationSqlParser = new SqlParser();
195196
Long queryId = null;
196197
Statement statement = null;
197-
long startTime = System.nanoTime();
198198
try {
199199
IClientSession clientSession = SESSION_MANAGER.getCurrSession();
200200
statement = createStatement(sql, clientSession, relationSqlParser);
@@ -233,12 +233,6 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex
233233
} catch (Exception e) {
234234
return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build();
235235
} finally {
236-
long costTime = System.nanoTime() - startTime;
237-
Optional.ofNullable(statement)
238-
.ifPresent(
239-
s ->
240-
CommonUtils.addStatementExecutionLatency(
241-
OperationType.EXECUTE_NON_QUERY_PLAN, s.toString(), costTime));
242236
if (queryId != null) {
243237
COORDINATOR.cleanupQueryExecution(queryId);
244238
}

‎external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
3535
import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
3636
import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator;
37+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceLastCache;
3738
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
3839
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableId;
3940
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
@@ -196,6 +197,10 @@ public Response executeFastLastQueryStatement(
196197
for (final Map.Entry<String, Pair<TSDataType, TimeValuePair>> measurementLastEntry :
197198
device2MeasurementLastEntry.getValue().entrySet()) {
198199
final TimeValuePair tvPair = measurementLastEntry.getValue().getRight();
200+
if (tvPair == TableDeviceLastCache.PLACEHOLDER_EMPTY_COLUMN
201+
|| tvPair.getValue() == null) {
202+
continue;
203+
}
199204
valueList.add(tvPair.getValue().getStringValue());
200205
dataTypeList.add(tvPair.getValue().getDataType().name());
201206
targetDataSet.addTimestampsItem(tvPair.getTimestamp());

‎integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeSharedClusterIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.sql.ResultSetMetaData;
4343
import java.sql.SQLException;
4444
import java.sql.Statement;
45+
import java.sql.Types;
4546
import java.util.Arrays;
4647
import java.util.HashSet;
4748
import java.util.LinkedList;
@@ -271,8 +272,10 @@ public static void callInferenceTest(Statement statement, AINodeTestUtils.FakeMo
271272
try (ResultSet resultSet = statement.executeQuery(callInferenceSQL)) {
272273
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
273274
checkHeader(resultSetMetaData, "Time,output");
275+
Assert.assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(2));
274276
int count = 0;
275277
while (resultSet.next()) {
278+
resultSet.getDouble("output");
276279
count++;
277280
}
278281
Assert.assertEquals(DEFAULT_OUTPUT_LENGTH, count);
@@ -288,8 +291,10 @@ public static void callInferenceByDefaultTest(
288291
try (ResultSet resultSet = statement.executeQuery(callInferenceSQL)) {
289292
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
290293
checkHeader(resultSetMetaData, "output");
294+
Assert.assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(1));
291295
int count = 0;
292296
while (resultSet.next()) {
297+
resultSet.getDouble("output");
293298
count++;
294299
}
295300
Assert.assertTrue(count > 0);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ public void setUp() {
8989
.setDnConnectionTimeoutMs(600000)
9090
.setPipeMemoryManagementEnabled(false)
9191
.setIsPipeEnableMemoryCheck(false);
92+
senderEnv
93+
.getConfig()
94+
.getDataNodeConfig()
95+
.setMetricReporterType(Collections.singletonList("PROMETHEUS"));
9296

9397
receiverEnv
9498
.getConfig()
@@ -102,6 +106,10 @@ public void setUp() {
102106
.setDnConnectionTimeoutMs(600000)
103107
.setPipeMemoryManagementEnabled(false)
104108
.setIsPipeEnableMemoryCheck(false);
109+
receiverEnv
110+
.getConfig()
111+
.getDataNodeConfig()
112+
.setMetricReporterType(Collections.singletonList("PROMETHEUS"));
105113

106114
senderEnv.initClusterEnvironment(3, 3, 180);
107115
receiverEnv.initClusterEnvironment(3, 3, 180);

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAutoSplitIT.java‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,7 @@ public void testSingleEnv() throws Exception {
9999
final List<TShowPipeInfo> showPipeResult =
100100
client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList;
101101
showPipeResult.removeIf(i -> i.getId().startsWith("__consensus"));
102-
Assert.assertEquals(2, showPipeResult.size());
103-
Assert.assertTrue(
104-
(Objects.equals(showPipeResult.get(0).id, "a2b_history")
105-
&& Objects.equals(showPipeResult.get(1).id, "a2b_realtime"))
106-
|| (Objects.equals(showPipeResult.get(1).id, "a2b_history")
107-
&& Objects.equals(showPipeResult.get(0).id, "a2b_realtime")));
102+
assertAutoSplitResult(showPipeResult, "a2b");
108103
}
109104

110105
// Do not split for pipes without insertion or non-full
@@ -149,10 +144,11 @@ public void testSingleEnv() throws Exception {
149144
final List<TShowPipeInfo> showPipeResult =
150145
client.showPipe(new TShowPipeReq().setUserName(SessionConfig.DEFAULT_USER)).pipeInfoList;
151146
showPipeResult.removeIf(i -> i.getId().startsWith("__consensus"));
152-
Assert.assertTrue(
153-
showPipeResult.stream()
154-
.filter(i -> Objects.equals(i.id, "a2b_history"))
155-
.anyMatch(i -> i.pipeConnector.contains("enable-send-tsfile-limit=false")));
147+
assertAutoSplitResult(showPipeResult, "a2b");
148+
showPipeResult.stream()
149+
.filter(i -> Objects.equals(i.id, "a2b_history"))
150+
.forEach(
151+
i -> Assert.assertTrue(i.pipeConnector.contains("enable-send-tsfile-limit=false")));
156152
}
157153

158154
TestUtils.assertDataEventuallyOnEnv(
@@ -161,4 +157,18 @@ public void testSingleEnv() throws Exception {
161157
"Time,root.test.device.field,",
162158
Collections.singleton("1,2.0,"));
163159
}
160+
161+
private void assertAutoSplitResult(
162+
final List<TShowPipeInfo> showPipeResult, final String pipeName) {
163+
// The history pipe may have already been auto-dropped after snapshot transfer completes.
164+
Assert.assertTrue(
165+
showPipeResult.stream().anyMatch(i -> Objects.equals(i.id, pipeName + "_realtime")));
166+
Assert.assertFalse(showPipeResult.stream().anyMatch(i -> Objects.equals(i.id, pipeName)));
167+
Assert.assertTrue(
168+
showPipeResult.stream()
169+
.allMatch(
170+
i ->
171+
Objects.equals(i.id, pipeName + "_history")
172+
|| Objects.equals(i.id, pipeName + "_realtime")));
173+
}
164174
}

0 commit comments

Comments
 (0)