Skip to content

Commit 5ea4280

Browse files
VGalaxiesVGalaxies
committed
Merge upstream master into consensus-subscription
Resolve consensus subscription conflicts and address review comments.
2 parents 566ea99 + a420681 commit 5ea4280

275 files changed

Lines changed: 9842 additions & 1883 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎LICENSE-binary‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -237,17 +237,17 @@ io.dropwizard.metrics:metrics-core:4.2.19
237237
io.dropwizard.metrics:metrics-jvm:3.2.2
238238
com.librato.metrics:metrics-librato:5.1.0
239239
com.github.moquette-io.moquette:moquette-broker:0.18
240-
io.netty:netty-buffer:4.1.126.Final
241-
io.netty:netty-codec:4.1.126.Final
242-
io.netty:netty-codec-http:4.1.126.Final
243-
io.netty:netty-codec-mqtt:4.1.126.Final
244-
io.netty:netty-common:4.1.126.Final
245-
io.netty:netty-handler:4.1.126.Final
246-
io.netty:netty-resolver:4.1.126.Final
247-
io.netty:netty-transport:4.1.126.Final
248-
io.netty:netty-transport-native-epoll:4.1.126.Final:linux-aarch_64
249-
io.netty:netty-transport-native-epoll:4.1.126.Final:linux-x86_64
250-
io.netty:netty-transport-native-unix-common:4.1.126.Final
240+
io.netty:netty-buffer:4.1.134.Final
241+
io.netty:netty-codec:4.1.134.Final
242+
io.netty:netty-codec-http:4.1.134.Final
243+
io.netty:netty-codec-mqtt:4.1.134.Final
244+
io.netty:netty-common:4.1.134.Final
245+
io.netty:netty-handler:4.1.134.Final
246+
io.netty:netty-resolver:4.1.134.Final
247+
io.netty:netty-transport:4.1.134.Final
248+
io.netty:netty-transport-native-epoll:4.1.134.Final:linux-aarch_64
249+
io.netty:netty-transport-native-epoll:4.1.134.Final:linux-x86_64
250+
io.netty:netty-transport-native-unix-common:4.1.134.Final
251251
org.osgi:org.osgi.core:7.0.0
252252
org.osgi:osgi.cmpn:7.0.0
253253
org.ops4j.pax.jdbc:pax-jdbc-common:1.5.6

‎external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/table/v1/impl/RestApiServiceImpl.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
3535
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert;
3636
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
37+
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
3738
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
3839
import org.apache.iotdb.db.utils.CommonUtils;
3940
import org.apache.iotdb.db.utils.SetThreadName;
@@ -139,7 +140,7 @@ public Response executeQueryStatement(SQL sql, SecurityContext securityContext)
139140
.ifPresent(
140141
s ->
141142
CommonUtils.addStatementExecutionLatency(
142-
OperationType.EXECUTE_QUERY_STATEMENT, s.toString(), costTime));
143+
OperationType.EXECUTE_QUERY_STATEMENT, StatementType.QUERY.name(), costTime));
143144
}
144145
}
145146

@@ -194,7 +195,6 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex
194195
SqlParser relationSqlParser = new SqlParser();
195196
Long queryId = null;
196197
Statement statement = null;
197-
long startTime = System.nanoTime();
198198
try {
199199
IClientSession clientSession = SESSION_MANAGER.getCurrSession();
200200
statement = createStatement(sql, clientSession, relationSqlParser);
@@ -233,12 +233,6 @@ public Response executeNonQueryStatement(SQL sql, SecurityContext securityContex
233233
} catch (Exception e) {
234234
return Response.ok().entity(ExceptionHandler.tryCatchException(e)).build();
235235
} finally {
236-
long costTime = System.nanoTime() - startTime;
237-
Optional.ofNullable(statement)
238-
.ifPresent(
239-
s ->
240-
CommonUtils.addStatementExecutionLatency(
241-
OperationType.EXECUTE_NON_QUERY_PLAN, s.toString(), costTime));
242236
if (queryId != null) {
243237
COORDINATOR.cleanupQueryExecution(queryId);
244238
}

‎external-service-impl/rest/src/main/java/org/apache/iotdb/rest/protocol/v2/impl/RestApiServiceImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
3535
import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
3636
import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator;
37+
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceLastCache;
3738
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
3839
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableId;
3940
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
@@ -196,6 +197,10 @@ public Response executeFastLastQueryStatement(
196197
for (final Map.Entry<String, Pair<TSDataType, TimeValuePair>> measurementLastEntry :
197198
device2MeasurementLastEntry.getValue().entrySet()) {
198199
final TimeValuePair tvPair = measurementLastEntry.getValue().getRight();
200+
if (tvPair == TableDeviceLastCache.PLACEHOLDER_EMPTY_COLUMN
201+
|| tvPair.getValue() == null) {
202+
continue;
203+
}
199204
valueList.add(tvPair.getValue().getStringValue());
200205
dataTypeList.add(tvPair.getValue().getDataType().name());
201206
targetDataSet.addTimestampsItem(tvPair.getTimestamp());

‎integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeSharedClusterIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.sql.ResultSetMetaData;
4343
import java.sql.SQLException;
4444
import java.sql.Statement;
45+
import java.sql.Types;
4546
import java.util.Arrays;
4647
import java.util.HashSet;
4748
import java.util.LinkedList;
@@ -271,8 +272,10 @@ public static void callInferenceTest(Statement statement, AINodeTestUtils.FakeMo
271272
try (ResultSet resultSet = statement.executeQuery(callInferenceSQL)) {
272273
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
273274
checkHeader(resultSetMetaData, "Time,output");
275+
Assert.assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(2));
274276
int count = 0;
275277
while (resultSet.next()) {
278+
resultSet.getDouble("output");
276279
count++;
277280
}
278281
Assert.assertEquals(DEFAULT_OUTPUT_LENGTH, count);
@@ -288,8 +291,10 @@ public static void callInferenceByDefaultTest(
288291
try (ResultSet resultSet = statement.executeQuery(callInferenceSQL)) {
289292
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
290293
checkHeader(resultSetMetaData, "output");
294+
Assert.assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(1));
291295
int count = 0;
292296
while (resultSet.next()) {
297+
resultSet.getDouble("output");
293298
count++;
294299
}
295300
Assert.assertTrue(count > 0);

‎integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.iotdb.it.framework.IoTDBTestRunner;
3434
import org.apache.iotdb.itbase.category.ClusterIT;
3535
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
36+
import org.apache.iotdb.itbase.exception.InconsistentDataException;
3637

3738
import org.apache.tsfile.enums.TSDataType;
3839
import org.apache.tsfile.exception.write.WriteProcessException;
@@ -222,22 +223,29 @@ public void testWithNewModFileAndLoadAttributes()
222223
tmpDir.getAbsolutePath(), databaseName));
223224

224225
boolean databaseFound = false;
225-
out:
226226
for (int i = 0; i < 10; i++) {
227227
try (final ResultSet resultSet = statement.executeQuery("show databases")) {
228228
while (resultSet.next()) {
229229
final String currentDatabase = resultSet.getString(1);
230230
if (databaseName.equalsIgnoreCase(currentDatabase)) {
231231
databaseFound = true;
232-
break out;
232+
break;
233233
}
234234
}
235+
} catch (InconsistentDataException ignored) {
236+
// Async load propagates the new database metadata to different DataNodes at
237+
// slightly different times, so "show databases" may be inconsistent transiently.
238+
}
235239

236-
try {
237-
Thread.sleep(1000);
238-
} catch (InterruptedException e) {
239-
break;
240-
}
240+
if (databaseFound) {
241+
break;
242+
}
243+
244+
try {
245+
Thread.sleep(1000);
246+
} catch (InterruptedException e) {
247+
Thread.currentThread().interrupt();
248+
break;
241249
}
242250
}
243251
Assert.assertTrue(

‎integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.iotdb.itbase.category.ClusterIT;
2727
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
2828

29+
import org.awaitility.Awaitility;
2930
import org.junit.After;
3031
import org.junit.Assert;
3132
import org.junit.Before;
@@ -40,6 +41,7 @@
4041
import java.sql.Statement;
4142
import java.util.Arrays;
4243
import java.util.List;
44+
import java.util.concurrent.TimeUnit;
4345

4446
import static org.junit.Assert.assertEquals;
4547
import static org.junit.Assert.assertTrue;
@@ -297,17 +299,15 @@ public void setSpaceQuotaTest5() {
297299
"create timeseries root.sg0.wf01.wt02.status0 with datatype=BOOLEAN,encoding=PLAIN;");
298300
adminStmt.execute(
299301
"create timeseries root.sg0.wf01.wt02.status1 with datatype=BOOLEAN,encoding=PLAIN;");
300-
Thread.sleep(2000);
301-
ResultSet resultSet1 = adminStmt.executeQuery("show space quota root.sg0;");
302302
String ans1 =
303303
"root.sg0,diskSize,0.09765625G,0.0G"
304304
+ ",\n"
305305
+ "root.sg0,deviceNum,3,2"
306306
+ ",\n"
307307
+ "root.sg0,timeSeriesNum,5,3"
308308
+ ",\n";
309-
validateResultSet(resultSet1, ans1);
310-
} catch (InterruptedException | SQLException e) {
309+
validateResultSetEventually(adminStmt, "show space quota root.sg0;", ans1);
310+
} catch (SQLException e) {
311311
Assert.fail(e.getMessage());
312312
}
313313
}
@@ -509,6 +509,22 @@ public void showSpaceQuotaTest1() throws SQLException {
509509
}
510510
}
511511

512+
private void validateResultSetEventually(Statement statement, String sql, String ans) {
513+
Awaitility.await()
514+
.pollInSameThread()
515+
.pollDelay(0, TimeUnit.MILLISECONDS)
516+
.pollInterval(1, TimeUnit.SECONDS)
517+
.atMost(30, TimeUnit.SECONDS)
518+
.untilAsserted(
519+
() -> {
520+
try {
521+
validateResultSet(statement.executeQuery(sql), ans);
522+
} catch (SQLException e) {
523+
Assert.fail(e.getMessage());
524+
}
525+
});
526+
}
527+
512528
private void validateResultSet(ResultSet set, String ans) throws SQLException {
513529
try {
514530
StringBuilder builder = new StringBuilder();
@@ -521,12 +537,12 @@ private void validateResultSet(ResultSet set, String ans) throws SQLException {
521537
builder.append("\n");
522538
}
523539
String result = builder.toString();
524-
assertEquals(ans.length(), result.length());
540+
assertEquals(result, ans.length(), result.length());
525541
List<String> ansLines = Arrays.asList(ans.split("\n"));
526542
List<String> resultLines = Arrays.asList(result.split("\n"));
527-
assertEquals(ansLines.size(), resultLines.size());
543+
assertEquals(result, ansLines.size(), resultLines.size());
528544
for (String resultLine : resultLines) {
529-
assertTrue(ansLines.contains(resultLine));
545+
assertTrue(result, ansLines.contains(resultLine));
530546
}
531547
} finally {
532548
set.close();

‎integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.junit.After;
2727
import org.junit.Assert;
2828
import org.junit.Before;
29+
import org.junit.Ignore;
2930
import org.junit.Test;
3031
import org.junit.experimental.categories.Category;
3132
import org.junit.runners.Parameterized;
@@ -494,6 +495,26 @@ public void showCountTimeSeries() throws SQLException {
494495
}
495496
}
496497

498+
@Test
499+
@Ignore
500+
public void showCountTimeSeriesExcludeInternalDatabaseAndIncludeView() throws SQLException {
501+
try (Connection connection = EnvFactory.getEnv().getConnection();
502+
Statement statement = connection.createStatement()) {
503+
final long baseVisibleCount = queryCount(statement, "COUNT TIMESERIES root.ln*.**");
504+
statement.execute("CREATE DATABASE root.count_it");
505+
statement.execute(
506+
"CREATE TIMESERIES root.count_it.src.s1 WITH DATATYPE = INT32, ENCODING = PLAIN");
507+
statement.execute(
508+
"CREATE TIMESERIES root.count_it.src.s2 WITH DATATYPE = INT32, ENCODING = PLAIN");
509+
statement.execute("CREATE VIEW root.count_it.dst.v1 AS SELECT s1 FROM root.count_it.src;");
510+
511+
final long localCount = queryCount(statement, "COUNT TIMESERIES root.count_it.**");
512+
assertEquals(3L, localCount);
513+
assertEquals(
514+
baseVisibleCount + localCount, queryCount(statement, "COUNT TIMESERIES root.**"));
515+
}
516+
}
517+
497518
@Test
498519
public void showCountTimeSeriesWithTag() throws SQLException {
499520
try (Connection connection = EnvFactory.getEnv().getConnection();
@@ -865,4 +886,11 @@ public void showDeadbandInfo() throws SQLException {
865886
}
866887
}
867888
}
889+
890+
private long queryCount(final Statement statement, final String sql) throws SQLException {
891+
try (ResultSet resultSet = statement.executeQuery(sql)) {
892+
Assert.assertTrue(resultSet.next());
893+
return resultSet.getLong(1);
894+
}
895+
}
868896
}

‎integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.junit.After;
2828
import org.junit.Assert;
2929
import org.junit.Before;
30+
import org.junit.Ignore;
3031
import org.junit.Test;
3132
import org.junit.experimental.categories.Category;
3233
import org.junit.runners.Parameterized;
@@ -236,6 +237,39 @@ public void testShowTimeSeries() {
236237
}
237238
}
238239

240+
@Test
241+
@Ignore
242+
public void testCountTimeSeriesWithTimeConditionIncludesView() {
243+
try (Connection connection = EnvFactory.getEnv().getConnection();
244+
Statement statement = connection.createStatement()) {
245+
statement.execute("CREATE DATABASE root.view_count");
246+
statement.execute(
247+
"CREATE TIMESERIES root.view_count.src.s1 WITH DATATYPE = INT32, ENCODING = PLAIN");
248+
statement.execute(
249+
"CREATE TIMESERIES root.view_count.src.s2 WITH DATATYPE = INT32, ENCODING = PLAIN");
250+
statement.execute("CREATE VIEW root.view_count.dst.v1 AS SELECT s1 FROM root.view_count.src");
251+
252+
checkResultSet(
253+
statement,
254+
"count timeseries root.view_count.**",
255+
new HashSet<>(Collections.singletonList("3,")));
256+
257+
statement.execute("insert into root.view_count.src(timestamp,s1) values(1,1)");
258+
259+
checkResultSet(
260+
statement,
261+
"count timeseries root.view_count.** where time>0",
262+
new HashSet<>(Collections.singletonList("2,")));
263+
checkResultSet(
264+
statement,
265+
"count timeseries root.view_count.dst.** where time>0",
266+
new HashSet<>(Collections.singletonList("1,")));
267+
} catch (Exception e) {
268+
e.printStackTrace();
269+
Assert.fail(e.getMessage());
270+
}
271+
}
272+
239273
@Test
240274
public void testShowDevices() {
241275
try (Connection connection = EnvFactory.getEnv().getConnection();

0 commit comments

Comments
 (0)