Skip to content

Commit 90055d5

Browse files
CaideyipiCaideyipi
authored
Fix pipe receiver type conversion load path (#17849)
* Add pipe type conversion semantic ITs * Fix float-to-date semantic IT expectation * Add aligned stream pipe type conversion IT * Fix pipe table type conversion load path * Fix alter type IT pipe memory flakiness
1 parent eb4409d commit 90055d5

10 files changed

Lines changed: 1003 additions & 32 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public class IoTDBAlterTimeSeriesTypeIT {
9898

9999
@BeforeClass
100100
public static void setUp() throws Exception {
101+
EnvFactory.getEnv().getConfig().getCommonConfig().setPipeMemoryManagementEnabled(false);
101102
EnvFactory.getEnv().getConfig().getDataNodeConfig().setCompactionScheduleInterval(1000);
102103
EnvFactory.getEnv().initClusterEnvironment();
103104
}
@@ -878,11 +879,7 @@ public void testLoadAndAlter()
878879
session.executeQueryStatement("select count(s1) from " + database + ".load_and_alter");
879880
RowRecord rec;
880881
rec = dataSet.next();
881-
// Due to the operation of load tsfile execute directly, don't access memtable or generate
882-
// InsertNode object, so don't need to check the data type.
883-
// When query this measurement point, will only find the data of TSDataType.INT32. So this is
884-
// reason what cause we can't find the data of TSDataType.DOUBLE. So result is 9, is not 15.
885-
// assertEquals(15, rec.getFields().get(0).getLongV());
882+
// Before alter, DOUBLE TsFiles loaded directly are invisible under the existing INT32 schema.
886883
assertEquals(9, rec.getFields().get(0).getLongV());
887884
assertFalse(dataSet.hasNext());
888885
}

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/TypeConversionSemanticCase.java‎

Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.pipe.it.dual;
21+
22+
import org.apache.iotdb.rpc.RpcUtils;
23+
24+
import org.apache.tsfile.enums.TSDataType;
25+
import org.apache.tsfile.utils.BytesUtils;
26+
27+
import java.nio.charset.StandardCharsets;
28+
import java.time.ZoneOffset;
29+
import java.util.Arrays;
30+
import java.util.List;
31+
32+
public class TypeConversionSemanticCase {
33+
34+
public static final int ROW_COUNT = 3;
35+
36+
public static final List<TypeConversionSemanticCase> CASES =
37+
Arrays.asList(
38+
c(
39+
"bool_to_int32",
40+
TSDataType.BOOLEAN,
41+
TSDataType.INT32,
42+
values("true", "false", "true"),
43+
values("1", "0", "1")),
44+
c(
45+
"bool_to_int64",
46+
TSDataType.BOOLEAN,
47+
TSDataType.INT64,
48+
values("true", "false", "true"),
49+
values("1", "0", "1")),
50+
c(
51+
"bool_to_float",
52+
TSDataType.BOOLEAN,
53+
TSDataType.FLOAT,
54+
values("true", "false", "true"),
55+
values("1.0", "0.0", "1.0")),
56+
c(
57+
"bool_to_double",
58+
TSDataType.BOOLEAN,
59+
TSDataType.DOUBLE,
60+
values("true", "false", "true"),
61+
values("1.0", "0.0", "1.0")),
62+
c(
63+
"bool_to_text",
64+
TSDataType.BOOLEAN,
65+
TSDataType.TEXT,
66+
values("true", "false", "true"),
67+
values("true", "false", "true")),
68+
c(
69+
"bool_to_blob",
70+
TSDataType.BOOLEAN,
71+
TSDataType.BLOB,
72+
values("true", "false", "true"),
73+
values(blobValue("true"), blobValue("false"), blobValue("true"))),
74+
c(
75+
"bool_to_string",
76+
TSDataType.BOOLEAN,
77+
TSDataType.STRING,
78+
values("true", "false", "true"),
79+
values("true", "false", "true")),
80+
c(
81+
"bool_to_date",
82+
TSDataType.BOOLEAN,
83+
TSDataType.DATE,
84+
values("true", "false", "true"),
85+
values("1970-01-02", "1970-01-01", "1970-01-02")),
86+
c(
87+
"bool_to_timestamp",
88+
TSDataType.BOOLEAN,
89+
TSDataType.TIMESTAMP,
90+
values("true", "false", "true"),
91+
values(timestampValue(1), timestampValue(0), timestampValue(1))),
92+
c(
93+
"int32_to_boolean",
94+
TSDataType.INT32,
95+
TSDataType.BOOLEAN,
96+
values("0", "2", "-1"),
97+
values("false", "true", "true")),
98+
c(
99+
"int32_to_timestamp",
100+
TSDataType.INT32,
101+
TSDataType.TIMESTAMP,
102+
values("0", "1", "86400000"),
103+
values(timestampValue(0), timestampValue(1), timestampValue(86400000))),
104+
c(
105+
"int32_to_date",
106+
TSDataType.INT32,
107+
TSDataType.DATE,
108+
values("19700102", "20240229", "42"),
109+
values("1970-01-02", "2024-02-29", "1970-01-01")),
110+
c(
111+
"int64_to_int32",
112+
TSDataType.INT64,
113+
TSDataType.INT32,
114+
values("2147483648", "-2147483649", "42"),
115+
values("-2147483648", "2147483647", "42")),
116+
c(
117+
"int64_to_boolean",
118+
TSDataType.INT64,
119+
TSDataType.BOOLEAN,
120+
values("0", "2", "-1"),
121+
values("false", "true", "true")),
122+
c(
123+
"int64_to_date",
124+
TSDataType.INT64,
125+
TSDataType.DATE,
126+
values("19700102", "2147483648", "19700103"),
127+
values("1970-01-02", "1970-01-01", "1970-01-03")),
128+
c(
129+
"float_to_int32",
130+
TSDataType.FLOAT,
131+
TSDataType.INT32,
132+
values("2.9", "-2.9", "0.0"),
133+
values("2", "-2", "0")),
134+
c(
135+
"float_to_boolean",
136+
TSDataType.FLOAT,
137+
TSDataType.BOOLEAN,
138+
values("0.0", "0.1", "-0.1"),
139+
values("false", "true", "true")),
140+
c(
141+
"float_to_date",
142+
TSDataType.FLOAT,
143+
TSDataType.DATE,
144+
values("19700102.0", "19700104.0", "42.9"),
145+
values("1970-01-02", "1970-01-04", "1970-01-01")),
146+
c(
147+
"double_to_int64",
148+
TSDataType.DOUBLE,
149+
TSDataType.INT64,
150+
values("3.9", "-3.9", "0.0"),
151+
values("3", "-3", "0")),
152+
c(
153+
"double_to_boolean",
154+
TSDataType.DOUBLE,
155+
TSDataType.BOOLEAN,
156+
values("0.0", "0.1", "-0.1"),
157+
values("false", "true", "true")),
158+
c(
159+
"double_to_timestamp",
160+
TSDataType.DOUBLE,
161+
TSDataType.TIMESTAMP,
162+
values("1.9", "86400000.9", "0.0"),
163+
values(timestampValue(1), timestampValue(86400000), timestampValue(0))),
164+
c(
165+
"text_to_int32",
166+
TSDataType.TEXT,
167+
TSDataType.INT32,
168+
values("'123.9'", "'bad'", "'-123.9'"),
169+
values("123", "0", "-123")),
170+
c(
171+
"string_to_int64",
172+
TSDataType.STRING,
173+
TSDataType.INT64,
174+
values("'456.9'", "'bad'", "'-456.9'"),
175+
values("456", "0", "-456")),
176+
c(
177+
"blob_to_float",
178+
TSDataType.BLOB,
179+
TSDataType.FLOAT,
180+
values(blobSql("7.5"), blobSql("bad"), blobSql("-7.5")),
181+
values("7.5", "0.0", "-7.5")),
182+
c(
183+
"text_to_double",
184+
TSDataType.TEXT,
185+
TSDataType.DOUBLE,
186+
values("'8.25'", "'bad'", "'-8.25'"),
187+
values("8.25", "0.0", "-8.25")),
188+
c(
189+
"text_to_boolean",
190+
TSDataType.TEXT,
191+
TSDataType.BOOLEAN,
192+
values("'true'", "'1'", "'TrUe'"),
193+
values("true", "false", "true")),
194+
c(
195+
"string_to_boolean",
196+
TSDataType.STRING,
197+
TSDataType.BOOLEAN,
198+
values("'TRUE'", "'false'", "'yes'"),
199+
values("true", "false", "false")),
200+
c(
201+
"blob_to_boolean",
202+
TSDataType.BLOB,
203+
TSDataType.BOOLEAN,
204+
values(blobSql("true"), blobSql("FALSE"), blobSql("0")),
205+
values("true", "false", "false")),
206+
c(
207+
"text_to_timestamp",
208+
TSDataType.TEXT,
209+
TSDataType.TIMESTAMP,
210+
values("'86400000'", "'1970-01-02T00:00:00.000'", "'bad'"),
211+
values(timestampValue(86400000), timestampValue(86400000), timestampValue(0))),
212+
c(
213+
"string_to_timestamp",
214+
TSDataType.STRING,
215+
TSDataType.TIMESTAMP,
216+
values("'1970-01-03T00:00:00.000'", "'bad'", "'86400000'"),
217+
values(timestampValue(172800000), timestampValue(0), timestampValue(86400000))),
218+
c(
219+
"blob_to_timestamp",
220+
TSDataType.BLOB,
221+
TSDataType.TIMESTAMP,
222+
values(blobSql("bad"), blobSql("1"), blobSql("1970-01-02T00:00:00.000")),
223+
values(timestampValue(0), timestampValue(1), timestampValue(86400000))),
224+
c(
225+
"text_to_date",
226+
TSDataType.TEXT,
227+
TSDataType.DATE,
228+
values("'19700102'", "'1970-01-04'", "'bad'"),
229+
values("1970-01-02", "1970-01-04", "1970-01-01")),
230+
c(
231+
"string_to_date",
232+
TSDataType.STRING,
233+
TSDataType.DATE,
234+
values("'1970-01-03'", "'19700105'", "'1970-01-07'"),
235+
values("1970-01-03", "1970-01-05", "1970-01-07")),
236+
c(
237+
"blob_to_date",
238+
TSDataType.BLOB,
239+
TSDataType.DATE,
240+
values(blobSql("bad"), blobSql("1970-01-06"), blobSql("19700108")),
241+
values("1970-01-01", "1970-01-06", "1970-01-08")),
242+
c(
243+
"timestamp_to_date",
244+
TSDataType.TIMESTAMP,
245+
TSDataType.DATE,
246+
values("0", "86399999", "86400000"),
247+
values("1970-01-01", "1970-01-01", "1970-01-02")),
248+
c(
249+
"date_to_timestamp",
250+
TSDataType.DATE,
251+
TSDataType.TIMESTAMP,
252+
values("'1970-01-01'", "'1970-01-02'", "'1970-01-03'"),
253+
values(timestampValue(0), timestampValue(86400000), timestampValue(172800000))),
254+
c(
255+
"timestamp_to_boolean",
256+
TSDataType.TIMESTAMP,
257+
TSDataType.BOOLEAN,
258+
values("0", "-1", "1"),
259+
values("false", "true", "true")),
260+
c(
261+
"date_to_boolean",
262+
TSDataType.DATE,
263+
TSDataType.BOOLEAN,
264+
values("'1970-01-01'", "'1970-01-02'", "'1969-12-31'"),
265+
values("false", "true", "true")));
266+
267+
public final String measurement;
268+
public final TSDataType sourceType;
269+
public final TSDataType targetType;
270+
public final String[] sourceSqlValues;
271+
public final String[] expectedValues;
272+
273+
private TypeConversionSemanticCase(
274+
final String measurement,
275+
final TSDataType sourceType,
276+
final TSDataType targetType,
277+
final String[] sourceSqlValues,
278+
final String[] expectedValues) {
279+
this.measurement = measurement;
280+
this.sourceType = sourceType;
281+
this.targetType = targetType;
282+
this.sourceSqlValues = sourceSqlValues;
283+
this.expectedValues = expectedValues;
284+
}
285+
286+
private static TypeConversionSemanticCase c(
287+
final String measurement,
288+
final TSDataType sourceType,
289+
final TSDataType targetType,
290+
final String[] sourceSqlValues,
291+
final String[] expectedValues) {
292+
return new TypeConversionSemanticCase(
293+
measurement, sourceType, targetType, sourceSqlValues, expectedValues);
294+
}
295+
296+
private static String[] values(final String... values) {
297+
return values;
298+
}
299+
300+
public static String timestampValue(final long timestamp) {
301+
return RpcUtils.formatDatetime("default", "ms", timestamp, ZoneOffset.UTC);
302+
}
303+
304+
private static String blobSql(final String value) {
305+
final StringBuilder builder = new StringBuilder("X'");
306+
for (final byte b : value.getBytes(StandardCharsets.UTF_8)) {
307+
builder.append(String.format("%02x", b & 0xFF));
308+
}
309+
return builder.append("'").toString();
310+
}
311+
312+
private static String blobValue(final String value) {
313+
return BytesUtils.parseBlobByteArrayToString(value.getBytes(StandardCharsets.UTF_8));
314+
}
315+
}

0 commit comments

Comments
 (0)