Skip to content

Commit f0dc49e

Browse files
authored
Fix pipe text date conversion (#17984) (#17997)
1 parent eb41f56 commit f0dc49e

3 files changed

Lines changed: 96 additions & 5 deletions

File tree

integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,7 @@ private Binary[] createTestDataForString() {
536536
"enable = true",
537537
"true",
538538
"false",
539+
"2024-06-28 08:00:00",
539540
"12345678910",
540541
"123231232132131233213123123123123123131312",
541542
"123231232132131233213123123123123123131312.212312321312312",

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -804,16 +804,36 @@ private static long parseTimestamp(final String value) {
804804
}
805805

806806
private static int parseDate(final String value) {
807-
if (value == null || value.isEmpty()) {
807+
if (value == null) {
808808
return DEFAULT_DATE;
809809
}
810-
try {
811-
if (TypeInferenceUtils.isNumber(value)) {
812-
int date = Integer.parseInt(value);
810+
final String trimmedValue = StringUtils.trim(value);
811+
if (trimmedValue.isEmpty()) {
812+
return DEFAULT_DATE;
813+
}
814+
if (TypeInferenceUtils.isNumber(trimmedValue)) {
815+
try {
816+
int date = Integer.parseInt(trimmedValue);
813817
DateUtils.parseIntToLocalDate(date);
814818
return date;
819+
} catch (final Exception e) {
820+
return DEFAULT_DATE;
815821
}
816-
return DateTimeUtils.parseDateExpressionToInt(StringUtils.trim(value));
822+
}
823+
try {
824+
return DateTimeUtils.parseDateExpressionToInt(trimmedValue);
825+
} catch (final Exception e) {
826+
return parseDateTimeToDate(trimmedValue);
827+
}
828+
}
829+
830+
private static int parseDateTimeToDate(final String value) {
831+
try {
832+
return DateUtils.parseDateExpressionToInt(
833+
Instant.ofEpochMilli(
834+
DateTimeUtils.convertDatetimeStrToLong(value, ZoneOffset.UTC, 0, "ms"))
835+
.atZone(ZoneOffset.UTC)
836+
.toLocalDate());
817837
} catch (final Exception e) {
818838
return DEFAULT_DATE;
819839
}
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.pipe.receiver.transform.converter;
21+
22+
import org.apache.tsfile.common.conf.TSFileConfig;
23+
import org.apache.tsfile.enums.TSDataType;
24+
import org.apache.tsfile.utils.Binary;
25+
import org.apache.tsfile.utils.DateUtils;
26+
import org.junit.Assert;
27+
import org.junit.Test;
28+
29+
public class ValueConverterTest {
30+
31+
@Test
32+
public void testTextLikeDateTimeToDate() {
33+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
34+
final Binary dateTime = binary("2024-06-28 08:00:00");
35+
36+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(dateTime));
37+
Assert.assertEquals(expectedDate, ValueConverter.convertStringToDate(dateTime));
38+
Assert.assertEquals(expectedDate, ValueConverter.convertBlobToDate(dateTime));
39+
}
40+
41+
@Test
42+
public void testTextLikeDateToDateKeepsExistingFallbacks() {
43+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
44+
final int defaultDate = DateUtils.parseDateExpressionToInt("1970-01-01");
45+
46+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(binary("20240628")));
47+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(binary("2024-06-28")));
48+
Assert.assertEquals(defaultDate, ValueConverter.convertTextToDate(binary("12345678910")));
49+
Assert.assertEquals(defaultDate, ValueConverter.convertTextToDate(binary("invalid-date")));
50+
}
51+
52+
@Test
53+
public void testTextArrayDateTimeToDate() {
54+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
55+
final int defaultDate = DateUtils.parseDateExpressionToInt("1970-01-01");
56+
57+
final int[] dates =
58+
(int[])
59+
ArrayConverter.convert(
60+
TSDataType.TEXT,
61+
TSDataType.DATE,
62+
new Binary[] {binary("2024-06-28 08:00:00"), binary("invalid-date")});
63+
64+
Assert.assertArrayEquals(new int[] {expectedDate, defaultDate}, dates);
65+
}
66+
67+
private static Binary binary(final String value) {
68+
return new Binary(value, TSFileConfig.STRING_CHARSET);
69+
}
70+
}

0 commit comments

Comments
 (0)