Skip to content

Commit 629860a

Browse files
authored
fix: Updated TSFile version to fix Load failure in creating aligned time series (#16502)
* Updated TSFile version to fix Load failure in creating aligned time series * add UT * update
1 parent 5621279 commit 629860a

1 file changed

Lines changed: 173 additions & 0 deletions

File tree

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.queryengine.plan.analyze.load;
21+
22+
import org.apache.tsfile.enums.TSDataType;
23+
import org.apache.tsfile.file.metadata.IDeviceID;
24+
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
25+
import org.apache.tsfile.read.TsFileSequenceReader;
26+
import org.apache.tsfile.read.TsFileSequenceReaderTimeseriesMetadataIterator;
27+
import org.apache.tsfile.write.TsFileWriter;
28+
import org.apache.tsfile.write.record.Tablet;
29+
import org.apache.tsfile.write.schema.IMeasurementSchema;
30+
import org.apache.tsfile.write.schema.MeasurementSchema;
31+
import org.junit.Assert;
32+
import org.junit.Test;
33+
34+
import java.io.File;
35+
import java.nio.charset.StandardCharsets;
36+
import java.util.ArrayList;
37+
import java.util.HashSet;
38+
import java.util.List;
39+
import java.util.Map;
40+
import java.util.Set;
41+
42+
public class TimeseriesMetadataIteratorTest {
43+
44+
private static final int DEVICE_COUNT = 1000;
45+
46+
private static final int COLUMN_COUNT = 3;
47+
48+
private static final long FIXED_TIMESTAMP = 1L;
49+
50+
@Test
51+
public void testTimeseriesMetadataIterator() throws Exception {
52+
String outputPath = "testTsFile.tsfile";
53+
File file = createTsFile(outputPath);
54+
55+
IDeviceID iDeviceID = null;
56+
Set<IDeviceID> set = new HashSet<>();
57+
try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) {
58+
TsFileSequenceReaderTimeseriesMetadataIterator iterator =
59+
new TsFileSequenceReaderTimeseriesMetadataIterator(reader, true, 2);
60+
while (iterator.hasNext()) {
61+
Map<IDeviceID, List<TimeseriesMetadata>> map = iterator.next();
62+
for (Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry : map.entrySet()) {
63+
if (iDeviceID != null) {
64+
if (!iDeviceID.equals(entry.getKey())) {
65+
if (set.contains(iDeviceID)) {
66+
Assert.fail("time series metadata iterator needs to be ordered by device id");
67+
}
68+
set.add(iDeviceID);
69+
}
70+
}
71+
iDeviceID = entry.getKey();
72+
}
73+
}
74+
} finally {
75+
if (file.exists()) {
76+
file.delete();
77+
}
78+
}
79+
}
80+
81+
public static File createTsFile(String outputPath) throws Exception {
82+
File file = new File(outputPath);
83+
if (file.exists()) {
84+
file.delete();
85+
}
86+
87+
try (TsFileWriter tsFileWriter = new TsFileWriter(file)) {
88+
createSchema(tsFileWriter);
89+
for (int deviceIndex = 0; deviceIndex < DEVICE_COUNT; deviceIndex++) {
90+
String deviceId = "root.d." + "device" + deviceIndex;
91+
Tablet tablet = createTablet(deviceId);
92+
93+
tsFileWriter.writeAligned(tablet);
94+
tsFileWriter.flush();
95+
}
96+
}
97+
98+
return file;
99+
}
100+
101+
private static void createSchema(TsFileWriter tsFileWriter) throws Exception {
102+
List<IMeasurementSchema> schemaList = new ArrayList<>();
103+
for (int colIndex = 0; colIndex < COLUMN_COUNT; colIndex++) {
104+
String measurementName = "s" + colIndex;
105+
TSDataType dataType = getDataType(colIndex);
106+
schemaList.add(new MeasurementSchema(measurementName, dataType));
107+
}
108+
for (int deviceIndex = 0; deviceIndex < DEVICE_COUNT; deviceIndex++) {
109+
tsFileWriter.registerAlignedTimeseries("root.d." + "device" + deviceIndex, schemaList);
110+
}
111+
}
112+
113+
private static TSDataType getDataType(int colIndex) {
114+
TSDataType[] types = {
115+
TSDataType.INT32,
116+
TSDataType.INT64,
117+
TSDataType.FLOAT,
118+
TSDataType.DOUBLE,
119+
TSDataType.BOOLEAN,
120+
TSDataType.TEXT
121+
};
122+
return types[colIndex % types.length];
123+
}
124+
125+
private static Tablet createTablet(String deviceId) {
126+
List<IMeasurementSchema> schemaList = new ArrayList<>();
127+
128+
for (int colIndex = 0; colIndex < COLUMN_COUNT; colIndex++) {
129+
String measurementName = "s" + colIndex;
130+
TSDataType dataType = getDataType(colIndex);
131+
132+
schemaList.add(new MeasurementSchema(measurementName, dataType));
133+
}
134+
135+
Tablet tablet = new Tablet(deviceId, schemaList, 1);
136+
137+
tablet.initBitMaps();
138+
139+
tablet.addTimestamp(0, FIXED_TIMESTAMP);
140+
141+
for (int colIndex = 0; colIndex < COLUMN_COUNT; colIndex++) {
142+
String measurementName = "s" + colIndex;
143+
TSDataType dataType = getDataType(colIndex);
144+
145+
Object value = generateValue(dataType, colIndex);
146+
tablet.addValue(measurementName, 0, value);
147+
}
148+
149+
tablet.setRowSize(1);
150+
151+
return tablet;
152+
}
153+
154+
private static Object generateValue(TSDataType dataType, int colIndex) {
155+
switch (dataType) {
156+
case INT32:
157+
return colIndex % 1000;
158+
case INT64:
159+
return (long) (colIndex * 1000);
160+
case FLOAT:
161+
return (float) (colIndex * 0.1);
162+
case DOUBLE:
163+
return (double) (colIndex * 0.01);
164+
case BOOLEAN:
165+
return colIndex % 2 == 0;
166+
case TEXT:
167+
return new org.apache.tsfile.utils.Binary(
168+
("value_" + colIndex).getBytes(StandardCharsets.UTF_8));
169+
default:
170+
return colIndex;
171+
}
172+
}
173+
}

0 commit comments

Comments
 (0)