Skip to content

Commit d25eda6

Browse files
jt2594838JackieTien97
jt2594838
authored and
JackieTien97
committed
Add configs to set default compression for each data type (#16117)
(cherry picked from commit 1dd3a0c)
1 parent 3b85dde commit d25eda6

2 files changed

Lines changed: 153 additions & 37 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java‎

Lines changed: 123 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.iotdb.db.protocol.thrift.OperationType;
2525
import org.apache.iotdb.isession.ISession;
2626
import org.apache.iotdb.isession.SessionDataSet;
27+
import org.apache.iotdb.isession.SessionDataSet.DataIterator;
2728
import org.apache.iotdb.it.env.EnvFactory;
2829
import org.apache.iotdb.it.framework.IoTDBTestRunner;
2930
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -53,7 +54,6 @@
5354
import org.apache.tsfile.write.schema.MeasurementSchema;
5455
import org.junit.After;
5556
import org.junit.AfterClass;
56-
import org.junit.Assert;
5757
import org.junit.BeforeClass;
5858
import org.junit.Test;
5959
import org.junit.experimental.categories.Category;
@@ -63,8 +63,10 @@
6363

6464
import java.io.File;
6565
import java.io.IOException;
66+
import java.nio.charset.StandardCharsets;
6667
import java.security.SecureRandom;
6768
import java.sql.Timestamp;
69+
import java.time.LocalDate;
6870
import java.util.ArrayList;
6971
import java.util.Arrays;
7072
import java.util.Collections;
@@ -74,6 +76,7 @@
7476
import java.util.Map;
7577
import java.util.Set;
7678
import java.util.concurrent.TimeUnit;
79+
import java.util.stream.Collectors;
7780

7881
import static org.junit.Assert.assertArrayEquals;
7982
import static org.junit.Assert.assertEquals;
@@ -214,7 +217,7 @@ public void insertPartialTabletsTest() {
214217
}
215218
fail();
216219
} catch (Exception e) {
217-
Assert.assertTrue(e.getMessage().contains("data type of root.sg.d2.s2 is not consistent"));
220+
assertTrue(e.getMessage().contains("data type of root.sg.d2.s2 is not consistent"));
218221
}
219222
}
220223

@@ -713,7 +716,7 @@ public void insertTabletWithWrongTimestampPrecisionTest() {
713716
tablet.reset();
714717
}
715718
} catch (Exception e) {
716-
Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
719+
assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
717720
}
718721
}
719722

@@ -741,7 +744,7 @@ public void insertTabletWithDuplicatedMeasurementsTest() {
741744
tablet.reset();
742745
}
743746
} catch (Exception e) {
744-
Assert.assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s0"));
747+
assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s0"));
745748
}
746749
}
747750

@@ -1123,7 +1126,7 @@ public void insertOneDeviceRecordsWithDuplicatedMeasurementsTest() {
11231126
Boolean.TRUE);
11241127
session.insertRecordsOfOneDevice("root.sg.d1", times, measurements, datatypes, values);
11251128
} catch (Exception e) {
1126-
Assert.assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s2"));
1129+
assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s2"));
11271130
}
11281131
}
11291132

@@ -1178,7 +1181,7 @@ public void insertRecordsWithDuplicatedMeasurementsTest() {
11781181
Boolean.TRUE);
11791182
session.insertRecords(devices, times, measurements, datatypes, values);
11801183
} catch (Exception e) {
1181-
Assert.assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s2"));
1184+
assertTrue(e.getMessage().contains("Insertion contains duplicated measurement: s2"));
11821185
}
11831186
}
11841187

@@ -1211,11 +1214,11 @@ public void insertRecordsWithExpiredDataTest()
12111214
session.insertRecords(devices, times, measurements, datatypes, values);
12121215
fail();
12131216
} catch (Exception e) {
1214-
Assert.assertTrue(e.getMessage().contains("less than ttl time bound"));
1217+
assertTrue(e.getMessage().contains("less than ttl time bound"));
12151218
}
12161219
session.executeNonQueryStatement("unset ttl to root.sg.d1");
12171220
SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg.d1");
1218-
Assert.assertFalse(dataSet.hasNext());
1221+
assertFalse(dataSet.hasNext());
12191222
}
12201223
}
12211224

@@ -1889,14 +1892,14 @@ public void illegalDatabaseNameTest() {
18891892
session.createDatabase("");
18901893
fail();
18911894
} catch (StatementExecutionException e) {
1892-
Assert.assertTrue(e.getMessage().contains(" is not a legal path"));
1895+
assertTrue(e.getMessage().contains(" is not a legal path"));
18931896
}
18941897

18951898
try {
18961899
session.deleteDatabases(Arrays.asList("root.db", ""));
18971900
fail();
18981901
} catch (StatementExecutionException e) {
1899-
Assert.assertTrue(e.getMessage().contains(" is not a legal path"));
1902+
assertTrue(e.getMessage().contains(" is not a legal path"));
19001903
}
19011904

19021905
session.deleteDatabase("root.db");
@@ -2166,38 +2169,37 @@ public void testQueryAllDataType() throws IoTDBConnectionException, StatementExe
21662169
session.insertTablet(tablet);
21672170

21682171
try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg.d1")) {
2169-
SessionDataSet.DataIterator iterator = dataSet.iterator();
2172+
DataIterator iterator = dataSet.iterator();
21702173
int count = 0;
21712174
while (iterator.next()) {
21722175
count++;
2173-
Assert.assertFalse(iterator.isNull("root.sg.d1.s1"));
2174-
Assert.assertEquals(1, iterator.getInt("root.sg.d1.s1"));
2175-
Assert.assertFalse(iterator.isNull("root.sg.d1.s2"));
2176-
Assert.assertEquals(1L, iterator.getLong("root.sg.d1.s2"));
2177-
Assert.assertFalse(iterator.isNull("root.sg.d1.s3"));
2178-
Assert.assertEquals(0, iterator.getFloat("root.sg.d1.s3"), 0.01);
2179-
Assert.assertFalse(iterator.isNull("root.sg.d1.s4"));
2180-
Assert.assertEquals(0, iterator.getDouble("root.sg.d1.s4"), 0.01);
2181-
Assert.assertFalse(iterator.isNull("root.sg.d1.s5"));
2182-
Assert.assertEquals("text_value", iterator.getString("root.sg.d1.s5"));
2183-
Assert.assertFalse(iterator.isNull("root.sg.d1.s6"));
2176+
assertFalse(iterator.isNull("root.sg.d1.s1"));
2177+
assertEquals(1, iterator.getInt("root.sg.d1.s1"));
2178+
assertFalse(iterator.isNull("root.sg.d1.s2"));
2179+
assertEquals(1L, iterator.getLong("root.sg.d1.s2"));
2180+
assertFalse(iterator.isNull("root.sg.d1.s3"));
2181+
assertEquals(0, iterator.getFloat("root.sg.d1.s3"), 0.01);
2182+
assertFalse(iterator.isNull("root.sg.d1.s4"));
2183+
assertEquals(0, iterator.getDouble("root.sg.d1.s4"), 0.01);
2184+
assertFalse(iterator.isNull("root.sg.d1.s5"));
2185+
assertEquals("text_value", iterator.getString("root.sg.d1.s5"));
2186+
assertFalse(iterator.isNull("root.sg.d1.s6"));
21842187
assertTrue(iterator.getBoolean("root.sg.d1.s6"));
2185-
Assert.assertFalse(iterator.isNull("root.sg.d1.s7"));
2186-
Assert.assertEquals(new Timestamp(1), iterator.getTimestamp("root.sg.d1.s7"));
2187-
Assert.assertFalse(iterator.isNull("root.sg.d1.s8"));
2188-
Assert.assertEquals(new Binary(new byte[] {1}), iterator.getBlob("root.sg.d1.s8"));
2189-
Assert.assertFalse(iterator.isNull("root.sg.d1.s9"));
2190-
Assert.assertEquals("string_value", iterator.getString("root.sg.d1.s9"));
2191-
Assert.assertFalse(iterator.isNull("root.sg.d1.s10"));
2192-
Assert.assertEquals(
2193-
DateUtils.parseIntToLocalDate(20250403), iterator.getDate("root.sg.d1.s10"));
2194-
Assert.assertTrue(iterator.isNull("root.sg.d1.s11"));
2195-
Assert.assertNull(iterator.getTimestamp("root.sg.d1.s11"));
2196-
2197-
Assert.assertEquals(new Timestamp(0), iterator.getTimestamp("Time"));
2198-
Assert.assertFalse(iterator.isNull("Time"));
2188+
assertFalse(iterator.isNull("root.sg.d1.s7"));
2189+
assertEquals(new Timestamp(1), iterator.getTimestamp("root.sg.d1.s7"));
2190+
assertFalse(iterator.isNull("root.sg.d1.s8"));
2191+
assertEquals(new Binary(new byte[] {1}), iterator.getBlob("root.sg.d1.s8"));
2192+
assertFalse(iterator.isNull("root.sg.d1.s9"));
2193+
assertEquals("string_value", iterator.getString("root.sg.d1.s9"));
2194+
assertFalse(iterator.isNull("root.sg.d1.s10"));
2195+
assertEquals(DateUtils.parseIntToLocalDate(20250403), iterator.getDate("root.sg.d1.s10"));
2196+
assertTrue(iterator.isNull("root.sg.d1.s11"));
2197+
assertNull(iterator.getTimestamp("root.sg.d1.s11"));
2198+
2199+
assertEquals(new Timestamp(0), iterator.getTimestamp("Time"));
2200+
assertFalse(iterator.isNull("Time"));
21992201
}
2200-
Assert.assertEquals(tablet.getRowSize(), count);
2202+
assertEquals(tablet.getRowSize(), count);
22012203
}
22022204
}
22032205
}
@@ -2216,4 +2218,88 @@ public void testInsertWrongTypeRecord() throws IoTDBConnectionException {
22162218
Collections.singletonList(1L)));
22172219
}
22182220
}
2221+
2222+
@Test
2223+
public void testAlterDefaultCompression()
2224+
throws IoTDBConnectionException, StatementExecutionException {
2225+
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
2226+
List<TSDataType> types =
2227+
Arrays.asList(
2228+
TSDataType.BOOLEAN,
2229+
TSDataType.INT32,
2230+
TSDataType.DATE,
2231+
TSDataType.INT64,
2232+
TSDataType.TIMESTAMP,
2233+
TSDataType.FLOAT,
2234+
TSDataType.DOUBLE,
2235+
TSDataType.TEXT,
2236+
TSDataType.STRING,
2237+
TSDataType.BLOB);
2238+
List<String> measurements =
2239+
types.stream().map(dataType -> "__" + dataType.toString()).collect(Collectors.toList());
2240+
List<Object> values =
2241+
Arrays.asList(
2242+
false,
2243+
1,
2244+
LocalDate.of(1000, 1, 1),
2245+
1L,
2246+
1L,
2247+
1.0f,
2248+
1.0,
2249+
new Binary("1".getBytes(StandardCharsets.UTF_8)),
2250+
new Binary("1".getBytes(StandardCharsets.UTF_8)),
2251+
new Binary("1".getBytes(StandardCharsets.UTF_8)));
2252+
String device1 = "root.test.d1";
2253+
session.insertRecord(device1, 0, measurements, types, values);
2254+
2255+
try (SessionDataSet dataSet =
2256+
session.executeQueryStatement("SHOW TIMESERIES root.test.d1.**")) {
2257+
int compressionIndex = dataSet.getColumnNames().indexOf("Compression");
2258+
while (dataSet.hasNext()) {
2259+
RowRecord rec = dataSet.next();
2260+
Field compressionField = rec.getFields().get(compressionIndex);
2261+
assertEquals("LZ4", compressionField.getStringValue());
2262+
}
2263+
}
2264+
2265+
for (TSDataType type : types) {
2266+
String configName = null;
2267+
switch (type) {
2268+
case INT32:
2269+
case INT64:
2270+
case FLOAT:
2271+
case DOUBLE:
2272+
case TEXT:
2273+
case BOOLEAN:
2274+
configName = type.name().toLowerCase();
2275+
break;
2276+
case STRING:
2277+
case BLOB:
2278+
configName = "text";
2279+
break;
2280+
case DATE:
2281+
configName = "int32";
2282+
break;
2283+
case TIMESTAMP:
2284+
configName = "int64";
2285+
break;
2286+
}
2287+
session.executeNonQueryStatement(
2288+
String.format("SET CONFIGURATION '%s_compressor'='GZIP'", configName));
2289+
}
2290+
2291+
String device2 = "root.test.d1";
2292+
session.insertRecord(device2, 0, measurements, types, values);
2293+
2294+
try (SessionDataSet dataSet =
2295+
session.executeQueryStatement("SHOW TIMESERIES root.test.d2.**")) {
2296+
int compressionIndex = dataSet.getColumnNames().indexOf("Compression");
2297+
while (dataSet.hasNext()) {
2298+
RowRecord rec = dataSet.next();
2299+
Field compressionField = rec.getFields().get(compressionIndex);
2300+
assertEquals("GZIP", compressionField.getStringValue());
2301+
}
2302+
}
2303+
}
2304+
}
22192305
}

‎iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,36 @@ default_double_encoding=GORILLA
977977
# Datatype: TSEncoding
978978
default_text_encoding=PLAIN
979979

980+
# BOOLEAN compression when creating schema automatically is enabled
981+
# effectiveMode: hot_reload
982+
# Datatype: TSEncoding
983+
boolean_compressor=lz4
984+
985+
# INT32/DATE compression when creating schema automatically is enabled
986+
# effectiveMode: hot_reload
987+
# Datatype: TSEncoding
988+
int32_compressor=lz4
989+
990+
# INT64/TIMESTAMP compression when creating schema automatically is enabled
991+
# effectiveMode: hot_reload
992+
# Datatype: TSEncoding
993+
int64_compressor=lz4
994+
995+
# FLOAT compression when creating schema automatically is enabled
996+
# effectiveMode: hot_reload
997+
# Datatype: TSEncoding
998+
float_compressor=lz4
999+
1000+
# DOUBLE compression when creating schema automatically is enabled
1001+
# effectiveMode: hot_reload
1002+
# Datatype: TSEncoding
1003+
double_compressor=lz4
1004+
1005+
# TEXT/BINARY/BLOB compression when creating schema automatically is enabled
1006+
# effectiveMode: hot_reload
1007+
# Datatype: TSEncoding
1008+
text_compressor=lz4
1009+
9801010
####################
9811011
### Query Configurations
9821012
####################

0 commit comments

Comments
 (0)