Skip to content

Commit 00501c2

Browse files
CaideyipirobertMileaNi
authored andcommitted
Pipe: merge batched aligned chunks in scan parser (#18010)
* Pipe: merge batched aligned chunks in scan parser * Test pipe batched aligned chunk memory boundaries * Pipe: fix batched aligned scan parser memory split * Update TsFileInsertionEventParserTest.java * Rename pending aligned chunk consumer
1 parent b31934e commit 00501c2

4 files changed

Lines changed: 662 additions & 120 deletions

File tree

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/SinglePageWholeChunkReader.java

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
2323

24+
import org.apache.tsfile.common.constant.TsFileConstant;
2425
import org.apache.tsfile.compress.IUnCompressor;
2526
import org.apache.tsfile.encoding.decoder.Decoder;
2627
import org.apache.tsfile.encrypt.EncryptParameter;
@@ -169,22 +170,49 @@ static List<Long> toSuffixMaxList(final List<Long> pageEstimatedMemoryUsageInByt
169170
static long estimatePageMemoryUsageInBytesWithBatchData(
170171
final PageHeader timePageHeader,
171172
final Chunk timeChunk,
172-
final List<TSDataType> valueDataTypeList) {
173+
final List<TSDataType> valueDataTypeList)
174+
throws IOException {
173175
return estimatePageMemoryUsageInBytesWithBatchData(
174176
timePageHeader.getUncompressedSize(),
175177
getPageRowCount(timePageHeader, timeChunk),
176178
valueDataTypeList);
177179
}
178180

179-
static int getPageRowCount(final PageHeader pageHeader, final Chunk chunk) {
181+
static int getPageRowCount(final PageHeader pageHeader, final Chunk chunk) throws IOException {
180182
if (isSinglePageChunk(chunk.getHeader())) {
181-
return Objects.isNull(chunk.getChunkStatistic())
182-
? 0
183-
: saturateToInt(chunk.getChunkStatistic().getCount());
183+
if (Objects.nonNull(chunk.getChunkStatistic())) {
184+
return saturateToInt(chunk.getChunkStatistic().getCount());
185+
}
186+
return isTimeChunk(chunk.getHeader()) ? countSinglePageTimeValues(chunk) : 0;
184187
}
185188
return saturateToInt(pageHeader.getNumOfValues());
186189
}
187190

191+
private static int countSinglePageTimeValues(final Chunk chunk) throws IOException {
192+
final ByteBuffer chunkDataBuffer = chunk.getData().duplicate();
193+
final PageHeader pageHeader = deserializePageHeader(chunkDataBuffer, chunk.getHeader());
194+
final ByteBuffer pageData =
195+
deserializePageData(
196+
pageHeader,
197+
chunkDataBuffer,
198+
chunk.getHeader(),
199+
IDecryptor.getDecryptor(chunk.getEncryptParam()));
200+
final Decoder decoder =
201+
Decoder.getDecoderByType(chunk.getHeader().getEncodingType(), TSDataType.INT64);
202+
203+
int rowCount = 0;
204+
while (decoder.hasNext(pageData)) {
205+
decoder.readLong(pageData);
206+
++rowCount;
207+
}
208+
return rowCount;
209+
}
210+
211+
private static boolean isTimeChunk(final ChunkHeader chunkHeader) {
212+
return (chunkHeader.getChunkType() & TsFileConstant.TIME_COLUMN_MASK)
213+
== TsFileConstant.TIME_COLUMN_MASK;
214+
}
215+
188216
private static int saturateToInt(final long value) {
189217
return (int) Math.min(Integer.MAX_VALUE, value);
190218
}

0 commit comments

Comments
 (0)