From 6df13fdb357c60f7215d0662c30685226daf67c7 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 18 Aug 2025 17:03:01 +0800 Subject: [PATCH 1/7] perf: Optimize the mark bitmap operation of AlignedTVList InsertTablet --- .../db/utils/datastructure/AlignedTVList.java | 160 +++++++++++++++--- 1 file changed, 140 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 80649eb182d15..8c919a71a259e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -89,6 +89,39 @@ public abstract class AlignedTVList extends TVList { private final AlignedTVList outer = this; + private static final byte[] HEAD_MASK = { + (byte) 0b1111_1111, + (byte) 0b0111_1111, + (byte) 0b0011_1111, + (byte) 0b0001_1111, + (byte) 0b0000_1111, + (byte) 0b0000_0111, + (byte) 0b0000_0011, + (byte) 0b0000_0001 + }; + + private static final byte[] TAIL_MASK = { + (byte) 0b0000_0000, + (byte) 0b1000_0000, + (byte) 0b1100_0000, + (byte) 0b1110_0000, + (byte) 0b1111_0000, + (byte) 0b1111_1000, + (byte) 0b1111_1100, + (byte) 0b1111_1110 + }; + + private static final byte[] BIT_MASK = { + 0b0000_0001, + 0b0000_0010, + 0b0000_0100, + 0b0000_1000, + 0b0001_0000, + 0b0010_0000, + 0b0100_0000, + (byte) 0b1000_0000 + }; + AlignedTVList(List types) { super(); dataTypes = types; @@ -786,17 +819,9 @@ public synchronized void putAlignedValues( if (indices != null) { indices.get(arrayIdx)[elementIdx + i] = rowCount; } - for (int j = 0; j < values.size(); j++) { - if (value[j] == null - || bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(idx + i) - || results != null - && results[idx + i] != null - && results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - markNullValue(j, arrayIdx, elementIdx + i); - } - } rowCount++; } + markNullBitmapRange(value, bitMaps, results, idx, elementIdx, inputRemaining, arrayIdx); break; } else { // the remaining inputs cannot fit the last array, fill the last array and create a new @@ -807,23 +832,74 @@ public synchronized void putAlignedValues( if (indices != null) { indices.get(arrayIdx)[elementIdx + i] = rowCount; } - for (int j = 0; j < values.size(); j++) { - if (value[j] == null - || bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(idx + i) - || results != null - && results[idx + i] != null - && results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - markNullValue(j, arrayIdx, elementIdx + i); - } - } rowCount++; } + markNullBitmapRange(value, bitMaps, results, idx, elementIdx, internalRemaining, arrayIdx); idx += internalRemaining; checkExpansion(); } } } + private void markNullBitmapRange( + Object[] values, + BitMap[] bitMaps, + TSStatus[] results, + int idx, + int elementIdx, + int len, + int arrayIndex) { + + /* 1. Build result-level bitmap (1 = failure row) */ + byte[] resultBitMap = + (results != null) ? buildResultBitMapBytes(results, idx, elementIdx, len) : null; + + for (int j = 0; j < values.length; j++) { + /* Fast-path: column is entirely null */ + if (values[j] == null) { + markNullValue(j, arrayIndex, elementIdx, len); + continue; + } + + /* 2.mask the column bitmap */ + if (bitMaps != null && bitMaps[j] != null) { + for (int i = 0; i < elementIdx; i++) { + if (bitMaps[j].isMarked(idx + i)) { + markNullValue(j, arrayIndex, elementIdx + i); + } + } + } + + /* 3. Overlay result bitmap (failure rows) */ + if (resultBitMap != null) { + markNullValue(j, arrayIndex, elementIdx, resultBitMap); + } + } + } + + public static byte[] buildResultBitMapBytes( + TSStatus[] results, int idx, int elementIdx, int length) { + int start = elementIdx & 7; + int totalBits = start + length; + byte[] bitmap = new byte[(totalBits + 7) >> 3]; + + if (results == null) { + return bitmap; + } + + for (int i = 0; i < length; i++) { + int globalBit = start + i; + int bytePos = globalBit >>> 3; + int bitPos = globalBit & 7; + + if (results[idx + i] != null + && results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + bitmap[bytePos] |= BIT_MASK[bitPos]; + } + } + return bitmap; + } + private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex, int remaining) { for (int i = 0; i < values.size(); i++) { if (value[i] == null) { @@ -871,7 +947,7 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex } } - private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { + private BitMap getBitMap(int columnIndex, int arrayIndex) { // init BitMaps if doesn't have if (bitMaps == null) { List> localBitMaps = new ArrayList<>(dataTypes.size()); @@ -895,8 +971,52 @@ private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE)); } + return bitMaps.get(columnIndex).get(arrayIndex); + } + + private void markRange(byte[] bitMap, int bitStart, int bitLen) { + if (bitLen <= 0) return; + + int bitEnd = bitStart + bitLen; + int byte0 = bitStart >>> 3; + int byte1 = (bitEnd - 1) >>> 3; + + if (byte0 == byte1) { + int mask = HEAD_MASK[bitStart & 7] & TAIL_MASK[bitEnd & 7]; + bitMap[byte0] |= (byte) mask; + return; + } + if ((bitStart & 7) != 0) { + bitMap[byte0] |= HEAD_MASK[bitStart & 7]; + byte0++; + } + + while (byte0 < byte1) { + bitMap[byte0++] = (byte) 0xFF; + } + + int tailBits = bitEnd & 7; + if (tailBits != 0) { + bitMap[byte1] |= TAIL_MASK[tailBits]; + } + } + + private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { // mark the null value in the current bitmap - bitMaps.get(columnIndex).get(arrayIndex).mark(elementIndex); + getBitMap(columnIndex, arrayIndex).mark(elementIndex); + } + + private void markNullValue(int columnIndex, int arrayIndex, int elementIndex, int length) { + markRange(getBitMap(columnIndex, arrayIndex).getByteArray(), elementIndex, length); + } + + private void markNullValue( + int columnIndex, int arrayIndex, int elementIndex, byte[] resultBitMap) { + byte[] bitMap = getBitMap(columnIndex, arrayIndex).getByteArray(); + int start = elementIndex >>> 3; + for (byte b : resultBitMap) { + bitMap[start++] |= b; + } } @Override From 04fc347cc98584c4009924ce3ded85f1554c63ec Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 Aug 2025 14:48:45 +0800 Subject: [PATCH 2/7] add UT --- .../db/utils/datastructure/AlignedTVList.java | 70 ++++++--------- .../dataregion/memtable/MarkRangeTest.java | 85 +++++++++++++++++++ 2 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 8c919a71a259e..9de8588287204 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -89,37 +89,26 @@ public abstract class AlignedTVList extends TVList { private final AlignedTVList outer = this; - private static final byte[] HEAD_MASK = { - (byte) 0b1111_1111, - (byte) 0b0111_1111, - (byte) 0b0011_1111, - (byte) 0b0001_1111, - (byte) 0b0000_1111, - (byte) 0b0000_0111, + private static final byte[] LOW_N_BITS = { + (byte) 0b0000_0001, (byte) 0b0000_0011, - (byte) 0b0000_0001 + (byte) 0b0000_0111, + (byte) 0b0000_1111, + (byte) 0b0001_1111, + (byte) 0b0011_1111, + (byte) 0b0111_1111, + (byte) 0b1111_1111, }; - private static final byte[] TAIL_MASK = { - (byte) 0b0000_0000, - (byte) 0b1000_0000, - (byte) 0b1100_0000, - (byte) 0b1110_0000, - (byte) 0b1111_0000, - (byte) 0b1111_1000, + private static final byte[] HIGH_N_BITS = { + (byte) 0b1111_1111, + (byte) 0b1111_1110, (byte) 0b1111_1100, - (byte) 0b1111_1110 - }; - - private static final byte[] BIT_MASK = { - 0b0000_0001, - 0b0000_0010, - 0b0000_0100, - 0b0000_1000, - 0b0001_0000, - 0b0010_0000, - 0b0100_0000, - (byte) 0b1000_0000 + (byte) 0b1111_1000, + (byte) 0b1111_0000, + (byte) 0b1110_0000, + (byte) 0b1100_0000, + (byte) 0b1000_0000, }; AlignedTVList(List types) { @@ -881,23 +870,19 @@ public static byte[] buildResultBitMapBytes( TSStatus[] results, int idx, int elementIdx, int length) { int start = elementIdx & 7; int totalBits = start + length; - byte[] bitmap = new byte[(totalBits + 7) >> 3]; + BitMap bitmap = new BitMap((totalBits + 7) >> 3); if (results == null) { - return bitmap; + return bitmap.getByteArray(); } for (int i = 0; i < length; i++) { - int globalBit = start + i; - int bytePos = globalBit >>> 3; - int bitPos = globalBit & 7; - if (results[idx + i] != null && results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - bitmap[bytePos] |= BIT_MASK[bitPos]; + bitmap.mark(start + i); } } - return bitmap; + return bitmap.getByteArray(); } private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex, int remaining) { @@ -974,7 +959,7 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) { return bitMaps.get(columnIndex).get(arrayIndex); } - private void markRange(byte[] bitMap, int bitStart, int bitLen) { + public static void markRange(byte[] bitMap, int bitStart, int bitLen) { if (bitLen <= 0) return; int bitEnd = bitStart + bitLen; @@ -982,23 +967,18 @@ private void markRange(byte[] bitMap, int bitStart, int bitLen) { int byte1 = (bitEnd - 1) >>> 3; if (byte0 == byte1) { - int mask = HEAD_MASK[bitStart & 7] & TAIL_MASK[bitEnd & 7]; + int mask = ((1 << (bitStart & 7)) - 1) ^ LOW_N_BITS[(bitEnd - 1) & 7]; bitMap[byte0] |= (byte) mask; return; } - if ((bitStart & 7) != 0) { - bitMap[byte0] |= HEAD_MASK[bitStart & 7]; - byte0++; - } + + bitMap[byte0++] |= HIGH_N_BITS[bitStart & 7]; while (byte0 < byte1) { bitMap[byte0++] = (byte) 0xFF; } - int tailBits = bitEnd & 7; - if (tailBits != 0) { - bitMap[byte1] |= TAIL_MASK[tailBits]; - } + bitMap[byte1] |= LOW_N_BITS[(bitEnd - 1) & 7]; } private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java new file mode 100644 index 0000000000000..56fb4c13afbaf --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.memtable; + +import org.apache.iotdb.db.utils.datastructure.AlignedTVList; + +import org.apache.tsfile.utils.BitMap; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class MarkRangeTest { + + @Test + public void emptyRange() { + byte[] map = new byte[1]; + AlignedTVList.markRange(map, 0, 0); + assertEquals((byte) 0x00, map[0]); + } + + @Test + public void singleByteAllBits() { + for (int i = 0; i < 8; i++) { + for (int j = 0; j <= 8 - i; j++) { + doTest(8, i, j); + } + } + } + + @Test + public void twoBytesHeadTail() { + for (int i = 0; i < 64; i++) { + for (int j = 0; j <= 64 - i; j += 8) { + doTest(64, i, j); + } + } + } + + @Test + public void twoBytesPartialHead() { + for (int i = 0; i < 64; i += 8) { + for (int j = 0; j <= 64 - i; j += 8) { + doTest(64, i, j); + } + } + } + + @Test + public void twoBytesPartialTail() { + int size = 64; + for (int i = 0; i < size; i += 8) { + for (int j = 1; j <= size - i; j++) { + doTest(size, i, j); + } + } + } + + private void doTest(int size, int start, int length) { + BitMap map = new BitMap(size); + BitMap bitMap = new BitMap(size); + AlignedTVList.markRange(map.getByteArray(), start, length); + for (int i = start; i < start + length; i++) { + bitMap.mark(i); + } + assertArrayEquals(map.getByteArray(), bitMap.getByteArray()); + } +} From d74692a291ff1f928caa4c18f6869bae76241739 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 Aug 2025 15:42:23 +0800 Subject: [PATCH 3/7] update --- .../apache/iotdb/db/utils/datastructure/AlignedTVList.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 9de8588287204..b4e1ee2ce3d02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -870,7 +870,8 @@ public static byte[] buildResultBitMapBytes( TSStatus[] results, int idx, int elementIdx, int length) { int start = elementIdx & 7; int totalBits = start + length; - BitMap bitmap = new BitMap((totalBits + 7) >> 3); + int size = (totalBits + 7) >> 3; + BitMap bitmap = new BitMap(size, new byte[size]); if (results == null) { return bitmap.getByteArray(); @@ -946,14 +947,14 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) { if (bitMaps.get(columnIndex) == null) { List columnBitMaps = new ArrayList<>(values.get(columnIndex).size()); for (int i = 0; i < values.get(columnIndex).size(); i++) { - columnBitMaps.add(new BitMap(ARRAY_SIZE)); + columnBitMaps.add(new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE])); } bitMaps.set(columnIndex, columnBitMaps); } // if the bitmap in arrayIndex is null, init the bitmap if (bitMaps.get(columnIndex).get(arrayIndex) == null) { - bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE)); + bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE])); } return bitMaps.get(columnIndex).get(arrayIndex); From a94f045b2caa836e32948251b3b44f03913fb51a Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 Aug 2025 17:30:34 +0800 Subject: [PATCH 4/7] fix --- .../org/apache/iotdb/db/utils/datastructure/AlignedTVList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index b4e1ee2ce3d02..51b15ae315471 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -852,7 +852,7 @@ private void markNullBitmapRange( /* 2.mask the column bitmap */ if (bitMaps != null && bitMaps[j] != null) { - for (int i = 0; i < elementIdx; i++) { + for (int i = 0; i < len; i++) { // 注意：循环到 len if (bitMaps[j].isMarked(idx + i)) { markNullValue(j, arrayIndex, elementIdx + i); } From 475f62a3805d7ffc316fdee1b08408f66a017200 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 19 Aug 2025 17:37:10 +0800 Subject: [PATCH 5/7] fix --- .../org/apache/iotdb/db/utils/datastructure/AlignedTVList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 51b15ae315471..b1dcd8baab622 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -852,7 +852,7 @@ private void markNullBitmapRange( /* 2.mask the column bitmap */ if (bitMaps != null && bitMaps[j] != null) { - for (int i = 0; i < len; i++) { // 注意：循环到 len + for (int i = 0; i < len; i++) { if (bitMaps[j].isMarked(idx + i)) { markNullValue(j, arrayIndex, elementIdx + i); } From 8f04621b856516ed04c9b6b269c321d48893c239 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 25 Aug 2025 17:56:31 +0800 Subject: [PATCH 6/7] update tsfile version --- .../memory/InsertNodeMemoryEstimator.java | 10 +- .../db/utils/datastructure/AlignedTVList.java | 128 +++++++----------- .../dataregion/memtable/MarkRangeTest.java | 85 ------------ pom.xml | 2 +- 4 files changed, 53 insertions(+), 172 deletions(-) delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index ed34f0de31039..476ec83b7877d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -650,10 +650,12 @@ private static long getBinarySize(final Binary[] binaries) { return 0L; } - return RamUsageEstimator.shallowSizeOf(binaries) - + Arrays.stream(binaries) - .mapToLong(InsertNodeMemoryEstimator::sizeOfBinary) - .reduce(0L, Long::sum); + long size = 0L; + for (Binary binary : binaries) { + size += InsertNodeMemoryEstimator.sizeOfBinary(binary); + } + + return size + RamUsageEstimator.shallowSizeOf(binaries); } public static long sizeOfValues( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index b1dcd8baab622..65a5fae25e7da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -89,28 +89,6 @@ public abstract class AlignedTVList extends TVList { private final AlignedTVList outer = this; - private static final byte[] LOW_N_BITS = { - (byte) 0b0000_0001, - (byte) 0b0000_0011, - (byte) 0b0000_0111, - (byte) 0b0000_1111, - (byte) 0b0001_1111, - (byte) 0b0011_1111, - (byte) 0b0111_1111, - (byte) 0b1111_1111, - }; - - private static final byte[] HIGH_N_BITS = { - (byte) 0b1111_1111, - (byte) 0b1111_1110, - (byte) 0b1111_1100, - (byte) 0b1111_1000, - (byte) 0b1111_0000, - (byte) 0b1110_0000, - (byte) 0b1100_0000, - (byte) 0b1000_0000, - }; - AlignedTVList(List types) { super(); dataTypes = types; @@ -846,17 +824,13 @@ private void markNullBitmapRange( for (int j = 0; j < values.length; j++) { /* Fast-path: column is entirely null */ if (values[j] == null) { - markNullValue(j, arrayIndex, elementIdx, len); + getBitMap(j, arrayIndex).markRange(elementIdx, len); continue; } /* 2.mask the column bitmap */ if (bitMaps != null && bitMaps[j] != null) { - for (int i = 0; i < len; i++) { - if (bitMaps[j].isMarked(idx + i)) { - markNullValue(j, arrayIndex, elementIdx + i); - } - } + getBitMap(j, arrayIndex).merge(bitMaps[j], idx, elementIdx, len); } /* 3. Overlay result bitmap (failure rows) */ @@ -960,37 +934,6 @@ private BitMap getBitMap(int columnIndex, int arrayIndex) { return bitMaps.get(columnIndex).get(arrayIndex); } - public static void markRange(byte[] bitMap, int bitStart, int bitLen) { - if (bitLen <= 0) return; - - int bitEnd = bitStart + bitLen; - int byte0 = bitStart >>> 3; - int byte1 = (bitEnd - 1) >>> 3; - - if (byte0 == byte1) { - int mask = ((1 << (bitStart & 7)) - 1) ^ LOW_N_BITS[(bitEnd - 1) & 7]; - bitMap[byte0] |= (byte) mask; - return; - } - - bitMap[byte0++] |= HIGH_N_BITS[bitStart & 7]; - - while (byte0 < byte1) { - bitMap[byte0++] = (byte) 0xFF; - } - - bitMap[byte1] |= LOW_N_BITS[(bitEnd - 1) & 7]; - } - - private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { - // mark the null value in the current bitmap - getBitMap(columnIndex, arrayIndex).mark(elementIndex); - } - - private void markNullValue(int columnIndex, int arrayIndex, int elementIndex, int length) { - markRange(getBitMap(columnIndex, arrayIndex).getByteArray(), elementIndex, length); - } - private void markNullValue( int columnIndex, int arrayIndex, int elementIndex, byte[] resultBitMap) { byte[] bitMap = getBitMap(columnIndex, arrayIndex).getByteArray(); @@ -1000,6 +943,11 @@ private void markNullValue( } } + private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) { + // mark the null value in the current bitmap + getBitMap(columnIndex, arrayIndex).mark(elementIndex); + } + @Override public TSDataType getDataType() { return TSDataType.VECTOR; @@ -1552,33 +1500,49 @@ public BitMap getAllValueColDeletedMap() { } byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1]; - for (int row = 0; row < rowCount; row += Byte.SIZE) { - boolean isFirstColumn = true; - byte rowBits = 0x00; - for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { - List columnBitMaps = bitMaps.get(columnIndex); - byte columnBits; - if (values.get(columnIndex) == null) { - columnBits = (byte) 0xFF; - } else if (columnBitMaps == null || columnBitMaps.get(row / ARRAY_SIZE) == null) { - // row exists when any column value exists - rowBits = 0x00; - break; - } else { - columnBits = - columnBitMaps.get(row / ARRAY_SIZE).getByteArray()[(row % ARRAY_SIZE) / Byte.SIZE]; + int bitsMapSize = rowCount / ARRAY_SIZE + 1; + boolean[] allNullArray = new boolean[bitsMapSize]; + Arrays.fill(rowBitsArr, (byte) 0xFF); + for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) { + List columnBitMaps = bitMaps.get(columnIndex); + if (columnBitMaps == null) { + Arrays.fill(rowBitsArr, (byte) 0x00); + break; + } else if (values.get(columnIndex) != null) { + int row = 0; + boolean isEnd = true; + for (int i = 0; i < bitsMapSize; i++) { + if (allNullArray[i]) { + row += ARRAY_SIZE; + continue; + } + + BitMap bitMap = columnBitMaps.get(i); + int index = row / Byte.SIZE; + int size = ((Math.min((rowCount - row), ARRAY_SIZE)) + 7) >>> 3; + row += ARRAY_SIZE; + + if (bitMap == null) { + Arrays.fill(rowBitsArr, index, index + size, (byte) 0x00); + allNullArray[i] = true; + continue; + } + + byte bits = (byte) 0XFF; + for (int j = 0; j < size; j++) { + rowBitsArr[index] &= bitMap.getByteArray()[j]; + bits &= rowBitsArr[index++]; + isEnd = false; + } + + allNullArray[i] = bits == (byte) 0; } - // set row to null when all column values are null - if (isFirstColumn) { - rowBits = columnBits; - isFirstColumn = false; - } else { - rowBits &= columnBits; + + if (isEnd) { + break; } } - rowBitsArr[row / Byte.SIZE] = rowBits; } - return new BitMap(rowCount, rowBitsArr); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java deleted file mode 100644 index 56fb4c13afbaf..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MarkRangeTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.storageengine.dataregion.memtable; - -import org.apache.iotdb.db.utils.datastructure.AlignedTVList; - -import org.apache.tsfile.utils.BitMap; -import org.junit.Test; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -public class MarkRangeTest { - - @Test - public void emptyRange() { - byte[] map = new byte[1]; - AlignedTVList.markRange(map, 0, 0); - assertEquals((byte) 0x00, map[0]); - } - - @Test - public void singleByteAllBits() { - for (int i = 0; i < 8; i++) { - for (int j = 0; j <= 8 - i; j++) { - doTest(8, i, j); - } - } - } - - @Test - public void twoBytesHeadTail() { - for (int i = 0; i < 64; i++) { - for (int j = 0; j <= 64 - i; j += 8) { - doTest(64, i, j); - } - } - } - - @Test - public void twoBytesPartialHead() { - for (int i = 0; i < 64; i += 8) { - for (int j = 0; j <= 64 - i; j += 8) { - doTest(64, i, j); - } - } - } - - @Test - public void twoBytesPartialTail() { - int size = 64; - for (int i = 0; i < size; i += 8) { - for (int j = 1; j <= size - i; j++) { - doTest(size, i, j); - } - } - } - - private void doTest(int size, int start, int length) { - BitMap map = new BitMap(size); - BitMap bitMap = new BitMap(size); - AlignedTVList.markRange(map.getByteArray(), start, length); - for (int i = start; i < start + length; i++) { - bitMap.mark(i); - } - assertArrayEquals(map.getByteArray(), bitMap.getByteArray()); - } -} diff --git a/pom.xml b/pom.xml index da509c4b16dc8..88393900286c4 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ 0.14.1 1.9 1.5.6-3 - 2.2.0-250730-SNAPSHOT + 2.2.0-250825-SNAPSHOT