Skip to content

Commit 51fe585

Browse files
luoluoyuyuluoluoyuyu
authored
perf: Optimize the mark bitmap operation of AlignedTVList InsertTablet (#16199)
* perf: Optimize the mark bitmap operation of AlignedTVList InsertTablet * add UT * update * fix * fix * update tsfile version * fix
1 parent 6a037a2 commit 51fe585

3 files changed

Lines changed: 118 additions & 50 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -650,10 +650,12 @@ private static long getBinarySize(final Binary[] binaries) {
650650
return 0L;
651651
}
652652

653-
return RamUsageEstimator.shallowSizeOf(binaries)
654-
+ Arrays.stream(binaries)
655-
.mapToLong(InsertNodeMemoryEstimator::sizeOfBinary)
656-
.reduce(0L, Long::sum);
653+
long size = 0L;
654+
for (Binary binary : binaries) {
655+
size += InsertNodeMemoryEstimator.sizeOfBinary(binary);
656+
}
657+
658+
return size + RamUsageEstimator.shallowSizeOf(binaries);
657659
}
658660

659661
public static long sizeOfValues(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java‎

Lines changed: 111 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -786,17 +786,9 @@ public synchronized void putAlignedValues(
786786
if (indices != null) {
787787
indices.get(arrayIdx)[elementIdx + i] = rowCount;
788788
}
789-
for (int j = 0; j < values.size(); j++) {
790-
if (value[j] == null
791-
|| bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(idx + i)
792-
|| results != null
793-
&& results[idx + i] != null
794-
&& results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
795-
markNullValue(j, arrayIdx, elementIdx + i);
796-
}
797-
}
798789
rowCount++;
799790
}
791+
markNullBitmapRange(value, bitMaps, results, idx, elementIdx, inputRemaining, arrayIdx);
800792
break;
801793
} else {
802794
// the remaining inputs cannot fit the last array, fill the last array and create a new
@@ -807,23 +799,67 @@ public synchronized void putAlignedValues(
807799
if (indices != null) {
808800
indices.get(arrayIdx)[elementIdx + i] = rowCount;
809801
}
810-
for (int j = 0; j < values.size(); j++) {
811-
if (value[j] == null
812-
|| bitMaps != null && bitMaps[j] != null && bitMaps[j].isMarked(idx + i)
813-
|| results != null
814-
&& results[idx + i] != null
815-
&& results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
816-
markNullValue(j, arrayIdx, elementIdx + i);
817-
}
818-
}
819802
rowCount++;
820803
}
804+
markNullBitmapRange(value, bitMaps, results, idx, elementIdx, internalRemaining, arrayIdx);
821805
idx += internalRemaining;
822806
checkExpansion();
823807
}
824808
}
825809
}
826810

811+
private void markNullBitmapRange(
812+
Object[] values,
813+
BitMap[] bitMaps,
814+
TSStatus[] results,
815+
int idx,
816+
int elementIdx,
817+
int len,
818+
int arrayIndex) {
819+
820+
/* 1. Build result-level bitmap (1 = failure row) */
821+
byte[] resultBitMap =
822+
(results != null) ? buildResultBitMapBytes(results, idx, elementIdx, len) : null;
823+
824+
for (int j = 0; j < values.length; j++) {
825+
/* Fast-path: column is entirely null */
826+
if (values[j] == null) {
827+
getBitMap(j, arrayIndex).markRange(elementIdx, len);
828+
continue;
829+
}
830+
831+
/* 2.mask the column bitmap */
832+
if (bitMaps != null && bitMaps[j] != null) {
833+
getBitMap(j, arrayIndex).merge(bitMaps[j], idx, elementIdx, len);
834+
}
835+
836+
/* 3. Overlay result bitmap (failure rows) */
837+
if (resultBitMap != null) {
838+
markNullValue(j, arrayIndex, elementIdx, resultBitMap);
839+
}
840+
}
841+
}
842+
843+
public static byte[] buildResultBitMapBytes(
844+
TSStatus[] results, int idx, int elementIdx, int length) {
845+
int start = elementIdx & 7;
846+
int totalBits = start + length;
847+
int size = (totalBits + 7) >> 3;
848+
BitMap bitmap = new BitMap(size, new byte[size]);
849+
850+
if (results == null) {
851+
return bitmap.getByteArray();
852+
}
853+
854+
for (int i = 0; i < length; i++) {
855+
if (results[idx + i] != null
856+
&& results[idx + i].code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
857+
bitmap.mark(start + i);
858+
}
859+
}
860+
return bitmap.getByteArray();
861+
}
862+
827863
private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex, int remaining) {
828864
for (int i = 0; i < values.size(); i++) {
829865
if (value[i] == null) {
@@ -871,7 +907,7 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex
871907
}
872908
}
873909

874-
private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) {
910+
private BitMap getBitMap(int columnIndex, int arrayIndex) {
875911
// init BitMaps if doesn't have
876912
if (bitMaps == null) {
877913
List<List<BitMap>> localBitMaps = new ArrayList<>(dataTypes.size());
@@ -885,18 +921,31 @@ private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) {
885921
if (bitMaps.get(columnIndex) == null) {
886922
List<BitMap> columnBitMaps = new ArrayList<>(values.get(columnIndex).size());
887923
for (int i = 0; i < values.get(columnIndex).size(); i++) {
888-
columnBitMaps.add(new BitMap(ARRAY_SIZE));
924+
columnBitMaps.add(new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE]));
889925
}
890926
bitMaps.set(columnIndex, columnBitMaps);
891927
}
892928

893929
// if the bitmap in arrayIndex is null, init the bitmap
894930
if (bitMaps.get(columnIndex).get(arrayIndex) == null) {
895-
bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE));
931+
bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE]));
896932
}
897933

934+
return bitMaps.get(columnIndex).get(arrayIndex);
935+
}
936+
937+
private void markNullValue(
938+
int columnIndex, int arrayIndex, int elementIndex, byte[] resultBitMap) {
939+
byte[] bitMap = getBitMap(columnIndex, arrayIndex).getByteArray();
940+
int start = elementIndex >>> 3;
941+
for (byte b : resultBitMap) {
942+
bitMap[start++] |= b;
943+
}
944+
}
945+
946+
private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) {
898947
// mark the null value in the current bitmap
899-
bitMaps.get(columnIndex).get(arrayIndex).mark(elementIndex);
948+
getBitMap(columnIndex, arrayIndex).mark(elementIndex);
900949
}
901950

902951
@Override
@@ -1451,33 +1500,50 @@ public BitMap getAllValueColDeletedMap() {
14511500
}
14521501

14531502
byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1];
1454-
for (int row = 0; row < rowCount; row += Byte.SIZE) {
1455-
boolean isFirstColumn = true;
1456-
byte rowBits = 0x00;
1457-
for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
1458-
List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
1459-
byte columnBits;
1460-
if (values.get(columnIndex) == null) {
1461-
columnBits = (byte) 0xFF;
1462-
} else if (columnBitMaps == null || columnBitMaps.get(row / ARRAY_SIZE) == null) {
1463-
// row exists when any column value exists
1464-
rowBits = 0x00;
1465-
break;
1466-
} else {
1467-
columnBits =
1468-
columnBitMaps.get(row / ARRAY_SIZE).getByteArray()[(row % ARRAY_SIZE) / Byte.SIZE];
1503+
int bitsMapSize =
1504+
rowCount % ARRAY_SIZE == 0 ? rowCount / ARRAY_SIZE : rowCount / ARRAY_SIZE + 1;
1505+
boolean[] allNotNullArray = new boolean[bitsMapSize];
1506+
Arrays.fill(rowBitsArr, (byte) 0xFF);
1507+
for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
1508+
List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
1509+
if (columnBitMaps == null) {
1510+
Arrays.fill(rowBitsArr, (byte) 0x00);
1511+
break;
1512+
} else if (values.get(columnIndex) != null) {
1513+
int row = 0;
1514+
boolean isEnd = true;
1515+
for (int i = 0; i < bitsMapSize; i++) {
1516+
if (allNotNullArray[i]) {
1517+
row += ARRAY_SIZE;
1518+
continue;
1519+
}
1520+
1521+
BitMap bitMap = columnBitMaps.get(i);
1522+
int index = row / Byte.SIZE;
1523+
int size = ((Math.min((rowCount - row), ARRAY_SIZE)) + 7) >>> 3;
1524+
row += ARRAY_SIZE;
1525+
1526+
if (bitMap == null) {
1527+
Arrays.fill(rowBitsArr, index, index + size, (byte) 0x00);
1528+
allNotNullArray[i] = true;
1529+
continue;
1530+
}
1531+
1532+
byte bits = (byte) 0X00;
1533+
for (int j = 0; j < size; j++) {
1534+
rowBitsArr[index] &= bitMap.getByteArray()[j];
1535+
bits |= rowBitsArr[index++];
1536+
isEnd = false;
1537+
}
1538+
1539+
allNotNullArray[i] = bits == (byte) 0;
14691540
}
1470-
// set row to null when all column values are null
1471-
if (isFirstColumn) {
1472-
rowBits = columnBits;
1473-
isFirstColumn = false;
1474-
} else {
1475-
rowBits &= columnBits;
1541+
1542+
if (isEnd) {
1543+
break;
14761544
}
14771545
}
1478-
rowBitsArr[row / Byte.SIZE] = rowBits;
14791546
}
1480-
14811547
return new BitMap(rowCount, rowBitsArr);
14821548
}
14831549

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@
176176
<thrift.version>0.14.1</thrift.version>
177177
<xz.version>1.9</xz.version>
178178
<zstd-jni.version>1.5.6-3</zstd-jni.version>
179-
<tsfile.version>2.2.0-250813-SNAPSHOT</tsfile.version>
179+
<tsfile.version>2.2.0-250825-SNAPSHOT</tsfile.version>
180180
</properties>
181181
<!--
182182
if we claim dependencies in dependencyManagement, then we do not claim

0 commit comments

Comments
 (0)