Skip to content

Commit efe71c2

Browse files
authored
GH-3404: fix CI TestByteBitPacking512VectorLE.unpackValuesUsingVector randomly oom (#3405)
1 parent 6c7cefd commit efe71c2

File tree

1 file changed

+32
-30
lines changed

1 file changed

+32
-30
lines changed

parquet-plugins/parquet-encoding-vector/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking512VectorLE.java

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@
2222

2323
import java.math.BigDecimal;
2424
import java.nio.ByteBuffer;
25-
import java.util.ArrayList;
26-
import java.util.List;
25+
import java.util.Arrays;
26+
import java.util.stream.IntStream;
27+
import java.util.stream.Stream;
2728
import org.junit.Assume;
2829
import org.junit.Test;
2930
import org.slf4j.Logger;
@@ -41,30 +42,31 @@ public void unpackValuesUsingVector() {
4142
}
4243

4344
private void unpackValuesUsingVectorBitWidth(int bitWidth) {
44-
List<int[]> intInputs = getRangeData(bitWidth);
45-
46-
for (int[] intInput : intInputs) {
47-
int pack8Count = intInput.length / 8;
48-
int byteOutputSize = pack8Count * bitWidth;
49-
byte[] byteOutput = new byte[byteOutputSize];
50-
int[] output1 = new int[intInput.length];
51-
int[] output2 = new int[intInput.length];
52-
int[] output3 = new int[intInput.length];
53-
54-
BytePacker bytePacker = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
55-
for (int i = 0; i < pack8Count; i++) {
56-
bytePacker.pack8Values(intInput, 8 * i, byteOutput, bitWidth * i);
57-
}
45+
try (Stream<int[]> intInputs = getRangeData(bitWidth)) {
46+
intInputs.forEach(intInput -> {
47+
int pack8Count = intInput.length / 8;
48+
int byteOutputSize = pack8Count * bitWidth;
49+
byte[] byteOutput = new byte[byteOutputSize];
50+
int[] output = new int[intInput.length];
51+
52+
BytePacker bytePacker = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
53+
for (int i = 0; i < pack8Count; i++) {
54+
bytePacker.pack8Values(intInput, 8 * i, byteOutput, bitWidth * i);
55+
}
5856

59-
unpack8Values(bitWidth, byteOutput, output1);
60-
unpackValuesUsingVectorArray(bitWidth, byteOutput, output2);
57+
unpack8Values(bitWidth, byteOutput, output);
58+
assertArrayEquals(intInput, output);
59+
Arrays.fill(output, 0);
6160

62-
ByteBuffer byteBuffer = ByteBuffer.wrap(byteOutput);
63-
unpackValuesUsingVectorByteBuffer(bitWidth, byteBuffer, output3);
61+
unpackValuesUsingVectorArray(bitWidth, byteOutput, output);
62+
assertArrayEquals(intInput, output);
63+
Arrays.fill(output, 0);
6464

65-
assertArrayEquals(intInput, output1);
66-
assertArrayEquals(intInput, output2);
67-
assertArrayEquals(intInput, output3);
65+
ByteBuffer byteBuffer = ByteBuffer.wrap(byteOutput);
66+
unpackValuesUsingVectorByteBuffer(bitWidth, byteBuffer, output);
67+
assertArrayEquals(intInput, output);
68+
Arrays.fill(output, 0);
69+
});
6870
}
6971
}
7072

@@ -119,8 +121,7 @@ public void unpackValuesUsingVectorByteBuffer(int bitWidth, ByteBuffer input, in
119121
}
120122
}
121123

122-
private List<int[]> getRangeData(int bitWidth) {
123-
List<int[]> result = new ArrayList<>();
124+
private Stream<int[]> getRangeData(int bitWidth) {
124125
int itemMax = 268435456;
125126

126127
long maxValue = getMaxValue(bitWidth);
@@ -131,9 +132,11 @@ private List<int[]> getRangeData(int bitWidth) {
131132
++itemCount;
132133
}
133134

134-
for (int i = 0; i < itemCount; i++) {
135+
final int finalItemCount = itemCount;
136+
137+
return IntStream.range(0, finalItemCount).mapToObj(i -> {
135138
int len;
136-
if ((i == itemCount - 1) && mode != 0) {
139+
if ((i == finalItemCount - 1) && mode != 0) {
137140
len = mode;
138141
} else {
139142
len = itemMax;
@@ -162,9 +165,8 @@ private List<int[]> getRangeData(int bitWidth) {
162165
array[j] = value;
163166
j++;
164167
}
165-
result.add(array);
166-
}
167-
return result;
168+
return array;
169+
});
168170
}
169171

170172
private long getMaxValue(int bitWidth) {

0 commit comments

Comments
 (0)