Skip to content

Commit dc23dbc

Browse files
committed
apacheGH-3522: Use pack32Values fast path in RLE hybrid encoder
Buffer four 8-value groups (32 values) before packing, then use the packer's pack32Values entry point instead of calling pack8Values four separate times. This reduces per-group overhead and enables the packer's optimized 32-value code path. When fewer than 32 values remain at run end, flushBitPackedValues falls back to per-8 packing via pack8Values. All 573 parquet-column tests pass.
1 parent 50e5552 commit dc23dbc

1 file changed

Lines changed: 43 additions & 3 deletions

File tree

parquet-column/src/main/java/org/apache/parquet/column/values/rle/RunLengthBitPackingHybridEncoder.java

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ public class RunLengthBitPackingHybridEncoder implements AutoCloseable {
7474
*/
7575
private final byte[] packBuffer;
7676

77+
/**
78+
* Buffer four 8-value groups so we can use the packer's 32-value fast path.
79+
*/
80+
private final int[] bitPackedValuesBuffer;
81+
7782
/**
7883
* Previous value written, used to detect repeated values
7984
*/
@@ -98,6 +103,8 @@ public class RunLengthBitPackingHybridEncoder implements AutoCloseable {
98103
*/
99104
private int bitPackedGroupCount;
100105

106+
private int numBitPackedValues;
107+
101108
/**
102109
* A "pointer" to a single byte in baos,
103110
* which we use as our bit-packed-header. It's really
@@ -125,7 +132,8 @@ public RunLengthBitPackingHybridEncoder(
125132

126133
this.bitWidth = bitWidth;
127134
this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
128-
this.packBuffer = new byte[bitWidth];
135+
this.packBuffer = new byte[bitWidth * 4];
136+
this.bitPackedValuesBuffer = new int[32];
129137
this.bufferedValues = new int[8];
130138
this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
131139
reset(false);
@@ -139,6 +147,7 @@ private void reset(boolean resetBaos) {
139147
this.numBufferedValues = 0;
140148
this.repeatCount = 0;
141149
this.bitPackedGroupCount = 0;
150+
this.numBitPackedValues = 0;
142151
this.bitPackedRunHeaderPointer = -1;
143152
this.toBytesCalled = false;
144153
}
@@ -196,8 +205,9 @@ private void writeOrAppendBitPackedRun() throws IOException {
196205
bitPackedRunHeaderPointer = baos.getCurrentIndex();
197206
}
198207

199-
packer.pack8Values(bufferedValues, 0, packBuffer, 0);
200-
baos.write(packBuffer);
208+
System.arraycopy(bufferedValues, 0, bitPackedValuesBuffer, numBitPackedValues, 8);
209+
numBitPackedValues += 8;
210+
flushBitPackedValuesIfFull();
201211

202212
// empty the buffer, they've all been written
203213
numBufferedValues = 0;
@@ -209,6 +219,34 @@ private void writeOrAppendBitPackedRun() throws IOException {
209219
++bitPackedGroupCount;
210220
}
211221

222+
private void flushBitPackedValuesIfFull() {
223+
if (numBitPackedValues == bitPackedValuesBuffer.length) {
224+
packer.pack32Values(bitPackedValuesBuffer, 0, packBuffer, 0);
225+
baos.write(packBuffer, 0, bitWidth * 4);
226+
numBitPackedValues = 0;
227+
}
228+
}
229+
230+
private void flushBitPackedValues() {
231+
if (numBitPackedValues == 0) {
232+
return;
233+
}
234+
235+
if (numBitPackedValues == bitPackedValuesBuffer.length) {
236+
packer.pack32Values(bitPackedValuesBuffer, 0, packBuffer, 0);
237+
baos.write(packBuffer, 0, bitWidth * 4);
238+
} else {
239+
int outPos = 0;
240+
for (int inPos = 0; inPos < numBitPackedValues; inPos += 8) {
241+
packer.pack8Values(bitPackedValuesBuffer, inPos, packBuffer, outPos);
242+
outPos += bitWidth;
243+
}
244+
baos.write(packBuffer, 0, outPos);
245+
}
246+
247+
numBitPackedValues = 0;
248+
}
249+
212250
/**
213251
* If we are currently writing a bit-packed-run, update the
214252
* bit-packed-header and consider this run to be over
@@ -221,6 +259,8 @@ private void endPreviousBitPackedRun() {
221259
return;
222260
}
223261

262+
flushBitPackedValues();
263+
224264
// create bit-packed-header, which needs to fit in 1 byte
225265
byte bitPackHeader = (byte) ((bitPackedGroupCount << 1) | 1);
226266

0 commit comments

Comments
 (0)