Skip to content

Commit 99962e1

Browse files
committed
apacheGH-3530: Optimize DICTIONARY encoding/decoding data structures and use ByteBuffer
Encoding improvements: - Replace LinkedOpenHashMap with OpenHashMap + ArrayList for all DictionaryValuesWriter subclasses, eliminating insertion-order overhead and enabling O(1) indexed access for dictionary page serialization and fallback - Make IntList.size() O(1) by tracking totalSize incrementally instead of summing across slab arrays Decoding improvements: - Convert PlainValuesDictionary numeric constructors (INT32, INT64, FLOAT, DOUBLE) from InputStream-based per-byte reads to direct ByteBuffer.getInt/getLong/getFloat/getDouble (JVM intrinsics) Binary hashCode caching: - Cache hashCode() for Binary instances that are not backed by reusable byte arrays, avoiding redundant recomputation during dictionary lookups (hash map probes) JMH benchmarks: - DictionaryEncodingBenchmark: scalar encoding for INT32, INT64, FLOAT, DOUBLE, BINARY, and FIXED_LEN_BYTE_ARRAY with LOW/HIGH cardinality and variable-length string/FLBA dimensions - DictionaryDecodingBenchmark: scalar decoding for all types with matching parameterization - TestDataFactory: shared data generation utility for reproducible benchmark inputs - BenchmarkEncodingUtils: helper to drain DictionaryValuesWriter into encoded dictionary page + data bytes for decoder setup
1 parent 5922bf4 commit 99962e1

11 files changed

Lines changed: 1015 additions & 122 deletions

File tree

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.parquet.benchmarks;
20+
21+
import java.io.IOException;
22+
import org.apache.parquet.column.page.DictionaryPage;
23+
import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter;
24+
25+
/**
26+
* Shared helpers for encode/decode micro-benchmarks.
27+
*/
28+
final class BenchmarkEncodingUtils {
29+
30+
private BenchmarkEncodingUtils() {}
31+
32+
/**
33+
* Container for the two artefacts produced by a dictionary-encoded page:
34+
* the encoded dictionary indices ({@link #dictData}) and the dictionary
35+
* page itself ({@link #dictPage}). The dictionary page may be {@code null}
36+
* if the writer fell back to plain encoding (for example, when the
37+
* dictionary exceeded its configured maximum size).
38+
*/
39+
static final class EncodedDictionary {
40+
final byte[] dictData;
41+
final DictionaryPage dictPage;
42+
43+
EncodedDictionary(byte[] dictData, DictionaryPage dictPage) {
44+
this.dictData = dictData;
45+
this.dictPage = dictPage;
46+
}
47+
48+
boolean fellBackToPlain() {
49+
return dictPage == null;
50+
}
51+
}
52+
53+
/**
54+
* Drains a {@link DictionaryValuesWriter} into an {@link EncodedDictionary}.
55+
*
56+
* <p>The writer's data bytes (the RLE-encoded indices) and the dictionary
57+
* page are returned separately so both pieces can be measured or fed to a
58+
* decoder symmetrically. The dictionary page buffer is copied so it remains
59+
* valid after the writer's allocator is released.
60+
*
61+
* <p>The writer is closed via {@code toDictPageAndClose()}; callers must not
62+
* call {@link DictionaryValuesWriter#close()} again afterwards.
63+
*/
64+
static EncodedDictionary drainDictionary(DictionaryValuesWriter writer) throws IOException {
65+
byte[] dictData = writer.getBytes().toByteArray();
66+
DictionaryPage rawPage = writer.toDictPageAndClose();
67+
DictionaryPage dictPage = rawPage == null ? null : rawPage.copy();
68+
return new EncodedDictionary(dictData, dictPage);
69+
}
70+
}

0 commit comments

Comments
 (0)