Skip to content

Commit 0673088

Browse files
committed
fix: account for initial hash table allocation in ArrowBytesMap/ArrowBytesViewMap
ArrowBytesMap and ArrowBytesViewMap initialize their hash table with with_capacity(INITIAL_MAP_CAPACITY) but set map_size to 0. The insert_accounted method only tracks incremental growth beyond the current capacity, so the initial allocation is never counted in size() — understating memory usage. Initialize map_size with map.allocation_size() to capture the pre-allocated memory.
1 parent 2aab559 commit 0673088

File tree

2 files changed

+8
-4
lines changed

2 files changed

+8
-4
lines changed

datafusion/physical-expr-common/src/binary_map.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,12 @@ where
244244
V: Debug + PartialEq + Eq + Clone + Copy + Default,
245245
{
246246
pub fn new(output_type: OutputType) -> Self {
247+
let map = hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY);
248+
let map_size = map.allocation_size();
247249
Self {
248250
output_type,
249-
map: hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY),
250-
map_size: 0,
251+
map,
252+
map_size,
251253
buffer: BufferBuilder::new(INITIAL_BUFFER_CAPACITY),
252254
offsets: vec![O::default()], // first offset is always 0
253255
random_state: RandomState::default(),

datafusion/physical-expr-common/src/binary_view_map.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,10 +155,12 @@ where
155155
V: Debug + PartialEq + Eq + Clone + Copy + Default,
156156
{
157157
pub fn new(output_type: OutputType) -> Self {
158+
let map = hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY);
159+
let map_size = map.allocation_size();
158160
Self {
159161
output_type,
160-
map: hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY),
161-
map_size: 0,
162+
map,
163+
map_size,
162164
views: Vec::new(),
163165
in_progress: Vec::new(),
164166
completed: Vec::new(),

0 commit comments

Comments
 (0)