Skip to content

Commit 47d4c50

Browse files
andygroveclaude
andcommitted
perf: optimize general path for mixed fixed/variable-length columns
Pre-compute variable-length column indices once per batch instead of calling is_variable_length() for every column in every row. In pass 2, only iterate over variable-length columns using the pre-computed indices. Also skip writing placeholder values for variable-length columns in pass 1, since they will be overwritten in pass 2. Performance improvement for primitive types (mixed with strings): - Before: 131ms (0.8X of Spark) - After: ~114ms (0.9X of Spark) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 04c49fb commit 47d4c50

1 file changed

Lines changed: 21 additions & 7 deletions

File tree

native/core/src/execution/columnar_to_row.rs

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,13 +1003,21 @@ impl ColumnarToRowContext {
10031003
.map(|(arr, dt)| TypedArray::from_array(arr, dt))
10041004
.collect::<CometResult<Vec<_>>>()?;
10051005

1006+
// Pre-compute variable-length column indices (once per batch, not per row)
1007+
let var_len_indices: Vec<usize> = typed_arrays
1008+
.iter()
1009+
.enumerate()
1010+
.filter(|(_, arr)| arr.is_variable_length())
1011+
.map(|(idx, _)| idx)
1012+
.collect();
1013+
10061014
// Process each row (general path for variable-length data)
10071015
for row_idx in 0..num_rows {
10081016
let row_start = self.buffer.len();
10091017
self.offsets.push(row_start as i32);
10101018

10111019
// Write fixed-width portion (null bitset + field values)
1012-
self.write_row_typed(&typed_arrays, row_idx)?;
1020+
self.write_row_typed(&typed_arrays, &var_len_indices, row_idx)?;
10131021

10141022
let row_end = self.buffer.len();
10151023
self.lengths.push((row_end - row_start) as i32);
@@ -1243,7 +1251,12 @@ impl ColumnarToRowContext {
12431251

12441252
/// Writes a complete row using pre-downcast TypedArrays.
12451253
/// This avoids type dispatch overhead in the inner loop.
1246-
fn write_row_typed(&mut self, typed_arrays: &[TypedArray], row_idx: usize) -> CometResult<()> {
1254+
fn write_row_typed(
1255+
&mut self,
1256+
typed_arrays: &[TypedArray],
1257+
var_len_indices: &[usize],
1258+
row_idx: usize,
1259+
) -> CometResult<()> {
12471260
let row_start = self.buffer.len();
12481261
let null_bitset_width = self.null_bitset_width;
12491262
let fixed_width_size = self.fixed_width_size;
@@ -1268,17 +1281,18 @@ impl ColumnarToRowContext {
12681281
);
12691282
word |= 1i64 << bit_idx;
12701283
self.buffer[word_offset..word_offset + 8].copy_from_slice(&word.to_le_bytes());
1271-
} else {
1272-
// Write field value at the correct offset
1284+
} else if !typed_arr.is_variable_length() {
1285+
// Write fixed-width field value (skip variable-length, they're handled in pass 2)
12731286
let field_offset = row_start + null_bitset_width + col_idx * 8;
12741287
let value = typed_arr.get_fixed_value(row_idx);
12751288
self.buffer[field_offset..field_offset + 8].copy_from_slice(&value.to_le_bytes());
12761289
}
12771290
}
12781291

1279-
// Second pass: write variable-length data directly to buffer
1280-
for (col_idx, typed_arr) in typed_arrays.iter().enumerate() {
1281-
if typed_arr.is_null(row_idx) || !typed_arr.is_variable_length() {
1292+
// Second pass: write variable-length data (only iterate over var-len columns)
1293+
for &col_idx in var_len_indices {
1294+
let typed_arr = &typed_arrays[col_idx];
1295+
if typed_arr.is_null(row_idx) {
12821296
continue;
12831297
}
12841298

0 commit comments

Comments
 (0)