Skip to content

Commit 95ce922

Browse files
joseph-isaacsgatesnclaude
authored
feat: Iterative Execution (#6824)
Changes the model of execution from internal iterator to external. We go from a recursive execution to an iterative execution with a explicit stack and VTable requesting execution on a child. This can help with stack overflow on high-depth array trees. ## Breaks Changed the execute method. Likely unused externally. Follow ups: - Will remove the usage of recursive calls from all arrays. - Will change the execution method to take a Session so its easier to see internal execution. --------- Signed-off-by: Nick Gates <nick@vortex.dev> Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk> Co-authored-by: Nicholas Gates <nick@nickgates.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent b933a4e commit 95ce922

File tree

57 files changed

+498
-248
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+498
-248
lines changed

‎encodings/alp/public-api.lock‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ pub fn vortex_alp::ALPRDVTable::deserialize(bytes: &[u8], _dtype: &vortex_array:
204204

205205
pub fn vortex_alp::ALPRDVTable::dtype(array: &vortex_alp::ALPRDArray) -> &vortex_array::dtype::DType
206206

207-
pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
207+
pub fn vortex_alp::ALPRDVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
208208

209209
pub fn vortex_alp::ALPRDVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
210210

@@ -308,7 +308,7 @@ pub fn vortex_alp::ALPVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::d
308308

309309
pub fn vortex_alp::ALPVTable::dtype(array: &vortex_alp::ALPArray) -> &vortex_array::dtype::DType
310310

311-
pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
311+
pub fn vortex_alp::ALPVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
312312

313313
pub fn vortex_alp::ALPVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
314314

‎encodings/alp/src/alp/array.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use vortex_array::ArrayRef;
1010
use vortex_array::DeserializeMetadata;
1111
use vortex_array::DynArray;
1212
use vortex_array::ExecutionCtx;
13+
use vortex_array::ExecutionStep;
1314
use vortex_array::IntoArray;
1415
use vortex_array::Precision;
1516
use vortex_array::ProstMetadata;
@@ -234,9 +235,11 @@ impl VTable for ALPVTable {
234235
Ok(())
235236
}
236237

237-
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
238+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
238239
// TODO(joe): take by value
239-
Ok(execute_decompress(array.clone(), ctx)?.into_array())
240+
Ok(ExecutionStep::Done(
241+
execute_decompress(array.clone(), ctx)?.into_array(),
242+
))
240243
}
241244

242245
fn reduce_parent(

‎encodings/alp/src/alp_rd/array.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use vortex_array::ArrayRef;
1111
use vortex_array::DeserializeMetadata;
1212
use vortex_array::DynArray;
1313
use vortex_array::ExecutionCtx;
14+
use vortex_array::ExecutionStep;
1415
use vortex_array::IntoArray;
1516
use vortex_array::Precision;
1617
use vortex_array::ProstMetadata;
@@ -295,7 +296,7 @@ impl VTable for ALPRDVTable {
295296
Ok(())
296297
}
297298

298-
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
299+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
299300
let left_parts = array.left_parts().clone().execute::<PrimitiveArray>(ctx)?;
300301
let right_parts = array.right_parts().clone().execute::<PrimitiveArray>(ctx)?;
301302

@@ -334,7 +335,7 @@ impl VTable for ALPRDVTable {
334335
)
335336
};
336337

337-
Ok(decoded_array.into_array())
338+
Ok(ExecutionStep::Done(decoded_array.into_array()))
338339
}
339340

340341
fn reduce_parent(

‎encodings/bytebool/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ pub fn vortex_bytebool::ByteBoolVTable::deserialize(_bytes: &[u8], _dtype: &vort
108108

109109
pub fn vortex_bytebool::ByteBoolVTable::dtype(array: &vortex_bytebool::ByteBoolArray) -> &vortex_array::dtype::DType
110110

111-
pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
111+
pub fn vortex_bytebool::ByteBoolVTable::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
112112

113113
pub fn vortex_bytebool::ByteBoolVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
114114

‎encodings/bytebool/src/array.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use vortex_array::ArrayHash;
99
use vortex_array::ArrayRef;
1010
use vortex_array::EmptyMetadata;
1111
use vortex_array::ExecutionCtx;
12+
use vortex_array::ExecutionStep;
1213
use vortex_array::IntoArray;
1314
use vortex_array::Precision;
1415
use vortex_array::arrays::BoolArray;
@@ -182,10 +183,12 @@ impl VTable for ByteBoolVTable {
182183
crate::rules::RULES.evaluate(array, parent, child_idx)
183184
}
184185

185-
fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
186+
fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
186187
let boolean_buffer = BitBuffer::from(array.as_slice());
187188
let validity = array.validity().clone();
188-
Ok(BoolArray::new(boolean_buffer, validity).into_array())
189+
Ok(ExecutionStep::Done(
190+
BoolArray::new(boolean_buffer, validity).into_array(),
191+
))
189192
}
190193

191194
fn execute_parent(

‎encodings/datetime-parts/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ pub fn vortex_datetime_parts::DateTimePartsVTable::deserialize(bytes: &[u8], _dt
182182

183183
pub fn vortex_datetime_parts::DateTimePartsVTable::dtype(array: &vortex_datetime_parts::DateTimePartsArray) -> &vortex_array::dtype::DType
184184

185-
pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
185+
pub fn vortex_datetime_parts::DateTimePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
186186

187187
pub fn vortex_datetime_parts::DateTimePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
188188

‎encodings/datetime-parts/src/array.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use vortex_array::ArrayRef;
1010
use vortex_array::DeserializeMetadata;
1111
use vortex_array::DynArray;
1212
use vortex_array::ExecutionCtx;
13+
use vortex_array::ExecutionStep;
1314
use vortex_array::IntoArray;
1415
use vortex_array::Precision;
1516
use vortex_array::ProstMetadata;
@@ -221,8 +222,10 @@ impl VTable for DateTimePartsVTable {
221222
Ok(())
222223
}
223224

224-
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
225-
Ok(decode_to_temporal(array, ctx)?.into_array())
225+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
226+
Ok(ExecutionStep::Done(
227+
decode_to_temporal(array, ctx)?.into_array(),
228+
))
226229
}
227230

228231
fn reduce_parent(

‎encodings/decimal-byte-parts/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::deserialize(bytes: &[u
112112

113113
pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsArray) -> &vortex_array::dtype::DType
114114

115-
pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
115+
pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
116116

117117
pub fn vortex_decimal_byte_parts::DecimalBytePartsVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
118118

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use vortex_array::ArrayHash;
1313
use vortex_array::ArrayRef;
1414
use vortex_array::DynArray;
1515
use vortex_array::ExecutionCtx;
16+
use vortex_array::ExecutionStep;
1617
use vortex_array::IntoArray;
1718
use vortex_array::Precision;
1819
use vortex_array::ProstMetadata;
@@ -189,8 +190,8 @@ impl VTable for DecimalBytePartsVTable {
189190
PARENT_RULES.evaluate(array, parent, child_idx)
190191
}
191192

192-
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
193-
to_canonical_decimal(array, ctx)
193+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
194+
to_canonical_decimal(array, ctx).map(ExecutionStep::Done)
194195
}
195196

196197
fn execute_parent(

‎encodings/fastlanes/public-api.lock‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ pub fn vortex_fastlanes::BitPackedVTable::deserialize(bytes: &[u8], _dtype: &vor
242242

243243
pub fn vortex_fastlanes::BitPackedVTable::dtype(array: &vortex_fastlanes::BitPackedArray) -> &vortex_array::dtype::DType
244244

245-
pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
245+
pub fn vortex_fastlanes::BitPackedVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
246246

247247
pub fn vortex_fastlanes::BitPackedVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
248248

@@ -372,7 +372,7 @@ pub fn vortex_fastlanes::DeltaVTable::deserialize(bytes: &[u8], _dtype: &vortex_
372372

373373
pub fn vortex_fastlanes::DeltaVTable::dtype(array: &vortex_fastlanes::DeltaArray) -> &vortex_array::dtype::DType
374374

375-
pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
375+
pub fn vortex_fastlanes::DeltaVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
376376

377377
pub fn vortex_fastlanes::DeltaVTable::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId
378378

@@ -510,7 +510,7 @@ pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_arr
510510

511511
pub fn vortex_fastlanes::FoRVTable::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType
512512

513-
pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
513+
pub fn vortex_fastlanes::FoRVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
514514

515515
pub fn vortex_fastlanes::FoRVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
516516

@@ -646,7 +646,7 @@ pub fn vortex_fastlanes::RLEVTable::deserialize(bytes: &[u8], _dtype: &vortex_ar
646646

647647
pub fn vortex_fastlanes::RLEVTable::dtype(array: &vortex_fastlanes::RLEArray) -> &vortex_array::dtype::DType
648648

649-
pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
649+
pub fn vortex_fastlanes::RLEVTable::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
650650

651651
pub fn vortex_fastlanes::RLEVTable::execute_parent(array: &Self::Array, parent: &vortex_array::array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::ArrayRef>>
652652

0 commit comments

Comments
 (0)