Skip to content

Commit 312d58f

Browse files
tekknolagitekknolagi
committed
ZJIT: Obviate optimize_getivar
1 parent a0c02a4 commit 312d58f

4 files changed

Lines changed: 226 additions & 210 deletions

File tree

‎zjit/src/hir.rs‎

Lines changed: 113 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -3922,13 +3922,22 @@ impl Function {
39223922
}
39233923

39243924
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state });
3925+
let id = unsafe { get_cme_def_body_attr_id(cme) };
39253926
if let Some(profiled_type) = profiled_type {
39263927
let argc = unsafe { vm_ci_argc(ci) } as i32;
3928+
// TODO: attr_writer SetIvar has a null inline cache and may target a receiver
3929+
// operand other than CFP self. Support it with a reprofile strategy that
3930+
// profiles the receiver operand even after the send insn has finished profiling.
3931+
let recompile = None;
39273932
recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) });
3933+
self.try_emit_optimized_setivar(block, recv, id, val, profiled_type, state, recompile).unwrap_or_else(|counter| {
3934+
self.count(block, counter);
3935+
self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state });
3936+
});
3937+
} else {
3938+
// No shape information, just static class information
3939+
self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state });
39283940
}
3929-
let id = unsafe { get_cme_def_body_attr_id(cme) };
3930-
3931-
self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state });
39323941
self.make_equal_to(insn_id, val);
39333942
} else if !has_block && def_type == VM_METHOD_TYPE_OPTIMIZED {
39343943
let opt_type: OptimizedMethodType = unsafe { get_cme_def_body_optimized_type(cme) }.into();
@@ -4938,106 +4947,97 @@ impl Function {
49384947
Ok(self.load_ivar(block, self_val, profiled_type, id))
49394948
}
49404949

4941-
fn optimize_getivar(&mut self) {
4942-
for block in self.reverse_post_order() {
4943-
let old_insns = std::mem::take(&mut self.blocks[block.0].insns);
4944-
assert!(self.blocks[block.0].insns.is_empty());
4945-
for insn_id in old_insns {
4946-
match self.find(insn_id) {
4947-
Insn::SetIvar { self_val, id, val, state, ic } => {
4948-
let Some(recv_type) = self.profiled_type_of_at(self_val, state) else {
4949-
// No (monomorphic/skewed polymorphic) profile info
4950-
self.count(block, Counter::setivar_fallback_not_monomorphic);
4951-
self.push_insn_id(block, insn_id); continue;
4952-
};
4953-
if recv_type.flags().is_immediate() {
4954-
// Instance variable lookups on immediate values are always nil
4955-
self.count(block, Counter::setivar_fallback_immediate);
4956-
self.push_insn_id(block, insn_id); continue;
4957-
}
4958-
assert!(recv_type.shape().is_valid());
4959-
if !recv_type.flags().is_t_object() {
4960-
// Check if the receiver is a T_OBJECT
4961-
self.count(block, Counter::setivar_fallback_not_t_object);
4962-
self.push_insn_id(block, insn_id); continue;
4963-
}
4964-
if recv_type.shape().is_complex() {
4965-
// too-complex shapes can't use index access
4966-
self.count(block, Counter::setivar_fallback_complex);
4967-
self.push_insn_id(block, insn_id); continue;
4968-
}
4969-
if recv_type.shape().is_frozen() {
4970-
// Can't set ivars on frozen objects
4971-
self.count(block, Counter::setivar_fallback_frozen);
4972-
self.push_insn_id(block, insn_id); continue;
4973-
}
4974-
if self.policy.no_side_exits {
4975-
// TODO: Support polymorphic SetIvar shape-specialized paths.
4976-
// https://github.com/Shopify/ruby/issues/927
4977-
// On the final version, keep the SetIvar fallback instead of another shape guard.
4978-
self.push_insn_id(block, insn_id); continue;
4979-
}
4980-
let mut ivar_index: attr_index_t = 0;
4981-
let mut next_shape_id = recv_type.shape();
4982-
if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } {
4983-
// Current shape does not contain this ivar; do a shape transition.
4984-
let current_shape_id = recv_type.shape();
4985-
let class = recv_type.class();
4986-
// We're only looking at T_OBJECT so ignore all of the imemo stuff.
4987-
assert!(recv_type.flags().is_t_object());
4988-
next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id.0, id, class) });
4989-
// If the VM ran out of shapes, or this class generated too many leaf,
4990-
// it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table).
4991-
let new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id.0) };
4992-
// TODO(max): Is it OK to bail out here after making a shape transition?
4993-
if new_shape_complex {
4994-
self.count(block, Counter::setivar_fallback_new_shape_complex);
4995-
self.push_insn_id(block, insn_id); continue;
4996-
}
4997-
let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) };
4998-
assert!(ivar_result, "New shape must have the ivar index");
4999-
let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) };
5000-
let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) };
5001-
// If the new shape has a different capacity, or is COMPLEX, we'll have to
5002-
// reallocate it.
5003-
let needs_extension = next_capacity != current_capacity;
5004-
if needs_extension {
5005-
self.count(block, Counter::setivar_fallback_new_shape_needs_extension);
5006-
self.push_insn_id(block, insn_id); continue;
5007-
}
5008-
// Fall through to emitting the ivar write
5009-
}
5010-
let self_val = self.guard_heap(block, self_val, state);
5011-
let shape = self.load_shape(block, self_val);
5012-
// TODO: attr_writer SetIvar has a null inline cache and may target a receiver
5013-
// operand other than CFP self. Support it with a reprofile strategy that
5014-
// profiles the receiver operand even after the send insn has finished profiling.
5015-
let recompile = if ic.is_null() { None } else { Some(Recompile::ProfileSelf) };
5016-
self.guard_shape(block, shape, recv_type.shape(), state, recompile);
5017-
// Current shape contains this ivar
5018-
let (ivar_storage, offset) = if recv_type.flags().is_embedded() {
5019-
// See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h
5020-
let offset = ROBJECT_OFFSET_AS_ARY + (SIZEOF_VALUE * ivar_index.to_usize()) as i32;
5021-
(self_val, offset)
5022-
} else {
5023-
let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: FieldName::as_heap, offset: ROBJECT_OFFSET_AS_HEAP_FIELDS, return_type: types::CPtr });
5024-
let offset = SIZEOF_VALUE_I32 * ivar_index as i32;
5025-
(as_heap, offset)
5026-
};
5027-
self.push_insn(block, Insn::StoreField { recv: ivar_storage, id: id.into(), offset, val });
5028-
self.push_insn(block, Insn::WriteBarrier { recv: self_val, val });
5029-
if next_shape_id != recv_type.shape() {
5030-
// Write the new shape ID
5031-
let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) });
5032-
let shape_id_offset = unsafe { rb_shape_id_offset() };
5033-
self.push_insn(block, Insn::StoreField { recv: self_val, id: FieldName::shape_id, offset: shape_id_offset, val: shape_id });
5034-
}
5035-
}
5036-
_ => { self.push_insn_id(block, insn_id); }
5037-
}
5038-
}
4950+
fn try_emit_optimized_setivar(&mut self, block: BlockId, self_val: InsnId, id: ID, val: InsnId, profiled_type: ProfiledType, state: InsnId, recompile: Option<Recompile>) -> Result<(), Counter> {
4951+
if profiled_type.flags().is_immediate() {
4952+
// Instance variable lookups on immediate values are always nil
4953+
return Err(Counter::setivar_fallback_immediate);
50394954
}
5040-
crate::stats::trace_compile_phase("infer_types", || self.infer_types());
4955+
if !profiled_type.flags().is_t_object() {
4956+
// Check if the receiver is a T_OBJECT
4957+
return Err(Counter::setivar_fallback_not_t_object);
4958+
}
4959+
assert!(profiled_type.shape().is_valid());
4960+
if profiled_type.shape().is_frozen() {
4961+
// Can't set ivars on frozen objects
4962+
return Err(Counter::setivar_fallback_frozen);
4963+
}
4964+
if profiled_type.shape().is_complex() {
4965+
// too-complex shapes can't use index access
4966+
return Err(Counter::setivar_fallback_complex);
4967+
}
4968+
if self.policy.no_side_exits {
4969+
// On the final version, skip GetIvar shape specialization.
4970+
// iseq_to_hir already generates polymorphic branches with a
4971+
// GetIvar C call fallback for getinstancevariable, so we don't
4972+
// need to wrap it again here.
4973+
return Err(Counter::setivar_fallback_no_side_exits);
4974+
}
4975+
4976+
let mut ivar_index: attr_index_t = 0;
4977+
let mut next_shape_id = profiled_type.shape();
4978+
if !unsafe { rb_shape_get_iv_index(profiled_type.shape().0, id, &mut ivar_index) } {
4979+
// Current shape does not contain this ivar; do a shape transition.
4980+
let current_shape_id = profiled_type.shape();
4981+
let class = profiled_type.class();
4982+
// We're only looking at T_OBJECT so ignore all of the imemo stuff.
4983+
assert!(profiled_type.flags().is_t_object());
4984+
next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id.0, id, class) });
4985+
// If the VM ran out of shapes, or this class generated too many leaf,
4986+
// it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table).
4987+
let new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id.0) };
4988+
// TODO(max): Is it OK to bail out here after making a shape transition?
4989+
if new_shape_complex {
4990+
return Err(Counter::setivar_fallback_new_shape_complex);
4991+
}
4992+
let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) };
4993+
assert!(ivar_result, "New shape must have the ivar index");
4994+
let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) };
4995+
let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) };
4996+
// If the new shape has a different capacity, or is COMPLEX, we'll have to
4997+
// reallocate it.
4998+
let needs_extension = next_capacity != current_capacity;
4999+
if needs_extension {
5000+
return Err(Counter::setivar_fallback_new_shape_needs_extension);
5001+
}
5002+
// Fall through to emitting the ivar write
5003+
}
5004+
let self_val = self.guard_heap(block, self_val, state);
5005+
let shape = self.load_shape(block, self_val);
5006+
self.guard_shape(block, shape, profiled_type.shape(), state, recompile);
5007+
// Current shape contains this ivar
5008+
let (ivar_storage, offset) = if profiled_type.flags().is_embedded() {
5009+
// See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h
5010+
let offset = ROBJECT_OFFSET_AS_ARY + (SIZEOF_VALUE * ivar_index.to_usize()) as i32;
5011+
(self_val, offset)
5012+
} else {
5013+
let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: FieldName::as_heap, offset: ROBJECT_OFFSET_AS_HEAP_FIELDS, return_type: types::CPtr });
5014+
let offset = SIZEOF_VALUE_I32 * ivar_index as i32;
5015+
(as_heap, offset)
5016+
};
5017+
self.push_insn(block, Insn::StoreField { recv: ivar_storage, id: id.into(), offset, val });
5018+
self.push_insn(block, Insn::WriteBarrier { recv: self_val, val });
5019+
if next_shape_id != profiled_type.shape() {
5020+
// Write the new shape ID
5021+
let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) });
5022+
let shape_id_offset = unsafe { rb_shape_id_offset() };
5023+
self.push_insn(block, Insn::StoreField { recv: self_val, id: FieldName::shape_id, offset: shape_id_offset, val: shape_id });
5024+
}
5025+
Ok(())
5026+
}
5027+
5028+
fn optimize_getivar(&mut self) {
5029+
// for block in self.reverse_post_order() {
5030+
// let old_insns = std::mem::take(&mut self.blocks[block.0].insns);
5031+
// assert!(self.blocks[block.0].insns.is_empty());
5032+
// for insn_id in old_insns {
5033+
// match self.find(insn_id) {
5034+
// Insn::SetIvar { self_val, id, val, state, ic } => {
5035+
// }
5036+
// _ => { self.push_insn_id(block, insn_id); }
5037+
// }
5038+
// }
5039+
// }
5040+
// crate::stats::trace_compile_phase("infer_types", || self.infer_types());
50415041
}
50425042

50435043
fn gen_patch_points_for_optimized_ccall(&mut self, block: BlockId, recv_class: VALUE, method_id: ID, cme: *const rb_callable_method_entry_struct, state: InsnId) {
@@ -9065,7 +9065,7 @@ fn add_iseq_to_hir(
90659065
}
90669066
YARVINSN_setinstancevariable => {
90679067
let id = ID(get_arg(pc, 0).as_u64());
9068-
let ic = get_arg(pc, 1).as_ptr();
9068+
let ic: *const iseq_inline_iv_cache_entry = get_arg(pc, 1).as_ptr();
90699069
// Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_setivar
90709070
// TODO: We only really need this if self_val is a class/module
90719071
if !fun.assume_single_ractor_mode(block, exit_id) {
@@ -9074,7 +9074,16 @@ fn add_iseq_to_hir(
90749074
break; // End the block
90759075
}
90769076
let val = state.stack_pop()?;
9077-
fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id });
9077+
if let Some(profiled_type) = fun.monomorphic_summary(&profiles, self_param, exit_id) {
9078+
// TODO(max): Assert ic is never null
9079+
let recompile = if ic.is_null() { None } else { Some(Recompile::ProfileSelf) };
9080+
fun.try_emit_optimized_setivar(block, self_param, id, val, profiled_type, exit_id, recompile).unwrap_or_else(|counter| {
9081+
fun.count(block, counter);
9082+
fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id });
9083+
});
9084+
} else {
9085+
fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id });
9086+
}
90789087
// SetIvar will raise if self is an immediate. If it raises, we will have
90799088
// exited JIT code. So upgrade the type within JIT code to a heap object.
90809089
self_param = fun.push_insn(block, Insn::RefineType { val: self_param, new_type: types::HeapBasicObject });

0 commit comments

Comments
 (0)