diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 80670585aa..9ffd83ecd5 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -34,21 +34,6 @@ pub struct ZendMMState { /// The engine's previous custom shutdown function, if there is one. prev_custom_mm_shutdown: Option, /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_alloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function - /// pointer, otherwise there will be dragons. - alloc: unsafe fn(size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_realloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid - /// function pointer, otherwise there will be dragons. - realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_free()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function - /// pointer, otherwise there will be dragons. - free: unsafe fn(*mut c_void), - /// Safety: this function pointer is only allowed to point to /// `alloc_prof_prev_gc()` when at the same time the /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function /// pointer, otherwise there will be dragons. @@ -78,9 +63,6 @@ impl ZendMMState { prev_custom_mm_free: None, prev_custom_mm_gc: None, prev_custom_mm_shutdown: None, - alloc: super::alloc_prof_panic_alloc, - realloc: super::alloc_prof_panic_realloc, - free: super::alloc_prof_panic_free, gc: alloc_prof_panic_gc, shutdown: alloc_prof_panic_shutdown, } @@ -127,15 +109,9 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_shutdown), ); } - zend_mm_state.alloc = alloc_prof_prev_alloc; - zend_mm_state.free = alloc_prof_prev_free; - zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.gc = alloc_prof_prev_gc; zend_mm_state.shutdown = alloc_prof_prev_shutdown; } else { - zend_mm_state.alloc = alloc_prof_orig_alloc; - zend_mm_state.free = alloc_prof_orig_free; - zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.gc = alloc_prof_orig_gc; zend_mm_state.shutdown = alloc_prof_orig_shutdown; @@ -150,14 +126,22 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { zend_mm_state.prev_custom_mm_shutdown = None; } - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); // install our custom handler to ZendMM unsafe { zend::zend_mm_set_custom_handlers_ex( heap, - Some(alloc_prof_malloc), + Some(malloc_handler), Some(free_handler), Some(realloc_handler), Some(alloc_prof_gc), @@ -213,10 +197,18 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { &mut custom_mm_shutdown, ); } - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); if custom_mm_free != Some(free_handler) - || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) || custom_mm_gc != Some(alloc_prof_gc) || custom_mm_shutdown != Some(alloc_prof_shutdown) @@ -280,9 +272,17 @@ unsafe fn restore_zend_heap(heap: *mut zend::_zend_mm_heap, custom_heap: c_int) ptr::write(heap as *mut c_int, custom_heap); } +fn alloc_prof_malloc_handler(has_previous_allocator: bool) -> zend::VmMmCustomAllocFn { + if has_previous_allocator { + alloc_prof_malloc_custom + } else { + alloc_prof_malloc + } +} + #[cfg(not(php_debug))] unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { - alloc_prof_malloc_impl(len) + alloc_prof_malloc_impl::(len) } #[cfg(php_debug)] @@ -293,17 +293,53 @@ unsafe extern "C" fn alloc_prof_malloc( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_malloc_impl(len) + alloc_prof_malloc_impl::(len) +} + +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +#[cfg(not(php_debug))] +unsafe extern "C" fn alloc_prof_malloc_custom(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +#[cold] +#[cfg(php_debug)] +unsafe extern "C" fn alloc_prof_malloc_custom( + len: size_t, + _file: *const c_char, + _line: c_uint, + _orig_file: *const c_char, + _orig_line: c_uint, +) -> *mut c_void { + alloc_prof_malloc_impl::(len) } #[inline(always)] -unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { +unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(alloc)(len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + let alloc = state.prev_custom_mm_alloc.unwrap(); + #[cfg(php_debug)] + let ptr = alloc(len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = alloc(len); + ptr + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_alloc(heap, len); + ptr + }; // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -318,128 +354,103 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { ptr } -unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_alloc` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let alloc = tls_zend_mm_state_get!(prev_custom_mm_alloc).unwrap(); - #[cfg(php_debug)] - { - alloc(len, ptr::null(), 0, ptr::null(), 0) - } - #[cfg(not(php_debug))] - alloc(len) -} - -unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_alloc(heap, len) -} - /// This function exists because when calling `zend_mm_set_custom_handlers()`, /// you need to pass a pointer to a `free()` function as well, otherwise your /// custom handlers won't be installed. We cannot just point to the original /// `zend::_zend_mm_free()` as the function definitions differ. #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { - alloc_prof_free_impl(ptr); +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr); } #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_free( +unsafe extern "C" fn alloc_prof_free( ptr: *mut c_void, _file: *const c_char, _line: c_uint, _orig_file: *const c_char, _orig_line: c_uint, ) { - alloc_prof_free_impl(ptr); -} - -fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { - if heap_live_enabled { - alloc_prof_free - } else { - alloc_prof_free_noop - } + alloc_prof_free_impl::(ptr); } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - tls_zend_mm_state_get!(free)(ptr); +unsafe extern "C" fn alloc_prof_free_custom(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr); } +#[cold] #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_free_noop( +unsafe extern "C" fn alloc_prof_free_custom( ptr: *mut c_void, _file: *const c_char, _line: c_uint, _orig_file: *const c_char, _orig_line: c_uint, ) { - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_free_impl::(ptr); +} + +fn alloc_prof_free_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomFreeFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_free::, + (false, false) => alloc_prof_free::, + (true, true) => alloc_prof_free_custom::, + (false, true) => alloc_prof_free_custom::, + } } #[inline(always)] -unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { - // Heap-live is enabled when this handler is registered. - if !ptr.is_null() { +unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { + if TRACK && !ptr.is_null() { untrack_allocation(ptr); } - tls_zend_mm_state_get!(free)(ptr); -} - -unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_free` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let free = tls_zend_mm_state_get!(prev_custom_mm_free).unwrap(); - #[cfg(php_debug)] - { - free(ptr, core::ptr::null(), 0, core::ptr::null(), 0) - } - #[cfg(not(php_debug))] - free(ptr) -} -unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_free(heap, ptr); + let state = tls_zend_mm_state_copy!(); + if CUSTOM { + let free = state.prev_custom_mm_free.unwrap(); + #[cfg(php_debug)] + free(ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + free(ptr); + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + zend::_zend_mm_free(heap, ptr); + } } -fn alloc_prof_realloc_handler(heap_live_enabled: bool) -> zend::VmMmCustomReallocFn { - if heap_live_enabled { - alloc_prof_realloc - } else { - alloc_prof_realloc_no_untrack +fn alloc_prof_realloc_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomReallocFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_realloc::, + (false, false) => alloc_prof_realloc::, + (true, true) => alloc_prof_realloc_custom::, + (false, true) => alloc_prof_realloc_custom::, } } #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) +unsafe extern "C" fn alloc_prof_realloc( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { + alloc_prof_realloc_impl::(prev_ptr, len) } #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_realloc( +unsafe extern "C" fn alloc_prof_realloc( prev_ptr: *mut c_void, len: size_t, _file: *const c_char, @@ -447,19 +458,22 @@ unsafe extern "C" fn alloc_prof_realloc( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } +#[cold] #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, _file: *const c_char, @@ -467,41 +481,49 @@ unsafe extern "C" fn alloc_prof_realloc_no_untrack( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } #[inline(always)] -unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { +unsafe fn alloc_prof_realloc_impl( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + let realloc = state.prev_custom_mm_realloc.unwrap(); + #[cfg(php_debug)] + let ptr = realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = realloc(prev_ptr, len); + ptr + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len); + ptr + }; // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it // before any userland-only early return, then let the new allocation be // re-sampled at the reported size. - if !prev_ptr.is_null() { + if UNTRACK && !prev_ptr.is_null() { untrack_allocation(prev_ptr); } alloc_prof_realloc_sample(ptr, len) } -#[inline(always)] -unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); - - alloc_prof_realloc_sample(ptr, len) -} - #[inline(always)] unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null @@ -521,33 +543,6 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_realloc` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let realloc = tls_zend_mm_state_get!(prev_custom_mm_realloc).unwrap(); - #[cfg(php_debug)] - { - realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0) - } - #[cfg(not(php_debug))] - realloc(prev_ptr, len) -} - -unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_realloc(heap, prev_ptr, len) -} - unsafe extern "C" fn alloc_prof_gc() -> size_t { tls_zend_mm_state_get!(gc)() } @@ -595,14 +590,39 @@ mod tests { use super::*; #[test] - fn free_handler_tracks_only_when_heap_live_is_enabled() { + fn handlers_are_selected_at_rinit() { + assert_eq!( + alloc_prof_free_handler(true, false) as usize, + alloc_prof_free:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(false, false) as usize, + alloc_prof_free:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(true, true) as usize, + alloc_prof_free_custom:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(false, true) as usize, + alloc_prof_free_custom:: as zend::VmMmCustomFreeFn as usize + ); + + assert_eq!( + alloc_prof_realloc_handler(true, false) as usize, + alloc_prof_realloc:: as zend::VmMmCustomReallocFn as usize + ); + assert_eq!( + alloc_prof_realloc_handler(false, false) as usize, + alloc_prof_realloc:: as zend::VmMmCustomReallocFn as usize + ); assert_eq!( - alloc_prof_free_handler(true) as usize, - alloc_prof_free as zend::VmMmCustomFreeFn as usize + alloc_prof_realloc_handler(true, true) as usize, + alloc_prof_realloc_custom:: as zend::VmMmCustomReallocFn as usize ); assert_eq!( - alloc_prof_free_handler(false) as usize, - alloc_prof_free_noop as zend::VmMmCustomFreeFn as usize + alloc_prof_realloc_handler(false, true) as usize, + alloc_prof_realloc_custom:: as zend::VmMmCustomReallocFn as usize ); } diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 12a1b37c58..e10b83b1c8 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -35,21 +35,6 @@ pub struct ZendMMState { /// The engine's previous custom free function, if there is one. prev_custom_mm_free: Option, prepare_restore_zend_heap: (ZendHeapPrepareFn, ZendHeapRestoreFn), - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_alloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function - /// pointer, otherwise there will be dragons. - alloc: unsafe fn(size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_realloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid - /// function pointer, otherwise there will be dragons. - realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_free()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function - /// pointer, otherwise there will be dragons. - free: unsafe fn(*mut c_void), } impl ZendMMState { @@ -61,9 +46,6 @@ impl ZendMMState { prev_custom_mm_realloc: None, prev_custom_mm_free: None, prepare_restore_zend_heap: (prepare_zend_heap, restore_zend_heap), - alloc: super::alloc_prof_panic_alloc, - realloc: super::alloc_prof_panic_realloc, - free: super::alloc_prof_panic_free, } } } @@ -108,15 +90,9 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_realloc), ); } - zend_mm_state.alloc = alloc_prof_prev_alloc; - zend_mm_state.free = alloc_prof_prev_free; - zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap_none, restore_zend_heap_none); } else { - zend_mm_state.alloc = alloc_prof_orig_alloc; - zend_mm_state.free = alloc_prof_orig_free; - zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap, restore_zend_heap); // Reset previous handlers to None. There might be a chaotic neighbor that @@ -128,14 +104,22 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { zend_mm_state.prev_custom_mm_realloc = None; } - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); // install our custom handler to ZendMM unsafe { zend::ddog_php_prof_zend_mm_set_custom_handlers( heap, - Some(alloc_prof_malloc), + Some(malloc_handler), Some(free_handler), Some(realloc_handler), ); @@ -185,10 +169,18 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { &mut custom_mm_realloc, ); } - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); if custom_mm_free != Some(free_handler) - || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) { // Custom handlers are installed, but it's not us. Someone, somewhere might have @@ -291,13 +283,47 @@ unsafe extern "C" fn alloc_prof_gc_mem_caches( } } +fn alloc_prof_malloc_handler(has_previous_allocator: bool) -> zend::VmMmCustomAllocFn { + if has_previous_allocator { + alloc_prof_malloc_custom + } else { + alloc_prof_malloc + } +} + unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_malloc_custom(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +#[inline(always)] +unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(alloc)(len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + state.prev_custom_mm_alloc.unwrap()(len) + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; + let custom_heap = prepare(heap); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_alloc(heap, len); + restore(heap, custom_heap); + ptr + }; // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -312,122 +338,118 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { ptr } -unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_alloc` is also initialised - let alloc = tls_zend_mm_state_get!(prev_custom_mm_alloc).unwrap(); - alloc(len) -} - -unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - let (prepare, restore) = tls_zend_mm_state_get!(prepare_restore_zend_heap); - let custom_heap = prepare(heap); - #[cfg(php_debug)] - let ptr: *mut c_void = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - let ptr: *mut c_void = zend::_zend_mm_alloc(heap, len); - restore(heap, custom_heap); - ptr -} - /// This function exists because when calling `zend_mm_set_custom_handlers()`, /// you need to pass a pointer to a `free()` function as well, otherwise your /// custom handlers won't be installed. We cannot just point to the original /// `zend::_zend_mm_free()` as the function definitions differ. -unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { - // Heap-live is enabled when this handler is registered. - if !ptr.is_null() { - untrack_allocation(ptr); - } - - tls_zend_mm_state_get!(free)(ptr); +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr) } -fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { - if heap_live_enabled { - alloc_prof_free - } else { - alloc_prof_free_noop - } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_free_custom(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr) } -unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - tls_zend_mm_state_get!(free)(ptr); +fn alloc_prof_free_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomFreeFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_free::, + (false, false) => alloc_prof_free::, + (true, true) => alloc_prof_free_custom::, + (false, true) => alloc_prof_free_custom::, + } } -unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_free` is also initialised - let free = tls_zend_mm_state_get!(prev_custom_mm_free).unwrap(); - free(ptr) -} +#[inline(always)] +unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { + if TRACK && !ptr.is_null() { + untrack_allocation(ptr); + } -unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - #[cfg(php_debug)] - zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_free(heap, ptr); + let state = tls_zend_mm_state_copy!(); + if CUSTOM { + state.prev_custom_mm_free.unwrap()(ptr); + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + zend::_zend_mm_free(heap, ptr); + } } -fn alloc_prof_realloc_handler(heap_live_enabled: bool) -> zend::VmMmCustomReallocFn { - if heap_live_enabled { - alloc_prof_realloc - } else { - alloc_prof_realloc_no_untrack +fn alloc_prof_realloc_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomReallocFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_realloc::, + (false, false) => alloc_prof_realloc::, + (true, true) => alloc_prof_realloc_custom::, + (false, true) => alloc_prof_realloc_custom::, } } -unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) +unsafe extern "C" fn alloc_prof_realloc( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { + alloc_prof_realloc_impl::(prev_ptr, len) } -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } #[inline(always)] -unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { +unsafe fn alloc_prof_realloc_impl( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + state.prev_custom_mm_realloc.unwrap()(prev_ptr, len) + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; + let custom_heap = prepare(heap); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len); + restore(heap, custom_heap); + ptr + }; // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it // before any userland-only early return, then let the new allocation be // re-sampled at the reported size. - if !prev_ptr.is_null() { + if UNTRACK && !prev_ptr.is_null() { untrack_allocation(prev_ptr); } alloc_prof_realloc_sample(ptr, len) } -#[inline(always)] -unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); - - alloc_prof_realloc_sample(ptr, len) -} - #[inline(always)] unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null @@ -447,29 +469,6 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_realloc` is also initialised - let realloc = tls_zend_mm_state_get!(prev_custom_mm_realloc).unwrap(); - realloc(prev_ptr, len) -} - -unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - let (prepare, restore) = tls_zend_mm_state_get!(prepare_restore_zend_heap); - let custom_heap = prepare(heap); - #[cfg(php_debug)] - let ptr: *mut c_void = - zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - let ptr: *mut c_void = zend::_zend_mm_realloc(heap, prev_ptr, len); - restore(heap, custom_heap); - ptr -} - /// safe wrapper for `zend::is_zend_mm()`. /// `true` means the internal ZendMM is being used, `false` means that a custom memory manager is /// installed. Upstream returns a `c_bool` as of PHP 8.0. PHP 7 returns a `c_int` @@ -489,14 +488,39 @@ mod tests { use super::*; #[test] - fn free_handler_tracks_only_when_heap_live_is_enabled() { + fn handlers_are_selected_at_rinit() { + assert_eq!( + alloc_prof_free_handler(true, false) as usize, + alloc_prof_free:: as usize + ); + assert_eq!( + alloc_prof_free_handler(false, false) as usize, + alloc_prof_free:: as usize + ); + assert_eq!( + alloc_prof_free_handler(true, true) as usize, + alloc_prof_free_custom:: as usize + ); + assert_eq!( + alloc_prof_free_handler(false, true) as usize, + alloc_prof_free_custom:: as usize + ); + + assert_eq!( + alloc_prof_realloc_handler(true, false) as usize, + alloc_prof_realloc:: as usize + ); + assert_eq!( + alloc_prof_realloc_handler(false, false) as usize, + alloc_prof_realloc:: as usize + ); assert_eq!( - alloc_prof_free_handler(true) as usize, - alloc_prof_free as usize + alloc_prof_realloc_handler(true, true) as usize, + alloc_prof_realloc_custom:: as usize ); assert_eq!( - alloc_prof_free_handler(false) as usize, - alloc_prof_free_noop as usize + alloc_prof_realloc_handler(false, true) as usize, + alloc_prof_realloc_custom:: as usize ); } diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 20f4d2280a..f366d6a1cb 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -80,6 +80,52 @@ pub mod allocation_ge84; #[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub mod allocation_le83; +// Handler-selection tests retain callbacks in a binary that is not loaded by PHP. +#[cfg(all(test, not(php_zts)))] +#[export_name = "executor_globals"] +static mut TEST_EXECUTOR_GLOBALS: core::mem::MaybeUninit = + core::mem::MaybeUninit::zeroed(); + +#[cfg(all(test, not(php_debug)))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_free(_heap: *mut zend::_zend_mm_heap, _ptr: *mut c_void) {} + +#[cfg(all(test, php_debug))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_free( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _file: *const libc::c_char, + _line: libc::c_uint, + _orig_file: *const libc::c_char, + _orig_line: libc::c_uint, +) { +} + +#[cfg(all(test, not(php_debug)))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_realloc( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _len: size_t, +) -> *mut c_void { + ptr::null_mut() +} + +#[cfg(all(test, php_debug))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_realloc( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _len: size_t, + _file: *const libc::c_char, + _line: libc::c_uint, + _orig_file: *const libc::c_char, + _orig_line: libc::c_uint, +) -> *mut c_void { + ptr::null_mut() +} + /// Default sampling interval in bytes (4 MiB). pub const DEFAULT_ALLOCATION_SAMPLING_INTERVAL: NonZeroU32 = NonZero::new(1024 * 4096).unwrap(); @@ -270,19 +316,8 @@ pub fn alloc_prof_rshutdown() { allocation_ge84::alloc_prof_rshutdown(heap_live_enabled); } +#[cfg(php_zend_mm_set_custom_handlers_ex)] #[track_caller] fn initialization_panic() -> ! { panic!("Allocation profiler was not initialized properly. Please fill an issue stating the PHP version and the backtrace from this panic."); } - -unsafe fn alloc_prof_panic_alloc(_len: size_t) -> *mut c_void { - initialization_panic(); -} - -unsafe fn alloc_prof_panic_realloc(_prev_ptr: *mut c_void, _len: size_t) -> *mut c_void { - initialization_panic(); -} - -unsafe fn alloc_prof_panic_free(_ptr: *mut c_void) { - initialization_panic(); -}