From 241f72a438a75754171b82e9cb81aafc8886d327 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 27 Jul 2026 16:39:30 +0200 Subject: [PATCH 1/9] perf(profiling): avoid FFI call on allocation hot path Read executor_globals.current_execute_data directly on NTS builds instead of calling through the C FFI wrapper for every allocation and reallocation. Keep the existing wrapper on ZTS builds. A 60-second macOS sample reduced get_current_execute_data self time from 0.412% (207/50,210 main-thread samples) to 0.002% (1/49,737). Known allocation-hook self time fell from 4.184% to 3.601%, a 13.9% relative reduction. Repeated end-to-end throughput remained within system scheduling noise. Validation: cargo test (22 passed); allocation sampling-distance, memory-peak, and GC PHPTs passed. --- profiling/src/allocation/allocation_ge84.rs | 7 ++++--- profiling/src/allocation/allocation_le83.rs | 7 ++++--- profiling/src/allocation/mod.rs | 9 +++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 606f32df14..80670585aa 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -1,5 +1,6 @@ use crate::allocation::{ - allocation_profiling_stats_should_collect, collect_allocation, untrack_allocation, + allocation_profiling_stats_should_collect, collect_allocation, current_execute_data, + untrack_allocation, }; use crate::bindings as zend; use crate::PROFILER_NAME; @@ -306,7 +307,7 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations - if zend::ddog_php_prof_get_current_execute_data().is_null() { + if current_execute_data().is_null() { return ptr; } @@ -505,7 +506,7 @@ unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations - if zend::ddog_php_prof_get_current_execute_data().is_null() { + if current_execute_data().is_null() { return ptr; } diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index c36de739e9..12a1b37c58 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -1,5 +1,6 @@ use crate::allocation::{ - allocation_profiling_stats_should_collect, collect_allocation, untrack_allocation, + allocation_profiling_stats_should_collect, collect_allocation, current_execute_data, + untrack_allocation, }; use crate::bindings::{ self as zend, datadog_php_install_handler, datadog_php_zif_handler, @@ -300,7 +301,7 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations - if zend::ddog_php_prof_get_current_execute_data().is_null() { + if current_execute_data().is_null() { return ptr; } @@ -431,7 +432,7 @@ unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations - if zend::ddog_php_prof_get_current_execute_data().is_null() { + if current_execute_data().is_null() { return ptr; } diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 9d4f46d53c..20f4d2280a 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -39,6 +39,15 @@ pub(crate) unsafe fn get_zend_mm_state() -> *mut Cell { ptr::addr_of_mut!((*globals).zend_mm_state) } +#[inline(always)] +pub(crate) unsafe fn current_execute_data() -> *mut zend::zend_execute_data { + #[cfg(not(php_zts))] + return ptr::addr_of!(zend::executor_globals.current_execute_data).read(); + + #[cfg(php_zts)] + zend::ddog_php_prof_get_current_execute_data() +} + /// Macros for accessing ZendMMState from PHP globals. /// These are shared between PHP 8.3- and 8.4+ implementations. /// They are exported at the crate root and can be used in submodules. From 6e8788349d19029cacd0c45aa3e49939d4265bb0 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 27 Jul 2026 17:02:46 +0200 Subject: [PATCH 2/9] perf(profiling): remove indirect allocator forwarding Forward PHP 8.4+ allocations and frees from a single ZendMMState read instead of loading a function pointer, making an indirect Rust call, and reloading the state. Preserve neighboring custom allocator support with a predictable previous-handler check. Use unchecked heap extraction in the callbacks: rinit stores the heap before they can be invoked, and rshutdown removes them before clearing it. This avoids an Option discriminant check on every allocation and free. Across six 60-second runs per binary, mean allocation throughput increased from 9,366,444/s to 10,018,262/s (+6.96%) and median throughput increased by 7.56%. A 60-second native sample reduced known forwarding-path self time from 3.599% to 2.438% (-32.3% relative). Validation: cargo test (22 passed); allocation sampling-distance, memory-peak, and GC PHPTs passed. https://datadoghq.atlassian.net/browse/PROF-15506 --- profiling/src/allocation/allocation_ge84.rs | 84 +++++++-------------- profiling/src/allocation/mod.rs | 2 + 2 files changed, 28 insertions(+), 58 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 80670585aa..f19ed81d88 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -34,21 +34,11 @@ pub struct ZendMMState { /// The engine's previous custom shutdown function, if there is one. prev_custom_mm_shutdown: Option, /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_alloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function - /// pointer, otherwise there will be dragons. - alloc: unsafe fn(size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to /// `alloc_prof_prev_realloc()` when at the same time the /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid /// function pointer, otherwise there will be dragons. realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_free()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function - /// pointer, otherwise there will be dragons. - free: unsafe fn(*mut c_void), - /// Safety: this function pointer is only allowed to point to /// `alloc_prof_prev_gc()` when at the same time the /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function /// pointer, otherwise there will be dragons. @@ -78,9 +68,7 @@ impl ZendMMState { prev_custom_mm_free: None, prev_custom_mm_gc: None, prev_custom_mm_shutdown: None, - alloc: super::alloc_prof_panic_alloc, realloc: super::alloc_prof_panic_realloc, - free: super::alloc_prof_panic_free, gc: alloc_prof_panic_gc, shutdown: alloc_prof_panic_shutdown, } @@ -127,14 +115,10 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_shutdown), ); } - zend_mm_state.alloc = alloc_prof_prev_alloc; - zend_mm_state.free = alloc_prof_prev_free; zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.gc = alloc_prof_prev_gc; zend_mm_state.shutdown = alloc_prof_prev_shutdown; } else { - zend_mm_state.alloc = alloc_prof_orig_alloc; - zend_mm_state.free = alloc_prof_orig_free; zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.gc = alloc_prof_orig_gc; zend_mm_state.shutdown = alloc_prof_orig_shutdown; @@ -303,7 +287,7 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(alloc)(len); + let ptr = alloc_prof_forward_alloc(len); // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -318,27 +302,19 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { ptr } -unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_alloc` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let alloc = tls_zend_mm_state_get!(prev_custom_mm_alloc).unwrap(); - #[cfg(php_debug)] - { - alloc(len, ptr::null(), 0, ptr::null(), 0) +#[inline(always)] +unsafe fn alloc_prof_forward_alloc(len: size_t) -> *mut c_void { + let state = tls_zend_mm_state_copy!(); + if let Some(alloc) = state.prev_custom_mm_alloc { + #[cfg(php_debug)] + return alloc(len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + return alloc(len); } - #[cfg(not(php_debug))] - alloc(len) -} -unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); #[cfg(php_debug)] return zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); #[cfg(not(php_debug))] @@ -375,7 +351,7 @@ fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { #[cfg(not(php_debug))] unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_forward_free(ptr); } #[cfg(php_debug)] @@ -386,7 +362,7 @@ unsafe extern "C" fn alloc_prof_free_noop( _orig_file: *const c_char, _orig_line: c_uint, ) { - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_forward_free(ptr); } #[inline(always)] @@ -395,30 +371,22 @@ unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { if !ptr.is_null() { untrack_allocation(ptr); } - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_forward_free(ptr); } -unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_free` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let free = tls_zend_mm_state_get!(prev_custom_mm_free).unwrap(); - #[cfg(php_debug)] - { - free(ptr, core::ptr::null(), 0, core::ptr::null(), 0) +#[inline(always)] +unsafe fn alloc_prof_forward_free(ptr: *mut c_void) { + let state = tls_zend_mm_state_copy!(); + if let Some(free) = state.prev_custom_mm_free { + #[cfg(php_debug)] + return free(ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + return free(ptr); } - #[cfg(not(php_debug))] - free(ptr) -} -unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); #[cfg(php_debug)] return zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); #[cfg(not(php_debug))] diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 20f4d2280a..febaf12f79 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -275,6 +275,7 @@ fn initialization_panic() -> ! { panic!("Allocation profiler was not initialized properly. Please fill an issue stating the PHP version and the backtrace from this panic."); } +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] unsafe fn alloc_prof_panic_alloc(_len: size_t) -> *mut c_void { initialization_panic(); } @@ -283,6 +284,7 @@ unsafe fn alloc_prof_panic_realloc(_prev_ptr: *mut c_void, _len: size_t) -> *mut initialization_panic(); } +#[cfg(not(php_zend_mm_set_custom_handlers_ex))] unsafe fn alloc_prof_panic_free(_ptr: *mut c_void) { initialization_panic(); } From 4147a6bd12ccf11452e296aecb1b623db34cf803 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 27 Jul 2026 19:57:32 +0200 Subject: [PATCH 3/9] perf(profiling): optimize legacy allocator forwarding Apply the single-state allocation and free forwarding path to PHP 8.3 and older. Preserve the required ZendMM prepare/restore calls and neighboring custom allocator support while removing the intermediate function pointers and wrappers. On ZTS this also avoids repeated TSRM lookups. Across six balanced 60-second runs per binary on PHP 8.3 ZTS, mean allocation throughput increased from 28,079,883/s to 32,661,234/s (+16.32%) and median throughput increased by 16.65%. A full 60-second native sample reduced known forwarding-path self time from 16.31% to 12.30% (-24.6% relative). Validation: PHP 8.3 ZTS cargo test (22 passed); allocation sampling-distance, memory-peak, and GC PHPTs passed; PHP 8.5 release build and cargo test (22 passed). https://datadoghq.atlassian.net/browse/PROF-15506 --- profiling/src/allocation/allocation_le83.rs | 64 +++++++-------------- profiling/src/allocation/mod.rs | 10 ---- 2 files changed, 22 insertions(+), 52 deletions(-) diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 12a1b37c58..88f61262e7 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -36,20 +36,10 @@ pub struct ZendMMState { prev_custom_mm_free: Option, prepare_restore_zend_heap: (ZendHeapPrepareFn, ZendHeapRestoreFn), /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_alloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_alloc` is initialised to a valid function - /// pointer, otherwise there will be dragons. - alloc: unsafe fn(size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to /// `alloc_prof_prev_realloc()` when at the same time the /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid /// function pointer, otherwise there will be dragons. realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_free()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_free` is initialised to a valid function - /// pointer, otherwise there will be dragons. - free: unsafe fn(*mut c_void), } impl ZendMMState { @@ -61,9 +51,7 @@ impl ZendMMState { prev_custom_mm_realloc: None, prev_custom_mm_free: None, prepare_restore_zend_heap: (prepare_zend_heap, restore_zend_heap), - alloc: super::alloc_prof_panic_alloc, realloc: super::alloc_prof_panic_realloc, - free: super::alloc_prof_panic_free, } } } @@ -108,14 +96,10 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_realloc), ); } - zend_mm_state.alloc = alloc_prof_prev_alloc; - zend_mm_state.free = alloc_prof_prev_free; zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap_none, restore_zend_heap_none); } else { - zend_mm_state.alloc = alloc_prof_orig_alloc; - zend_mm_state.free = alloc_prof_orig_free; zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap, restore_zend_heap); @@ -297,7 +281,7 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(alloc)(len); + let ptr = alloc_prof_forward_alloc(len); // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -312,19 +296,17 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { ptr } -unsafe fn alloc_prof_prev_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_alloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_alloc` is also initialised - let alloc = tls_zend_mm_state_get!(prev_custom_mm_alloc).unwrap(); - alloc(len) -} +#[inline(always)] +unsafe fn alloc_prof_forward_alloc(len: size_t) -> *mut c_void { + let state = tls_zend_mm_state_copy!(); + if let Some(alloc) = state.prev_custom_mm_alloc { + return alloc(len); + } -unsafe fn alloc_prof_orig_alloc(len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - let (prepare, restore) = tls_zend_mm_state_get!(prepare_restore_zend_heap); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; let custom_heap = prepare(heap); #[cfg(php_debug)] let ptr: *mut c_void = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); @@ -344,7 +326,7 @@ unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { untrack_allocation(ptr); } - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_forward_free(ptr); } fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { @@ -356,21 +338,19 @@ fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { } unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - tls_zend_mm_state_get!(free)(ptr); + alloc_prof_forward_free(ptr); } -unsafe fn alloc_prof_prev_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.prev_custom_mm_free` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_free` is also initialised - let free = tls_zend_mm_state_get!(prev_custom_mm_free).unwrap(); - free(ptr) -} +#[inline(always)] +unsafe fn alloc_prof_forward_free(ptr: *mut c_void) { + let state = tls_zend_mm_state_copy!(); + if let Some(free) = state.prev_custom_mm_free { + return free(ptr); + } -unsafe fn alloc_prof_orig_free(ptr: *mut c_void) { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); #[cfg(php_debug)] zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); #[cfg(not(php_debug))] diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index febaf12f79..32124f64c9 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -275,16 +275,6 @@ fn initialization_panic() -> ! { panic!("Allocation profiler was not initialized properly. Please fill an issue stating the PHP version and the backtrace from this panic."); } -#[cfg(not(php_zend_mm_set_custom_handlers_ex))] -unsafe fn alloc_prof_panic_alloc(_len: size_t) -> *mut c_void { - initialization_panic(); -} - unsafe fn alloc_prof_panic_realloc(_prev_ptr: *mut c_void, _len: size_t) -> *mut c_void { initialization_panic(); } - -#[cfg(not(php_zend_mm_set_custom_handlers_ex))] -unsafe fn alloc_prof_panic_free(_ptr: *mut c_void) { - initialization_panic(); -} From a4aafc9ca7afabc779af64e8243edfae94010f6a Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Mon, 27 Jul 2026 20:42:32 +0200 Subject: [PATCH 4/9] test(profiling): stub ZendMM free for unit tests The free-handler selection test retains allocation callbacks that now call _zend_mm_free directly, but Rust unit-test binaries are not loaded by PHP. Provide a test-only stub with release and debug PHP signatures so both ZendMM API implementations link. Verified with cargo test on PHP 8.5 NTS and PHP 8.3 ZTS; both pass 22 tests. --- profiling/src/allocation/mod.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 32124f64c9..94e4b8eb0c 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -80,6 +80,23 @@ pub mod allocation_ge84; #[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub mod allocation_le83; +// Handler-selection tests retain the free callbacks in a binary that is not loaded by PHP. +#[cfg(all(test, not(php_debug)))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_free(_heap: *mut zend::_zend_mm_heap, _ptr: *mut c_void) {} + +#[cfg(all(test, php_debug))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_free( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _file: *const libc::c_char, + _line: libc::c_uint, + _orig_file: *const libc::c_char, + _orig_line: libc::c_uint, +) { +} + /// Default sampling interval in bytes (4 MiB). pub const DEFAULT_ALLOCATION_SAMPLING_INTERVAL: NonZeroU32 = NonZero::new(1024 * 4096).unwrap(); From a30b72ddd777b428b800f33a81dbe9dae6540007 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 28 Jul 2026 07:37:41 +0200 Subject: [PATCH 5/9] perf(profiling): remove indirect realloc forwarding Forward reallocations from one ZendMMState read instead of loading a function pointer, making an indirect Rust call, and reloading state. Preserve neighboring custom allocator support and the PHP 8.3-and-older heap prepare/restore calls. Remove the realloc forwarding field, intermediate wrappers, and initialization panic callback from both ZendMM implementations. Validation: PHP 8.5 NTS and PHP 8.3 ZTS cargo test (22 passed each); five allocation PHPTs passed on both builds. https://datadoghq.atlassian.net/browse/PROF-15506 --- profiling/src/allocation/allocation_ge84.rs | 42 +++++++-------------- profiling/src/allocation/allocation_le83.rs | 34 ++++++----------- profiling/src/allocation/mod.rs | 5 +-- 3 files changed, 26 insertions(+), 55 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index f19ed81d88..be7273559a 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -34,11 +34,6 @@ pub struct ZendMMState { /// The engine's previous custom shutdown function, if there is one. prev_custom_mm_shutdown: Option, /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_realloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid - /// function pointer, otherwise there will be dragons. - realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, - /// Safety: this function pointer is only allowed to point to /// `alloc_prof_prev_gc()` when at the same time the /// `ZEND_MM_STATE.prev_custom_mm_gc` is initialised to a valid function /// pointer, otherwise there will be dragons. @@ -68,7 +63,6 @@ impl ZendMMState { prev_custom_mm_free: None, prev_custom_mm_gc: None, prev_custom_mm_shutdown: None, - realloc: super::alloc_prof_panic_realloc, gc: alloc_prof_panic_gc, shutdown: alloc_prof_panic_shutdown, } @@ -115,11 +109,9 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_shutdown), ); } - zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.gc = alloc_prof_prev_gc; zend_mm_state.shutdown = alloc_prof_prev_shutdown; } else { - zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.gc = alloc_prof_orig_gc; zend_mm_state.shutdown = alloc_prof_orig_shutdown; @@ -445,7 +437,7 @@ unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_ #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let ptr = alloc_prof_forward_realloc(prev_ptr, len); // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it @@ -465,7 +457,7 @@ unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let ptr = alloc_prof_forward_realloc(prev_ptr, len); alloc_prof_realloc_sample(ptr, len) } @@ -489,27 +481,19 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_realloc` is also initialised. - // Note: We use `.unwrap()` instead of `.unwrap_unchecked()` here because a - // neighboring extension could misbehave. If that happens, we want a proper - // panic with backtrace for debugging rather than undefined behavior. - let realloc = tls_zend_mm_state_get!(prev_custom_mm_realloc).unwrap(); - #[cfg(php_debug)] - { - realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0) +#[inline(always)] +unsafe fn alloc_prof_forward_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + let state = tls_zend_mm_state_copy!(); + if let Some(realloc) = state.prev_custom_mm_realloc { + #[cfg(php_debug)] + return realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + return realloc(prev_ptr, len); } - #[cfg(not(php_debug))] - realloc(prev_ptr, len) -} -unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers only point to this function after successful init. Using `unwrap_unchecked()` is - // safe here as we have full control over ZendMM with no neighboring extensions. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); #[cfg(php_debug)] return zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); #[cfg(not(php_debug))] diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 88f61262e7..7a8b2fa430 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -35,11 +35,6 @@ pub struct ZendMMState { /// The engine's previous custom free function, if there is one. prev_custom_mm_free: Option, prepare_restore_zend_heap: (ZendHeapPrepareFn, ZendHeapRestoreFn), - /// Safety: this function pointer is only allowed to point to - /// `alloc_prof_prev_realloc()` when at the same time the - /// `ZEND_MM_STATE.prev_custom_mm_realloc` is initialised to a valid - /// function pointer, otherwise there will be dragons. - realloc: unsafe fn(*mut c_void, size_t) -> *mut c_void, } impl ZendMMState { @@ -51,7 +46,6 @@ impl ZendMMState { prev_custom_mm_realloc: None, prev_custom_mm_free: None, prepare_restore_zend_heap: (prepare_zend_heap, restore_zend_heap), - realloc: super::alloc_prof_panic_realloc, } } } @@ -96,11 +90,9 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { ptr::addr_of_mut!(zend_mm_state.prev_custom_mm_realloc), ); } - zend_mm_state.realloc = alloc_prof_prev_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap_none, restore_zend_heap_none); } else { - zend_mm_state.realloc = alloc_prof_orig_realloc; zend_mm_state.prepare_restore_zend_heap = (prepare_zend_heap, restore_zend_heap); // Reset previous handlers to None. There might be a chaotic neighbor that @@ -383,7 +375,7 @@ unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_ #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let ptr = alloc_prof_forward_realloc(prev_ptr, len); // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it @@ -403,7 +395,7 @@ unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = tls_zend_mm_state_get!(realloc)(prev_ptr, len); + let ptr = alloc_prof_forward_realloc(prev_ptr, len); alloc_prof_realloc_sample(ptr, len) } @@ -427,19 +419,17 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -unsafe fn alloc_prof_prev_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.prev_custom_mm_realloc` will be initialised in - // `alloc_prof_rinit()` and only point to this function when - // `prev_custom_mm_realloc` is also initialised - let realloc = tls_zend_mm_state_get!(prev_custom_mm_realloc).unwrap(); - realloc(prev_ptr, len) -} +#[inline(always)] +unsafe fn alloc_prof_forward_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { + let state = tls_zend_mm_state_copy!(); + if let Some(realloc) = state.prev_custom_mm_realloc { + return realloc(prev_ptr, len); + } -unsafe fn alloc_prof_orig_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - // Safety: `ZEND_MM_STATE.heap` will be initialised in `alloc_prof_rinit()` and custom ZendMM - // handlers are only installed and pointing to this function if initialization was succesful. - let heap = tls_zend_mm_state_get!(heap).unwrap_unchecked(); - let (prepare, restore) = tls_zend_mm_state_get!(prepare_restore_zend_heap); + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; let custom_heap = prepare(heap); #[cfg(php_debug)] let ptr: *mut c_void = diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 94e4b8eb0c..30241604c2 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -287,11 +287,8 @@ pub fn alloc_prof_rshutdown() { allocation_ge84::alloc_prof_rshutdown(heap_live_enabled); } +#[cfg(php_zend_mm_set_custom_handlers_ex)] #[track_caller] fn initialization_panic() -> ! { panic!("Allocation profiler was not initialized properly. Please fill an issue stating the PHP version and the backtrace from this panic."); } - -unsafe fn alloc_prof_panic_realloc(_prev_ptr: *mut c_void, _len: size_t) -> *mut c_void { - initialization_panic(); -} From 4aaeba6e9dc5e2f52cec94e76fde05144116ef84 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 28 Jul 2026 13:49:41 +0200 Subject: [PATCH 6/9] perf(profiling): select legacy allocator callback at rinit On PHP 8.3 and older, select the direct ZendMM or neighboring custom allocator callback once during rinit. The normal allocation callback no longer loads or branches on prev_custom_mm_alloc for every allocation; the custom allocator callback remains as a documented cold compatibility path. Across six balanced 60-second PHP 8.3 ZTS runs per binary, mean throughput increased from 31,167,496/s to 32,646,366/s (+4.74%) and median throughput increased by 4.56%. Native samples reduced alloc_prof_malloc self time from 6.13% to 3.54% (-42.3% relative). The equivalent PHP 8.5 NTS experiment measured within system noise (+0.79% mean / +0.39% median) and was not retained. Validation: PHP 8.5 NTS cargo test (22 passed), PHP 8.3 ZTS and PHP 7.3 ZTS cargo check, and five PHP 8.3 allocation PHPTs. https://datadoghq.atlassian.net/browse/PROF-15506 --- profiling/src/allocation/allocation_ge84.rs | 1 + profiling/src/allocation/allocation_le83.rs | 64 +++++++++++++-------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index be7273559a..024b9c8b0a 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -297,6 +297,7 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[inline(always)] unsafe fn alloc_prof_forward_alloc(len: size_t) -> *mut c_void { let state = tls_zend_mm_state_copy!(); + // Compatibility path for another extension's previously installed custom allocator. if let Some(alloc) = state.prev_custom_mm_alloc { #[cfg(php_debug)] return alloc(len, ptr::null(), 0, ptr::null(), 0); diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index 7a8b2fa430..f784cd027d 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -104,6 +104,8 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { zend_mm_state.prev_custom_mm_realloc = None; } + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); let free_handler = alloc_prof_free_handler(heap_live_enabled); let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); @@ -111,7 +113,7 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { unsafe { zend::ddog_php_prof_zend_mm_set_custom_handlers( heap, - Some(alloc_prof_malloc), + Some(malloc_handler), Some(free_handler), Some(realloc_handler), ); @@ -161,10 +163,12 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { &mut custom_mm_realloc, ); } + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); let free_handler = alloc_prof_free_handler(heap_live_enabled); let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); if custom_mm_free != Some(free_handler) - || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) { // Custom handlers are installed, but it's not us. Someone, somewhere might have @@ -267,13 +271,47 @@ unsafe extern "C" fn alloc_prof_gc_mem_caches( } } +fn alloc_prof_malloc_handler(has_previous_allocator: bool) -> zend::VmMmCustomAllocFn { + if has_previous_allocator { + alloc_prof_malloc_custom + } else { + alloc_prof_malloc + } +} + unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_malloc_custom(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +#[inline(always)] +unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = alloc_prof_forward_alloc(len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + state.prev_custom_mm_alloc.unwrap()(len) + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; + let custom_heap = prepare(heap); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_alloc(heap, len); + restore(heap, custom_heap); + ptr + }; // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -288,26 +326,6 @@ unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { ptr } -#[inline(always)] -unsafe fn alloc_prof_forward_alloc(len: size_t) -> *mut c_void { - let state = tls_zend_mm_state_copy!(); - if let Some(alloc) = state.prev_custom_mm_alloc { - return alloc(len); - } - - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - let (prepare, restore) = state.prepare_restore_zend_heap; - let custom_heap = prepare(heap); - #[cfg(php_debug)] - let ptr: *mut c_void = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - let ptr: *mut c_void = zend::_zend_mm_alloc(heap, len); - restore(heap, custom_heap); - ptr -} - /// This function exists because when calling `zend_mm_set_custom_handlers()`, /// you need to pass a pointer to a `free()` function as well, otherwise your /// custom handlers won't be installed. We cannot just point to the original From e85b2a55fde27e3714dc8508e30caf50c41dbcc6 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 28 Jul 2026 14:17:22 +0200 Subject: [PATCH 7/9] perf(profiling): select modern allocator callback at rinit Apply the same RINIT-selected allocation callback used for PHP 8.3 and older to PHP 8.4 and newer. The normal callback has no previous-allocator load or branch, while a separate cold callback preserves neighboring custom allocator support. The PHP 8.5 NTS benchmark was inconclusive but non-negative: a clean six-run retry measured +0.40% mean and +0.69% median with about 1.8% run variance. Keep the implementation for symmetry across ZendMM APIs rather than as a claimed performance win. Validation: PHP 8.5 NTS cargo test (22 passed) and profiler-release build. https://datadoghq.atlassian.net/browse/PROF-15506 --- profiling/src/allocation/allocation_ge84.rs | 80 ++++++++++++++------- 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index 024b9c8b0a..d309711983 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -126,6 +126,8 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { zend_mm_state.prev_custom_mm_shutdown = None; } + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); let free_handler = alloc_prof_free_handler(heap_live_enabled); let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); @@ -133,7 +135,7 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { unsafe { zend::zend_mm_set_custom_handlers_ex( heap, - Some(alloc_prof_malloc), + Some(malloc_handler), Some(free_handler), Some(realloc_handler), Some(alloc_prof_gc), @@ -189,10 +191,12 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { &mut custom_mm_shutdown, ); } + let malloc_handler = + alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); let free_handler = alloc_prof_free_handler(heap_live_enabled); let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); if custom_mm_free != Some(free_handler) - || custom_mm_malloc != Some(alloc_prof_malloc) + || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) || custom_mm_gc != Some(alloc_prof_gc) || custom_mm_shutdown != Some(alloc_prof_shutdown) @@ -256,9 +260,17 @@ unsafe fn restore_zend_heap(heap: *mut zend::_zend_mm_heap, custom_heap: c_int) ptr::write(heap as *mut c_int, custom_heap); } +fn alloc_prof_malloc_handler(has_previous_allocator: bool) -> zend::VmMmCustomAllocFn { + if has_previous_allocator { + alloc_prof_malloc_custom + } else { + alloc_prof_malloc + } +} + #[cfg(not(php_debug))] unsafe extern "C" fn alloc_prof_malloc(len: size_t) -> *mut c_void { - alloc_prof_malloc_impl(len) + alloc_prof_malloc_impl::(len) } #[cfg(php_debug)] @@ -269,17 +281,53 @@ unsafe extern "C" fn alloc_prof_malloc( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_malloc_impl(len) + alloc_prof_malloc_impl::(len) +} + +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +#[cfg(not(php_debug))] +unsafe extern "C" fn alloc_prof_malloc_custom(len: size_t) -> *mut c_void { + alloc_prof_malloc_impl::(len) +} + +#[cold] +#[cfg(php_debug)] +unsafe extern "C" fn alloc_prof_malloc_custom( + len: size_t, + _file: *const c_char, + _line: c_uint, + _orig_file: *const c_char, + _orig_line: c_uint, +) -> *mut c_void { + alloc_prof_malloc_impl::(len) } #[inline(always)] -unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { +unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = alloc_prof_forward_alloc(len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + let alloc = state.prev_custom_mm_alloc.unwrap(); + #[cfg(php_debug)] + let ptr = alloc(len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = alloc(len); + ptr + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_alloc(heap, len); + ptr + }; // during startup, minit, rinit, ... current_execute_data is null // we are only interested in allocations during userland operations @@ -294,26 +342,6 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void { ptr } -#[inline(always)] -unsafe fn alloc_prof_forward_alloc(len: size_t) -> *mut c_void { - let state = tls_zend_mm_state_copy!(); - // Compatibility path for another extension's previously installed custom allocator. - if let Some(alloc) = state.prev_custom_mm_alloc { - #[cfg(php_debug)] - return alloc(len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - return alloc(len); - } - - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_alloc(heap, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_alloc(heap, len) -} - /// This function exists because when calling `zend_mm_set_custom_handlers()`, /// you need to pass a pointer to a `free()` function as well, otherwise your /// custom handlers won't be installed. We cannot just point to the original From ae4de114677b21fadd402370fdbedb3aca37ef32 Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 28 Jul 2026 17:24:30 +0200 Subject: [PATCH 8/9] perf(profiling): select free and realloc callbacks at rinit --- profiling/src/allocation/allocation_ge84.rs | 219 ++++++++++++-------- profiling/src/allocation/allocation_le83.rs | 196 +++++++++++------- 2 files changed, 245 insertions(+), 170 deletions(-) diff --git a/profiling/src/allocation/allocation_ge84.rs b/profiling/src/allocation/allocation_ge84.rs index d309711983..9ffd83ecd5 100644 --- a/profiling/src/allocation/allocation_ge84.rs +++ b/profiling/src/allocation/allocation_ge84.rs @@ -128,8 +128,14 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { let malloc_handler = alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); // install our custom handler to ZendMM unsafe { @@ -193,8 +199,14 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { } let malloc_handler = alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); if custom_mm_free != Some(free_handler) || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) @@ -347,88 +359,98 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void /// custom handlers won't be installed. We cannot just point to the original /// `zend::_zend_mm_free()` as the function definitions differ. #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { - alloc_prof_free_impl(ptr); +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr); } #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_free( +unsafe extern "C" fn alloc_prof_free( ptr: *mut c_void, _file: *const c_char, _line: c_uint, _orig_file: *const c_char, _orig_line: c_uint, ) { - alloc_prof_free_impl(ptr); -} - -fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { - if heap_live_enabled { - alloc_prof_free - } else { - alloc_prof_free_noop - } + alloc_prof_free_impl::(ptr); } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - alloc_prof_forward_free(ptr); +unsafe extern "C" fn alloc_prof_free_custom(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr); } +#[cold] #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_free_noop( +unsafe extern "C" fn alloc_prof_free_custom( ptr: *mut c_void, _file: *const c_char, _line: c_uint, _orig_file: *const c_char, _orig_line: c_uint, ) { - alloc_prof_forward_free(ptr); + alloc_prof_free_impl::(ptr); +} + +fn alloc_prof_free_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomFreeFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_free::, + (false, false) => alloc_prof_free::, + (true, true) => alloc_prof_free_custom::, + (false, true) => alloc_prof_free_custom::, + } } #[inline(always)] -unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { - // Heap-live is enabled when this handler is registered. - if !ptr.is_null() { +unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { + if TRACK && !ptr.is_null() { untrack_allocation(ptr); } - alloc_prof_forward_free(ptr); -} -#[inline(always)] -unsafe fn alloc_prof_forward_free(ptr: *mut c_void) { let state = tls_zend_mm_state_copy!(); - if let Some(free) = state.prev_custom_mm_free { + if CUSTOM { + let free = state.prev_custom_mm_free.unwrap(); + #[cfg(php_debug)] + free(ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + free(ptr); + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); #[cfg(php_debug)] - return free(ptr, core::ptr::null(), 0, core::ptr::null(), 0); + zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); #[cfg(not(php_debug))] - return free(ptr); + zend::_zend_mm_free(heap, ptr); } - - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_free(heap, ptr); } -fn alloc_prof_realloc_handler(heap_live_enabled: bool) -> zend::VmMmCustomReallocFn { - if heap_live_enabled { - alloc_prof_realloc - } else { - alloc_prof_realloc_no_untrack +fn alloc_prof_realloc_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomReallocFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_realloc::, + (false, false) => alloc_prof_realloc::, + (true, true) => alloc_prof_realloc_custom::, + (false, true) => alloc_prof_realloc_custom::, } } #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) +unsafe extern "C" fn alloc_prof_realloc( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { + alloc_prof_realloc_impl::(prev_ptr, len) } #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_realloc( +unsafe extern "C" fn alloc_prof_realloc( prev_ptr: *mut c_void, len: size_t, _file: *const c_char, @@ -436,19 +458,22 @@ unsafe extern "C" fn alloc_prof_realloc( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] #[cfg(not(php_debug))] -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } +#[cold] #[cfg(php_debug)] -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, _file: *const c_char, @@ -456,41 +481,49 @@ unsafe extern "C" fn alloc_prof_realloc_no_untrack( _orig_file: *const c_char, _orig_line: c_uint, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } #[inline(always)] -unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { +unsafe fn alloc_prof_realloc_impl( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = alloc_prof_forward_realloc(prev_ptr, len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + let realloc = state.prev_custom_mm_realloc.unwrap(); + #[cfg(php_debug)] + let ptr = realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = realloc(prev_ptr, len); + ptr + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len); + ptr + }; // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it // before any userland-only early return, then let the new allocation be // re-sampled at the reported size. - if !prev_ptr.is_null() { + if UNTRACK && !prev_ptr.is_null() { untrack_allocation(prev_ptr); } alloc_prof_realloc_sample(ptr, len) } -#[inline(always)] -unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - - let ptr = alloc_prof_forward_realloc(prev_ptr, len); - - alloc_prof_realloc_sample(ptr, len) -} - #[inline(always)] unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null @@ -510,25 +543,6 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -#[inline(always)] -unsafe fn alloc_prof_forward_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - let state = tls_zend_mm_state_copy!(); - if let Some(realloc) = state.prev_custom_mm_realloc { - #[cfg(php_debug)] - return realloc(prev_ptr, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - return realloc(prev_ptr, len); - } - - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - #[cfg(php_debug)] - return zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_realloc(heap, prev_ptr, len) -} - unsafe extern "C" fn alloc_prof_gc() -> size_t { tls_zend_mm_state_get!(gc)() } @@ -576,14 +590,39 @@ mod tests { use super::*; #[test] - fn free_handler_tracks_only_when_heap_live_is_enabled() { + fn handlers_are_selected_at_rinit() { + assert_eq!( + alloc_prof_free_handler(true, false) as usize, + alloc_prof_free:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(false, false) as usize, + alloc_prof_free:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(true, true) as usize, + alloc_prof_free_custom:: as zend::VmMmCustomFreeFn as usize + ); + assert_eq!( + alloc_prof_free_handler(false, true) as usize, + alloc_prof_free_custom:: as zend::VmMmCustomFreeFn as usize + ); + + assert_eq!( + alloc_prof_realloc_handler(true, false) as usize, + alloc_prof_realloc:: as zend::VmMmCustomReallocFn as usize + ); + assert_eq!( + alloc_prof_realloc_handler(false, false) as usize, + alloc_prof_realloc:: as zend::VmMmCustomReallocFn as usize + ); assert_eq!( - alloc_prof_free_handler(true) as usize, - alloc_prof_free as zend::VmMmCustomFreeFn as usize + alloc_prof_realloc_handler(true, true) as usize, + alloc_prof_realloc_custom:: as zend::VmMmCustomReallocFn as usize ); assert_eq!( - alloc_prof_free_handler(false) as usize, - alloc_prof_free_noop as zend::VmMmCustomFreeFn as usize + alloc_prof_realloc_handler(false, true) as usize, + alloc_prof_realloc_custom:: as zend::VmMmCustomReallocFn as usize ); } diff --git a/profiling/src/allocation/allocation_le83.rs b/profiling/src/allocation/allocation_le83.rs index f784cd027d..e10b83b1c8 100644 --- a/profiling/src/allocation/allocation_le83.rs +++ b/profiling/src/allocation/allocation_le83.rs @@ -106,8 +106,14 @@ pub fn alloc_prof_rinit(heap_live_enabled: bool) { let malloc_handler = alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); // install our custom handler to ZendMM unsafe { @@ -165,8 +171,14 @@ pub fn alloc_prof_rshutdown(heap_live_enabled: bool) { } let malloc_handler = alloc_prof_malloc_handler(zend_mm_state.prev_custom_mm_alloc.is_some()); - let free_handler = alloc_prof_free_handler(heap_live_enabled); - let realloc_handler = alloc_prof_realloc_handler(heap_live_enabled); + let free_handler = alloc_prof_free_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_free.is_some(), + ); + let realloc_handler = alloc_prof_realloc_handler( + heap_live_enabled, + zend_mm_state.prev_custom_mm_realloc.is_some(), + ); if custom_mm_free != Some(free_handler) || custom_mm_malloc != Some(malloc_handler) || custom_mm_realloc != Some(realloc_handler) @@ -330,94 +342,114 @@ unsafe fn alloc_prof_malloc_impl(len: size_t) -> *mut c_void /// you need to pass a pointer to a `free()` function as well, otherwise your /// custom handlers won't be installed. We cannot just point to the original /// `zend::_zend_mm_free()` as the function definitions differ. -unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { - // Heap-live is enabled when this handler is registered. - if !ptr.is_null() { - untrack_allocation(ptr); - } - - alloc_prof_forward_free(ptr); +unsafe extern "C" fn alloc_prof_free(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr) } -fn alloc_prof_free_handler(heap_live_enabled: bool) -> zend::VmMmCustomFreeFn { - if heap_live_enabled { - alloc_prof_free - } else { - alloc_prof_free_noop - } +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_free_custom(ptr: *mut c_void) { + alloc_prof_free_impl::(ptr) } -unsafe extern "C" fn alloc_prof_free_noop(ptr: *mut c_void) { - alloc_prof_forward_free(ptr); +fn alloc_prof_free_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomFreeFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_free::, + (false, false) => alloc_prof_free::, + (true, true) => alloc_prof_free_custom::, + (false, true) => alloc_prof_free_custom::, + } } #[inline(always)] -unsafe fn alloc_prof_forward_free(ptr: *mut c_void) { - let state = tls_zend_mm_state_copy!(); - if let Some(free) = state.prev_custom_mm_free { - return free(ptr); +unsafe fn alloc_prof_free_impl(ptr: *mut c_void) { + if TRACK && !ptr.is_null() { + untrack_allocation(ptr); } - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - #[cfg(php_debug)] - zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); - #[cfg(not(php_debug))] - zend::_zend_mm_free(heap, ptr); + let state = tls_zend_mm_state_copy!(); + if CUSTOM { + state.prev_custom_mm_free.unwrap()(ptr); + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + #[cfg(php_debug)] + zend::_zend_mm_free(heap, ptr, core::ptr::null(), 0, core::ptr::null(), 0); + #[cfg(not(php_debug))] + zend::_zend_mm_free(heap, ptr); + } } -fn alloc_prof_realloc_handler(heap_live_enabled: bool) -> zend::VmMmCustomReallocFn { - if heap_live_enabled { - alloc_prof_realloc - } else { - alloc_prof_realloc_no_untrack +fn alloc_prof_realloc_handler( + heap_live_enabled: bool, + has_previous_allocator: bool, +) -> zend::VmMmCustomReallocFn { + match (heap_live_enabled, has_previous_allocator) { + (true, false) => alloc_prof_realloc::, + (false, false) => alloc_prof_realloc::, + (true, true) => alloc_prof_realloc_custom::, + (false, true) => alloc_prof_realloc_custom::, } } -unsafe extern "C" fn alloc_prof_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - alloc_prof_realloc_impl(prev_ptr, len) +unsafe extern "C" fn alloc_prof_realloc( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { + alloc_prof_realloc_impl::(prev_ptr, len) } -unsafe extern "C" fn alloc_prof_realloc_no_untrack( +// Compatibility path for another extension's previously installed custom allocator. +#[cold] +unsafe extern "C" fn alloc_prof_realloc_custom( prev_ptr: *mut c_void, len: size_t, ) -> *mut c_void { - alloc_prof_realloc_no_untrack_impl(prev_ptr, len) + alloc_prof_realloc_impl::(prev_ptr, len) } #[inline(always)] -unsafe fn alloc_prof_realloc_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { +unsafe fn alloc_prof_realloc_impl( + prev_ptr: *mut c_void, + len: size_t, +) -> *mut c_void { #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); #[cfg(feature = "debug_stats")] ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - let ptr = alloc_prof_forward_realloc(prev_ptr, len); + let state = tls_zend_mm_state_copy!(); + let ptr = if CUSTOM { + state.prev_custom_mm_realloc.unwrap()(prev_ptr, len) + } else { + // SAFETY: this callback is only invoked after rinit stores the heap and + // before rshutdown clears it. + let heap = state.heap.unwrap_unchecked(); + let (prepare, restore) = state.prepare_restore_zend_heap; + let custom_heap = prepare(heap); + #[cfg(php_debug)] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); + #[cfg(not(php_debug))] + let ptr = zend::_zend_mm_realloc(heap, prev_ptr, len); + restore(heap, custom_heap); + ptr + }; // ZendMM allocation failures raise a fatal error and bail out instead of // returning NULL. If realloc returns, prev_ptr has been consumed: untrack it // before any userland-only early return, then let the new allocation be // re-sampled at the reported size. - if !prev_ptr.is_null() { + if UNTRACK && !prev_ptr.is_null() { untrack_allocation(prev_ptr); } alloc_prof_realloc_sample(ptr, len) } -#[inline(always)] -unsafe fn alloc_prof_realloc_no_untrack_impl(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_COUNT.fetch_add(1, Relaxed); - #[cfg(feature = "debug_stats")] - ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed); - - let ptr = alloc_prof_forward_realloc(prev_ptr, len); - - alloc_prof_realloc_sample(ptr, len) -} - #[inline(always)] unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void { // during startup, minit, rinit, ... current_execute_data is null @@ -437,27 +469,6 @@ unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_voi ptr } -#[inline(always)] -unsafe fn alloc_prof_forward_realloc(prev_ptr: *mut c_void, len: size_t) -> *mut c_void { - let state = tls_zend_mm_state_copy!(); - if let Some(realloc) = state.prev_custom_mm_realloc { - return realloc(prev_ptr, len); - } - - // SAFETY: this callback is only invoked after rinit stores the heap and - // before rshutdown clears it. - let heap = state.heap.unwrap_unchecked(); - let (prepare, restore) = state.prepare_restore_zend_heap; - let custom_heap = prepare(heap); - #[cfg(php_debug)] - let ptr: *mut c_void = - zend::_zend_mm_realloc(heap, prev_ptr, len, ptr::null(), 0, ptr::null(), 0); - #[cfg(not(php_debug))] - let ptr: *mut c_void = zend::_zend_mm_realloc(heap, prev_ptr, len); - restore(heap, custom_heap); - ptr -} - /// safe wrapper for `zend::is_zend_mm()`. /// `true` means the internal ZendMM is being used, `false` means that a custom memory manager is /// installed. Upstream returns a `c_bool` as of PHP 8.0. PHP 7 returns a `c_int` @@ -477,14 +488,39 @@ mod tests { use super::*; #[test] - fn free_handler_tracks_only_when_heap_live_is_enabled() { + fn handlers_are_selected_at_rinit() { + assert_eq!( + alloc_prof_free_handler(true, false) as usize, + alloc_prof_free:: as usize + ); + assert_eq!( + alloc_prof_free_handler(false, false) as usize, + alloc_prof_free:: as usize + ); + assert_eq!( + alloc_prof_free_handler(true, true) as usize, + alloc_prof_free_custom:: as usize + ); + assert_eq!( + alloc_prof_free_handler(false, true) as usize, + alloc_prof_free_custom:: as usize + ); + + assert_eq!( + alloc_prof_realloc_handler(true, false) as usize, + alloc_prof_realloc:: as usize + ); + assert_eq!( + alloc_prof_realloc_handler(false, false) as usize, + alloc_prof_realloc:: as usize + ); assert_eq!( - alloc_prof_free_handler(true) as usize, - alloc_prof_free as usize + alloc_prof_realloc_handler(true, true) as usize, + alloc_prof_realloc_custom:: as usize ); assert_eq!( - alloc_prof_free_handler(false) as usize, - alloc_prof_free_noop as usize + alloc_prof_realloc_handler(false, true) as usize, + alloc_prof_realloc_custom:: as usize ); } From 010c2f717b67c705fd3aa69210bf7977adf0113f Mon Sep 17 00:00:00 2001 From: Florian Engelhardt Date: Tue, 28 Jul 2026 20:04:13 +0200 Subject: [PATCH 9/9] test(profiling): stub realloc callback dependencies --- profiling/src/allocation/mod.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/profiling/src/allocation/mod.rs b/profiling/src/allocation/mod.rs index 30241604c2..f366d6a1cb 100644 --- a/profiling/src/allocation/mod.rs +++ b/profiling/src/allocation/mod.rs @@ -80,7 +80,12 @@ pub mod allocation_ge84; #[cfg(not(php_zend_mm_set_custom_handlers_ex))] pub mod allocation_le83; -// Handler-selection tests retain the free callbacks in a binary that is not loaded by PHP. +// Handler-selection tests retain callbacks in a binary that is not loaded by PHP. +#[cfg(all(test, not(php_zts)))] +#[export_name = "executor_globals"] +static mut TEST_EXECUTOR_GLOBALS: core::mem::MaybeUninit = + core::mem::MaybeUninit::zeroed(); + #[cfg(all(test, not(php_debug)))] #[no_mangle] unsafe extern "C" fn _zend_mm_free(_heap: *mut zend::_zend_mm_heap, _ptr: *mut c_void) {} @@ -97,6 +102,30 @@ unsafe extern "C" fn _zend_mm_free( ) { } +#[cfg(all(test, not(php_debug)))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_realloc( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _len: size_t, +) -> *mut c_void { + ptr::null_mut() +} + +#[cfg(all(test, php_debug))] +#[no_mangle] +unsafe extern "C" fn _zend_mm_realloc( + _heap: *mut zend::_zend_mm_heap, + _ptr: *mut c_void, + _len: size_t, + _file: *const libc::c_char, + _line: libc::c_uint, + _orig_file: *const libc::c_char, + _orig_line: libc::c_uint, +) -> *mut c_void { + ptr::null_mut() +} + /// Default sampling interval in bytes (4 MiB). pub const DEFAULT_ALLOCATION_SAMPLING_INTERVAL: NonZeroU32 = NonZero::new(1024 * 4096).unwrap();