perf(profiling): reuse TSRM cache in allocation hook

profiling/build.rs

@@ -389,8 +389,11 @@ fn cfg_frameless(vernum: u64) -> bool {
 }
 
 fn cfg_php_feature_flags(vernum: u64) {
-    println!("cargo::rustc-check-cfg=cfg(php_gc_status, php_zend_compile_string_has_position, php_gc_status_extended, php_frameless, php_opcache_restart_hook, php_zend_mm_set_custom_handlers_ex)");
+    println!("cargo::rustc-check-cfg=cfg(php_gc_status, php_zend_compile_string_has_position, php_gc_status_extended, php_frameless, php_opcache_restart_hook, php_zend_mm_set_custom_handlers_ex, php_zts_fast_globals)");
 
+    if vernum >= 70400 {
+        println!("cargo:rustc-cfg=php_zts_fast_globals");
+    }
     if vernum >= 70300 {
         println!("cargo:rustc-cfg=php_gc_status");
     }
@@ -630,6 +633,7 @@ fn apple_linker_flags() {
         "_sapi_module",
         // TSRM (ZTS builds only; harmless to list on NTS — they simply
         // won't appear as undefined)
+        "_executor_globals_id",
         "_tsrm_get_ls_cache",
         "_tsrm_set_new_thread_end_handler",
         // ZTS globals offsets (replace direct globals on ZTS)

profiling/src/allocation/allocation_ge84.rs

@@ -1,15 +1,16 @@
-use crate::allocation::{
-    allocation_profiling_stats_should_collect, collect_allocation, current_execute_data,
-    untrack_allocation,
-};
+use crate::allocation::{collect_allocation, untrack_allocation};
 use crate::bindings as zend;
+use crate::module_globals::{self, ProfilerGlobals};
 use crate::PROFILER_NAME;
 use core::ptr;
 use libc::{c_char, c_int, c_void, size_t};
 use log::{debug, trace, warn};
 use std::sync::atomic::Ordering::Relaxed;
 use std::sync::LazyLock;
 
+#[cfg(php_zts)]
+use crate::allocation::current_execute_data_from_cache;
+
 #[cfg(php_debug)]
 use libc::c_uint;
 
@@ -322,7 +323,14 @@ unsafe fn alloc_prof_malloc_impl<const CUSTOM: bool>(len: size_t) -> *mut c_void
     #[cfg(feature = "debug_stats")]
     ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed);
 
-    let state = tls_zend_mm_state_copy!();
+    #[cfg(php_zts)]
+    let ls_cache = module_globals::get_tsrm_ls_cache();
+    #[cfg(php_zts)]
+    let globals = module_globals::get_profiler_globals_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let globals = module_globals::get_profiler_globals();
+    let state = (*globals).zend_mm_state.get();
+
     let ptr = if CUSTOM {
         let alloc = state.prev_custom_mm_alloc.unwrap();
         #[cfg(php_debug)]
@@ -343,12 +351,21 @@ unsafe fn alloc_prof_malloc_impl<const CUSTOM: bool>(len: size_t) -> *mut c_void
 
     // during startup, minit, rinit, ... current_execute_data is null
     // we are only interested in allocations during userland operations
-    if current_execute_data().is_null() {
+    #[cfg(php_zts)]
+    let execute_data = current_execute_data_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let execute_data = ptr::addr_of!(zend::executor_globals.current_execute_data).read();
+    if execute_data.is_null() {
         return ptr;
     }
 
-    if allocation_profiling_stats_should_collect(len) {
-        collect_allocation(ptr, len);
+    if ProfilerGlobals::should_collect(globals, len) {
+        collect_allocation(
+            unsafe { &(*globals).interrupt_count },
+            execute_data,
+            ptr,
+            len,
+        );
     }
 
     ptr
@@ -494,7 +511,14 @@ unsafe fn alloc_prof_realloc_impl<const UNTRACK: bool, const CUSTOM: bool>(
     #[cfg(feature = "debug_stats")]
     ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed);
 
-    let state = tls_zend_mm_state_copy!();
+    #[cfg(php_zts)]
+    let ls_cache = module_globals::get_tsrm_ls_cache();
+    #[cfg(php_zts)]
+    let globals = module_globals::get_profiler_globals_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let globals = module_globals::get_profiler_globals();
+    let state = (*globals).zend_mm_state.get();
+
     let ptr = if CUSTOM {
         let realloc = state.prev_custom_mm_realloc.unwrap();
         #[cfg(php_debug)]
@@ -521,23 +545,24 @@ unsafe fn alloc_prof_realloc_impl<const UNTRACK: bool, const CUSTOM: bool>(
         untrack_allocation(prev_ptr);
     }
 
-    alloc_prof_realloc_sample(ptr, len)
-}
+    #[cfg(php_zts)]
+    let execute_data = current_execute_data_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let execute_data = ptr::addr_of!(zend::executor_globals.current_execute_data).read();
 
-#[inline(always)]
-unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void {
     // during startup, minit, rinit, ... current_execute_data is null
     // we are only interested in allocations during userland operations
-    if current_execute_data().is_null() {
-        return ptr;
-    }
-
-    if ptr.is_null() {
+    if execute_data.is_null() || ptr.is_null() {
         return ptr;
     }
 
-    if allocation_profiling_stats_should_collect(len) {
-        collect_allocation(ptr, len);
+    if ProfilerGlobals::should_collect(globals, len) {
+        collect_allocation(
+            unsafe { &(*globals).interrupt_count },
+            execute_data,
+            ptr,
+            len,
+        );
     }
 
     ptr

profiling/src/allocation/allocation_le83.rs

@@ -1,18 +1,19 @@
-use crate::allocation::{
-    allocation_profiling_stats_should_collect, collect_allocation, current_execute_data,
-    untrack_allocation,
-};
+use crate::allocation::{collect_allocation, untrack_allocation};
 use crate::bindings::{
     self as zend, datadog_php_install_handler, datadog_php_zif_handler,
     ddog_php_prof_copy_long_into_zval,
 };
+use crate::module_globals::{self, ProfilerGlobals};
 use crate::{RefCellExt, PROFILER_NAME, REQUEST_LOCALS};
 use core::ptr;
 use libc::{c_char, c_int, c_void, size_t};
 use log::{debug, trace, warn};
 use std::sync::atomic::Ordering::Relaxed;
 use std::sync::LazyLock;
 
+#[cfg(php_zts)]
+use crate::allocation::current_execute_data_from_cache;
+
 #[cfg(feature = "debug_stats")]
 use crate::allocation::{ALLOCATION_PROFILING_COUNT, ALLOCATION_PROFILING_SIZE};
 
@@ -308,7 +309,14 @@ unsafe fn alloc_prof_malloc_impl<const CUSTOM: bool>(len: size_t) -> *mut c_void
     #[cfg(feature = "debug_stats")]
     ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed);
 
-    let state = tls_zend_mm_state_copy!();
+    #[cfg(php_zts)]
+    let ls_cache = module_globals::get_tsrm_ls_cache();
+    #[cfg(php_zts)]
+    let globals = module_globals::get_profiler_globals_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let globals = module_globals::get_profiler_globals();
+    let state = (*globals).zend_mm_state.get();
+
     let ptr = if CUSTOM {
         state.prev_custom_mm_alloc.unwrap()(len)
     } else {
@@ -327,12 +335,21 @@ unsafe fn alloc_prof_malloc_impl<const CUSTOM: bool>(len: size_t) -> *mut c_void
 
     // during startup, minit, rinit, ... current_execute_data is null
     // we are only interested in allocations during userland operations
-    if current_execute_data().is_null() {
+    #[cfg(php_zts)]
+    let execute_data = current_execute_data_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let execute_data = ptr::addr_of!(zend::executor_globals.current_execute_data).read();
+    if execute_data.is_null() {
         return ptr;
     }
 
-    if allocation_profiling_stats_should_collect(len) {
-        collect_allocation(ptr, len);
+    if ProfilerGlobals::should_collect(globals, len) {
+        collect_allocation(
+            unsafe { &(*globals).interrupt_count },
+            execute_data,
+            ptr,
+            len,
+        );
     }
 
     ptr
@@ -422,7 +439,14 @@ unsafe fn alloc_prof_realloc_impl<const UNTRACK: bool, const CUSTOM: bool>(
     #[cfg(feature = "debug_stats")]
     ALLOCATION_PROFILING_SIZE.fetch_add(len as u64, Relaxed);
 
-    let state = tls_zend_mm_state_copy!();
+    #[cfg(php_zts)]
+    let ls_cache = module_globals::get_tsrm_ls_cache();
+    #[cfg(php_zts)]
+    let globals = module_globals::get_profiler_globals_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let globals = module_globals::get_profiler_globals();
+    let state = (*globals).zend_mm_state.get();
+
     let ptr = if CUSTOM {
         state.prev_custom_mm_realloc.unwrap()(prev_ptr, len)
     } else {
@@ -447,23 +471,24 @@ unsafe fn alloc_prof_realloc_impl<const UNTRACK: bool, const CUSTOM: bool>(
         untrack_allocation(prev_ptr);
     }
 
-    alloc_prof_realloc_sample(ptr, len)
-}
+    #[cfg(php_zts)]
+    let execute_data = current_execute_data_from_cache(ls_cache);
+    #[cfg(not(php_zts))]
+    let execute_data = ptr::addr_of!(zend::executor_globals.current_execute_data).read();
 
-#[inline(always)]
-unsafe fn alloc_prof_realloc_sample(ptr: *mut c_void, len: size_t) -> *mut c_void {
     // during startup, minit, rinit, ... current_execute_data is null
     // we are only interested in allocations during userland operations
-    if current_execute_data().is_null() {
-        return ptr;
-    }
-
-    if ptr.is_null() {
+    if execute_data.is_null() || ptr.is_null() {
         return ptr;
     }
 
-    if allocation_profiling_stats_should_collect(len) {
-        collect_allocation(ptr, len);
+    if ProfilerGlobals::should_collect(globals, len) {
+        collect_allocation(
+            unsafe { &(*globals).interrupt_count },
+            execute_data,
+            ptr,
+            len,
+        );
     }
 
     ptr

profiling/src/allocation/mod.rs

@@ -14,7 +14,7 @@ use log::{debug, trace};
 use rand_distr::{Distribution, Poisson};
 use std::ffi::c_void;
 use std::num::{NonZero, NonZeroU32, NonZeroU64};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
 
 #[cfg(not(php_zts))]
 use rand::rngs::StdRng;
@@ -39,13 +39,26 @@ pub(crate) unsafe fn get_zend_mm_state() -> *mut Cell<ZendMMState> {
     ptr::addr_of_mut!((*globals).zend_mm_state)
 }
 
+#[cfg(php_zts)]
 #[inline(always)]
-pub(crate) unsafe fn current_execute_data() -> *mut zend::zend_execute_data {
-    #[cfg(not(php_zts))]
-    return ptr::addr_of!(zend::executor_globals.current_execute_data).read();
-
-    #[cfg(php_zts)]
-    zend::ddog_php_prof_get_current_execute_data()
+pub(crate) unsafe fn current_execute_data_from_cache(
+    ls_cache: *mut c_void,
+) -> *mut zend::zend_execute_data {
+    // PHP 7.4 introduced fast globals offsets. Older versions use the TSRM resource ID.
+    #[cfg(php_zts_fast_globals)]
+    let globals = {
+        let offset = ptr::addr_of!(zend::executor_globals_offset).read();
+        ls_cache
+            .byte_add(offset)
+            .cast::<zend::zend_executor_globals>()
+    };
+    #[cfg(not(php_zts_fast_globals))]
+    let globals = {
+        let id = ptr::addr_of!(zend::executor_globals_id).read();
+        module_globals::get_tsrm_resource_from_cache(ls_cache, id)
+            .cast::<zend::zend_executor_globals>()
+    };
+    ptr::addr_of!((*globals).current_execute_data).read()
 }
 
 /// Macros for accessing ZendMMState from PHP globals.
@@ -146,7 +159,7 @@ pub static ALLOCATION_PROFILING_COUNT: AtomicU64 = AtomicU64::new(0);
 pub static ALLOCATION_PROFILING_SIZE: AtomicU64 = AtomicU64::new(0);
 
 pub struct AllocationProfilingStats {
-    /// number of bytes until next sample collection
+    /// Number of bytes remaining until the next sample collection.
     next_sample: i64,
     poisson: Poisson<f64>,
     #[cfg(php_zts)]
@@ -177,42 +190,46 @@ impl AllocationProfilingStats {
 
     fn should_collect_allocation(&mut self, len: size_t) -> bool {
         self.next_sample -= len as i64;
-
         if self.next_sample > 0 {
             return false;
         }
 
         self.next_sampling_interval();
-
         true
     }
 }
 
 /// Collect an allocation sample and optionally track it for live heap profiling.
 ///
+/// # Safety
+/// `execute_data` must be null or a valid pointer provided by the engine. The
+/// profiler may walk the execution frames reachable through it.
+///
 /// # Arguments
 /// * `ptr` - The pointer returned by the allocator (used for live heap tracking)
 /// * `len` - The size of the allocation in bytes
 #[cold]
-pub fn collect_allocation(ptr: *mut c_void, len: size_t) {
+pub unsafe fn collect_allocation(
+    interrupt_count: &AtomicU32,
+    execute_data: *mut zend::zend_execute_data,
+    ptr: *mut c_void,
+    len: size_t,
+) {
     if let Some(profiler) = Profiler::get() {
         // Check if there's a pending time interrupt that we can handle now
         // instead of waiting for an interrupt handler. This is slightly more
         // accurate and efficient, win-win.
-        // SAFETY: allocation samples are collected on an initialized PHP request thread.
-        let globals = unsafe { module_globals::get_profiler_globals() };
-        // SAFETY: the current thread's module globals are valid through GSHUTDOWN.
-        let interrupt_count = unsafe { (*globals).interrupt_count.swap(0, Ordering::Relaxed) };
+        let pending_interrupts = interrupt_count.swap(0, Ordering::Relaxed);
 
         // SAFETY: execute_data was provided by the engine, and the profiler
-        // doesn't mutate it.
+        // only reads the execution frames reachable through it.
         unsafe {
             profiler.collect_allocations(
-                zend::ddog_php_prof_get_current_execute_data(),
+                execute_data,
                 ptr,
                 1_i64,
                 len as i64,
-                (interrupt_count > 0).then_some(interrupt_count),
+                (pending_interrupts > 0).then_some(pending_interrupts),
             )
         };
     }