ZJIT: Add stats for cfuncs that are not optimized (ruby#14638)

* ZJIT: Add stats for cfuncs that are not optimized

* ZJIT: Add IncrCounterPtr HIR instead

From `lobsters`

```
Top-20 Unoptimized C functions (73.0% of total 15,276,688):
                               Kernel#is_a?: 2,052,363 (13.4%)
                              Class#current: 1,892,623 (12.4%)
                                String#to_s:   975,973 ( 6.4%)
                                  Hash#key?:   677,623 ( 4.4%)
                              String#empty?:   636,468 ( 4.2%)
                              TrueClass#===:   457,232 ( 3.0%)
                                   Hash#[]=:   455,908 ( 3.0%)
                             FalseClass#===:   448,798 ( 2.9%)
         ActiveSupport::OrderedOptions#_get:   377,468 ( 2.5%)
                            Kernel#kind_of?:   339,551 ( 2.2%)
                                 Kernel#dup:   329,371 ( 2.2%)
                                  String#==:   324,286 ( 2.1%)
                            String#include?:   297,528 ( 1.9%)
                                    Hash#[]:   294,561 ( 1.9%)
                             Array#include?:   287,145 ( 1.9%)
                        Kernel#block_given?:   283,633 ( 1.9%)
                             BasicObject#!=:   278,874 ( 1.8%)
                                Hash#delete:   250,951 ( 1.6%)
                               Set#include?:   246,447 ( 1.6%)
                               NilClass#===:   242,776 ( 1.6%)
```

From `liquid-render`

```
Top-20 Unoptimized C functions (99.8% of total 5,195,549):
                          Hash#key?: 2,459,048 (47.3%)
                        String#to_s: 1,119,758 (21.6%)
                       Set#include?:   799,469 (15.4%)
                       Kernel#is_a?:   214,223 ( 4.1%)
                         Integer#<<:   171,073 ( 3.3%)
                          Integer#/:   127,622 ( 2.5%)
          CGI::EscapeExt#escapeHTML:    56,971 ( 1.1%)
                         Regexp#===:    50,008 ( 1.0%)
                      String#empty?:    43,990 ( 0.8%)
                         String#===:    36,838 ( 0.7%)
                          String#==:    21,309 ( 0.4%)
                      Time#strftime:    21,251 ( 0.4%)
                       String#strip:    15,271 ( 0.3%)
                        String#scan:    13,753 ( 0.3%)
                          String#+@:    12,603 ( 0.2%)
                     Array#include?:     8,059 ( 0.2%)
                           String#+:     5,295 ( 0.1%)
                         String#dup:     4,606 ( 0.1%)
                          String#-@:     3,213 ( 0.1%)
                     Class#generate:     3,011 ( 0.1%)
```

Author: st0012

zjit.rb

@@ -43,6 +43,7 @@ def stats_string
     print_counters_with_prefix(prefix: 'dynamic_send_type_', prompt: 'dynamic send types', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'unspecialized_def_type_', prompt: 'send fallback unspecialized def_types', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'send_fallback_', prompt: 'dynamic send types', buf:, stats:, limit: 20)
+    print_counters_with_prefix(prefix: 'not_optimized_cfuncs_', prompt: 'unoptimized sends to C functions', buf:, stats:, limit: 20)
 
     # Show exit counters, ordered by the typical amount of exits for the prefix at the time
     print_counters_with_prefix(prefix: 'unhandled_yarv_insn_', prompt: 'unhandled YARV insns', buf:, stats:, limit: 20)

zjit/src/codegen.rs

@@ -419,6 +419,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
         Insn::GetSpecialSymbol { symbol_type, state: _ } => gen_getspecial_symbol(asm, *symbol_type),
         Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)),
         &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)),
+        Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)),
         Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)),
         &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))),
         &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) },
@@ -1528,15 +1529,20 @@ fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd,
     val
 }
 
+/// Generate code that records unoptimized C functions if --zjit-stats is enabled
+fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) {
+    if get_option!(stats) {
+        let ptr_reg = asm.load(Opnd::const_ptr(counter_ptr as *const u8));
+        let counter_opnd = Opnd::mem(64, ptr_reg, 0);
+        asm.incr_counter(counter_opnd, Opnd::UImm(1));
+    }
+}
+
 /// Generate code that increments a counter if --zjit-stats
 fn gen_incr_counter(asm: &mut Assembler, counter: Counter) {
     if get_option!(stats) {
         let ptr = counter_ptr(counter);
-        let ptr_reg = asm.load(Opnd::const_ptr(ptr as *const u8));
-        let counter_opnd = Opnd::mem(64, ptr_reg, 0);
-
-        // Increment and store the updated value
-        asm.incr_counter(counter_opnd, Opnd::UImm(1));
+        gen_incr_counter_ptr(asm, ptr);
     }
 }
 

zjit/src/hir.rs

@@ -711,6 +711,9 @@ pub enum Insn {
     /// Increment a counter in ZJIT stats
     IncrCounter(Counter),
 
+    /// Increment a counter in ZJIT stats for the given counter pointer
+    IncrCounterPtr { counter_ptr: *mut u64 },
+
     /// Equivalent of RUBY_VM_CHECK_INTS. Automatically inserted by the compiler before jumps and
     /// return instructions.
     CheckInterrupts { state: InsnId },
@@ -724,7 +727,7 @@ impl Insn {
             | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::EntryPoint { .. } | Insn::Return { .. }
             | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::ArrayExtend { .. }
             | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. }
-            | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_)
+            | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. }
             | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } => false,
             _ => true,
         }
@@ -978,6 +981,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> {
                 }
                 Ok(())
             },
+            Insn::IncrCounterPtr { .. } => write!(f, "IncrCounterPtr"),
             Insn::Snapshot { state } => write!(f, "Snapshot {}", state.print(self.ptr_map)),
             Insn::Defined { op_type, v, .. } => {
                 // op_type (enum defined_type) printing logic from iseq.c.
@@ -1385,6 +1389,7 @@ impl Function {
                     | EntryPoint {..}
                     | LoadPC
                     | LoadSelf
+                    | IncrCounterPtr {..}
                     | IncrCounter(_)) => result.clone(),
             &Snapshot { state: FrameState { iseq, insn_idx, pc, ref stack, ref locals } } =>
                 Snapshot {
@@ -1534,7 +1539,7 @@ impl Function {
             | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } | Insn::Throw { .. }
             | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::ArrayExtend { .. }
             | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } | Insn::IncrCounter(_)
-            | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } =>
+            | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::IncrCounterPtr { .. } =>
                 panic!("Cannot infer type of instruction with no output: {}", self.insns[insn.0]),
             Insn::Const { val: Const::Value(val) } => Type::from_value(*val),
             Insn::Const { val: Const::CBool(val) } => Type::from_cbool(*val),
@@ -2154,9 +2159,9 @@ impl Function {
             self_type: Type,
             send: Insn,
             send_insn_id: InsnId,
-        ) -> Result<(), ()> {
+        ) -> Result<(), Option<*const rb_callable_method_entry_struct>> {
             let Insn::SendWithoutBlock { mut recv, cd, mut args, state, .. } = send else {
-                return Err(());
+                return Err(None);
             };
 
             let call_info = unsafe { (*cd).ci };
@@ -2168,20 +2173,20 @@ impl Function {
                 (class, None)
             } else {
                 let iseq_insn_idx = fun.frame_state(state).insn_idx;
-                let Some(recv_type) = fun.profiled_type_of_at(recv, iseq_insn_idx) else { return Err(()) };
+                let Some(recv_type) = fun.profiled_type_of_at(recv, iseq_insn_idx) else { return Err(None) };
                 (recv_type.class(), Some(recv_type))
             };
 
             // Do method lookup
-            let method = unsafe { rb_callable_method_entry(recv_class, method_id) };
+            let method: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) };
             if method.is_null() {
-                return Err(());
+                return Err(None);
             }
 
             // Filter for C methods
             let def_type = unsafe { get_cme_def_type(method) };
             if def_type != VM_METHOD_TYPE_CFUNC {
-                return Err(());
+                return Err(None);
             }
 
             // Find the `argc` (arity) of the C method, which describes the parameters it expects
@@ -2193,15 +2198,15 @@ impl Function {
                     //
                     // Bail on argc mismatch
                     if argc != cfunc_argc as u32 {
-                        return Err(());
+                        return Err(Some(method));
                     }
 
                     // Filter for a leaf and GC free function
                     use crate::cruby_methods::FnProperties;
                     let Some(FnProperties { leaf: true, no_gc: true, return_type, elidable }) =
                         ZJITState::get_method_annotations().get_cfunc_properties(method)
                     else {
-                        return Err(());
+                        return Err(Some(method));
                     };
 
                     let ci_flags = unsafe { vm_ci_flag(call_info) };
@@ -2218,13 +2223,13 @@ impl Function {
                         cfunc_args.append(&mut args);
                         let ccall = fun.push_insn(block, Insn::CCall { cfun, args: cfunc_args, name: method_id, return_type, elidable });
                         fun.make_equal_to(send_insn_id, ccall);
-                        return Ok(());
+                        return Ok(())
                     }
                 }
                 // Variadic method
                 -1 => {
                     if unsafe { rb_zjit_method_tracing_currently_enabled() } {
-                        return Err(());
+                        return Err(None);
                     }
                     // The method gets a pointer to the first argument
                     // func(int argc, VALUE *argv, VALUE recv)
@@ -2256,8 +2261,9 @@ impl Function {
                         });
 
                         fun.make_equal_to(send_insn_id, ccall);
-                        return Ok(());
+                        return Ok(())
                     }
+
                     // Fall through for complex cases (splat, kwargs, etc.)
                 }
                 -2 => {
@@ -2267,7 +2273,7 @@ impl Function {
                 _ => unreachable!("unknown cfunc kind: argc={argc}")
             }
 
-            Err(())
+            Err(Some(method))
         }
 
         for block in self.rpo() {
@@ -2276,8 +2282,23 @@ impl Function {
             for insn_id in old_insns {
                 if let send @ Insn::SendWithoutBlock { recv, .. } = self.find(insn_id) {
                     let recv_type = self.type_of(recv);
-                    if reduce_to_ccall(self, block, recv_type, send, insn_id).is_ok() {
-                        continue;
+                    match reduce_to_ccall(self, block, recv_type, send, insn_id) {
+                        Ok(()) => continue,
+                        Err(Some(cme)) => {
+                            if get_option!(stats) {
+                                let owner = unsafe { (*cme).owner };
+                                let called_id = unsafe { (*cme).called_id };
+                                let class_name = get_class_name(owner);
+                                let method_name = called_id.contents_lossy();
+                                let qualified_method_name = format!("{}#{}", class_name, method_name);
+                                let unoptimized_cfunc_counter_pointers = ZJITState::get_unoptimized_cfunc_counter_pointers();
+                                let counter_ptr = unoptimized_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0));
+                                let counter_ptr = &mut **counter_ptr as *mut u64;
+
+                                self.push_insn(block, Insn::IncrCounterPtr { counter_ptr });
+                            }
+                        }
+                        _ => {}
                     }
                 }
                 self.push_insn_id(block, insn_id);
@@ -2423,7 +2444,8 @@ impl Function {
             | &Insn::LoadSelf
             | &Insn::GetLocal { .. }
             | &Insn::PutSpecialObject { .. }
-            | &Insn::IncrCounter(_) =>
+            | &Insn::IncrCounter(_)
+            | &Insn::IncrCounterPtr { .. } =>
                 {}
             &Insn::PatchPoint { state, .. }
             | &Insn::CheckInterrupts { state }

zjit/src/state.rs

@@ -8,6 +8,7 @@ use crate::asm::CodeBlock;
 use crate::options::get_option;
 use crate::stats::{Counters, ExitCounters};
 use crate::virtualmem::CodePtr;
+use std::collections::HashMap;
 
 #[allow(non_upper_case_globals)]
 #[unsafe(no_mangle)]
@@ -46,6 +47,9 @@ pub struct ZJITState {
 
     /// Trampoline to call function_stub_hit
     function_stub_hit_trampoline: CodePtr,
+
+    /// Counter pointers for unoptimized C functions
+    unoptimized_cfunc_counter_pointers: HashMap<String, Box<u64>>,
 }
 
 /// Private singleton instance of the codegen globals
@@ -80,6 +84,7 @@ impl ZJITState {
             exit_trampoline,
             function_stub_hit_trampoline,
             exit_trampoline_with_counter: exit_trampoline,
+            unoptimized_cfunc_counter_pointers: HashMap::new(),
         };
         unsafe { ZJIT_STATE = Some(zjit_state); }
 
@@ -138,6 +143,11 @@ impl ZJITState {
         &mut ZJITState::get_instance().exit_counters
     }
 
+    /// Get a mutable reference to unoptimized cfunc counter pointers
+    pub fn get_unoptimized_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> {
+        &mut ZJITState::get_instance().unoptimized_cfunc_counter_pointers
+    }
+
     /// Was --zjit-save-compiled-iseqs specified?
     pub fn should_log_compiled_iseqs() -> bool {
         get_option!(log_compiled_iseqs).is_some()

zjit/src/stats.rs

@@ -384,6 +384,13 @@ pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) ->
         set_stat_f64!(hash, "ratio_in_zjit", 100.0 * zjit_insn_count as f64 / total_insn_count as f64);
     }
 
+    // Set unoptimized cfunc counters
+    let unoptimized_cfuncs = ZJITState::get_unoptimized_cfunc_counter_pointers();
+    for (signature, counter) in unoptimized_cfuncs.iter() {
+        let key_string = format!("not_optimized_cfuncs_{}", signature);
+        set_stat_usize!(hash, &key_string, **counter);
+    }
+
     hash
 }