feat(reg-vm): native-call ABI slice 4 — mutual recursion end-to-end

Wires the vm-jit group-compilation core (91a0b41) into reg_vm so mutually-
recursive scalar-Int functions run natively from source. is_even/is_odd now
execute on Cranelift via co-compiled CallGroup, byte-identical to the interpreter.

- `native_recursive_int_group`: the call-graph SCC containing a function, if it is
  a cycle of >= 2 all-scalar-Int members whose every CallKnown targets a member.
  Refactored the self-recursion candidate into the group-aware
  `compute_recursive_int_member_inner` (self = a 1-member group).
- `translate_to_native_jit_with_calls` gains `group_call_sites` (ip -> group index):
  a CallKnown to a group member lowers to `JitInstr::CallGroup`; group members use
  re-run-from-top deopt (precise_resume_safe forced off).
- `try_native_mutual_recursive_int`: detect the group, translate each member with
  its group_call_sites, `compile_recursive_group` once, cache every member's
  CompiledId (NativeState.mutual_recursive_native), and dispatch the called member.
  Hooked in the exec.rs CallKnown handler after the self-recursive check; a deopt
  (incl. the entry depth-cap bail) falls back to the interpreter.

INT-only, like self-recursion: Bool-returning mutual recursion declines to native
(tco_leaves_mutual_recursion_untouched stays interpreter-only).

Verified: differential 33/0, jit_acceptance 87/0 (new
native_mutual_recursion_runs_native_and_matches_interpreter), vm-jit 83/0, default
workspace build clean.

Completes the native-call ABI (slices 1-4). Self-recursion perf: fib(32) 11.44x.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: olwang

crates/rsscript/src/reg_vm/exec.rs

@@ -1319,6 +1319,19 @@ impl RegVm {
                                 self.set_reg(base + *dst, value);
                                 continue;
                             }
+                            // Native mutual recursion (native-call-ABI slice 4): a
+                            // member of a mutually-recursive scalar-Int cycle runs via
+                            // the co-compiled native group; deep recursion falls back.
+                            #[cfg(feature = "native-jit")]
+                            if self.jit_enabled
+                                && mut_args.is_empty()
+                                && self.limits.mem_budget.is_none()
+                                && let Some(value) =
+                                    self.try_native_mutual_recursive_int(unit, *callee_id, base, args)
+                            {
+                                self.set_reg(base + *dst, value);
+                                continue;
+                            }
                             let callee = Rc::clone(&unit.functions[*callee_id]);
                             self.prepare_frame(next_base, callee.regs)?;
                             for (index, reg) in args.iter().enumerate() {

crates/rsscript/src/reg_vm/mod.rs

@@ -2183,6 +2183,11 @@ struct NativeState {
     /// (`*const RegFunction` key) compiled `CallSelf` entry. `None` = known not
     /// natively self-recursion-compilable (fall back to the tier-0 scalar executor).
     self_recursive_native: HashMap<usize, Option<vm_jit::CompiledId>>,
+    /// Native mutual-recursion cache (native-call-ABI slice 4): per-function
+    /// (`*const RegFunction` key) compiled group-member `CompiledId`. Compiling any
+    /// member of a recursive cycle compiles+caches the whole group. `None` = known
+    /// not a natively-compilable mutual-recursion member (fall back to interpreter).
+    mutual_recursive_native: HashMap<usize, Option<vm_jit::CompiledId>>,
     /// Reusable per-call marshalling scratch buffers (TV2 arg/len words and the
     /// flat-list `Rc` keep-alive set). Held here and `mem::take`n into the call
     /// frame so a hot per-iteration native dispatch (e.g. a tiny leaf/closure
@@ -5730,6 +5735,7 @@ impl NativeState {
             osr_enabled,
             osr_cache: HashMap::new(),
             self_recursive_native: HashMap::new(),
+            mutual_recursive_native: HashMap::new(),
             scratch_args: Vec::new(),
             scratch_lens: Vec::new(),
             scratch_flat_owned: Vec::new(),

crates/rsscript/src/reg_vm/native/translate.rs

@@ -168,20 +168,24 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_compiled_callees(
     Vec<Rc<String>>,
     bool,
 )> {
-    translate_to_native_jit_with_calls(unit, func, compiled_callees, &HashSet::new())
+    translate_to_native_jit_with_calls(unit, func, compiled_callees, &HashSet::new(), &HashMap::new())
 }
 
 /// Like [`translate_to_native_jit_with_compiled_callees`], but `self_call_sites`
 /// names the original ips of `CallKnown` instructions that call `func` itself —
 /// emitted as `JitInstr::CallSelf` for native self-recursion (native-call-ABI
-/// slice 3). Such functions use re-run-from-top deopt (`precise_resume_safe` forced
-/// off), so a bail anywhere in the recursion unwinds to the interpreter.
+/// slice 3) — and `group_call_sites` maps the original ip of a `CallKnown` to a
+/// *mutually-recursive group member* to that member's group index, emitted as
+/// `JitInstr::CallGroup` (slice 4). Such functions use re-run-from-top deopt
+/// (`precise_resume_safe` forced off), so a bail anywhere in the recursion unwinds
+/// to the interpreter.
 #[cfg(feature = "native-jit")]
 pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
     unit: &RegUnit,
     func: &RegFunction,
     compiled_callees: &HashMap<usize, NativeCompiledCallee>,
     self_call_sites: &HashSet<usize>,
+    group_call_sites: &HashMap<usize, u32>,
 ) -> Option<(
     vm_jit::JitFunction,
     NativeTy,
@@ -202,6 +206,7 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
         .keys()
         .copied()
         .chain(self_call_sites.iter().copied())
+        .chain(group_call_sites.keys().copied())
         .collect();
     let (code, n_regs, mut ip_map) = native_inline_leaf_calls_preserving_known_calls(
         unit,
@@ -255,7 +260,8 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
     for (i, instr) in code.iter().enumerate() {
         let compiled_call = matches!(instr, RegInstr::CallKnown { .. })
             && (compiled_callees.contains_key(&ip_map[i])
-                || self_call_sites.contains(&ip_map[i]));
+                || self_call_sites.contains(&ip_map[i])
+                || group_call_sites.contains_key(&ip_map[i]));
         if reachable[i] && !compiled_call && !native_subset_instruction(instr) {
             return None;
         }
@@ -338,6 +344,17 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
                     }
                     ok && native_set_ty(ty, *dst, NativeTy::Int, c)
                 }
+                RegInstr::CallKnown {
+                    dst, args, mut_args, ..
+                } if group_call_sites.contains_key(&ip_map[i]) => {
+                    // Mutually-recursive group call (native-call-ABI slice 4): the
+                    // mutual-recursive int group is all-scalar-Int, so args/result Int.
+                    let mut ok = mut_args.is_empty();
+                    for arg in args {
+                        ok = ok && native_set_ty(ty, *arg, NativeTy::Int, c);
+                    }
+                    ok && native_set_ty(ty, *dst, NativeTy::Int, c)
+                }
                 RegInstr::CallKnown {
                     dst,
                     args,
@@ -922,6 +939,26 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
                     args: args.iter().map(|arg| r(*arg)).collect(),
                 }
             }
+            RegInstr::CallKnown {
+                dst,
+                args,
+                mut_args,
+                ..
+            } if group_call_sites.contains_key(&ip_map[i]) => {
+                // Mutually-recursive native call (native-call-ABI slice 4): lower to
+                // `CallGroup` targeting the member's group index. Scalar params only.
+                let group_index = group_call_sites[&ip_map[i]];
+                require(mut_args.is_empty())?;
+                require(ty[*dst].is_some())?;
+                for arg in args {
+                    require(ty[*arg].is_some())?;
+                }
+                JitInstr::CallGroup {
+                    group_index,
+                    dst: r(*dst),
+                    args: args.iter().map(|arg| r(*arg)).collect(),
+                }
+            }
             RegInstr::CallKnown {
                 dst,
                 args,
@@ -1696,6 +1733,7 @@ pub(in crate::reg_vm) fn translate_to_native_jit_with_calls(
     // non-chaining and its native frame chain has no bounded deopt payload), so it is
     // never precise-resumable regardless of ip-map identity.
     let precise_resume_safe = self_call_sites.is_empty()
+        && group_call_sites.is_empty()
         && n_regs == func.regs
         && ip_map.iter().enumerate().all(|(ip, &orig)| ip == orig);
     Some((

crates/rsscript/src/reg_vm/tier.rs

@@ -2608,6 +2608,7 @@ impl RegVm {
                             func,
                             &std::collections::HashMap::new(),
                             &self_call_sites,
+                            &std::collections::HashMap::new(),
                         )
                         .and_then(|(jit_fn, ret, _params, _literals, _precise)| {
                             // This scalar fast path returns an `Int`; other return
@@ -2647,6 +2648,135 @@ impl RegVm {
         }
     }
 
+    /// Native mutual recursion (native-call-ABI slice 4): if `function_id` is part of
+    /// a mutually-recursive scalar-`Int` cycle, compile the whole group together
+    /// (declare the cycle, then define each) and dispatch the called member natively.
+    /// Returns the result on a clean completion, or `None` to fall back to the
+    /// interpreter (not eligible, or a deopt incl. the entry depth-cap bail). The
+    /// group is compiled once and every member cached.
+    #[cfg(feature = "native-jit")]
+    pub(super) fn try_native_mutual_recursive_int(
+        &mut self,
+        unit: &RegUnit,
+        function_id: usize,
+        caller_base: usize,
+        args: &[usize],
+    ) -> Option<VmValue> {
+        let func = unit.functions.get(function_id)?;
+        if args.len() != func.params {
+            return None;
+        }
+        let key = Rc::as_ptr(func) as usize;
+        let id = {
+            let native = self.native.as_mut()?;
+            if native.force_bail
+                || native.forced_safepoint.is_some()
+                || native.force_all_safepoints
+            {
+                return None;
+            }
+            if let Some(cached) = native.mutual_recursive_native.get(&key) {
+                (*cached)?
+            } else {
+                let group = native_recursive_int_group(unit, function_id);
+                let compiled = group.as_ref().and_then(|scc| {
+                    let index_of: std::collections::HashMap<usize, u32> = scc
+                        .iter()
+                        .enumerate()
+                        .map(|(i, &fid)| (fid, i as u32))
+                        .collect();
+                    let mut jit_funcs = Vec::with_capacity(scc.len());
+                    for &member in scc {
+                        let mfunc = unit.functions.get(member)?;
+                        let group_call_sites: std::collections::HashMap<usize, u32> = mfunc
+                            .code
+                            .iter()
+                            .enumerate()
+                            .filter_map(|(ip, instr)| match instr {
+                                RegInstr::CallKnown {
+                                    function, mut_args, ..
+                                } if mut_args.is_empty() && index_of.contains_key(function) => {
+                                    Some((ip, index_of[function]))
+                                }
+                                _ => None,
+                            })
+                            .collect();
+                        let (jit_fn, ret, _p, _l, _pr) = translate_to_native_jit_with_calls(
+                            unit,
+                            mfunc,
+                            &std::collections::HashMap::new(),
+                            &std::collections::HashSet::new(),
+                            &group_call_sites,
+                        )?;
+                        if ret != NativeTy::Int {
+                            return None;
+                        }
+                        jit_funcs.push(jit_fn);
+                    }
+                    native.module.compile_recursive_group(&jit_funcs).ok()
+                });
+                match (group, compiled) {
+                    (Some(scc), Some(ids)) if ids.len() == scc.len() => {
+                        let mut my_id = None;
+                        for (i, &member) in scc.iter().enumerate() {
+                            let mkey = Rc::as_ptr(&unit.functions[member]) as usize;
+                            native.mutual_recursive_native.insert(mkey, Some(ids[i]));
+                            if member == function_id {
+                                my_id = Some(ids[i]);
+                            }
+                        }
+                        my_id?
+                    }
+                    (group, _) => {
+                        // Cache ineligibility (for the whole detected group, or this key).
+                        match group {
+                            Some(scc) => {
+                                for member in scc {
+                                    let mkey = Rc::as_ptr(&unit.functions[member]) as usize;
+                                    native.mutual_recursive_native.insert(mkey, None);
+                                }
+                            }
+                            None => {
+                                native.mutual_recursive_native.insert(key, None);
+                            }
+                        }
+                        return None;
+                    }
+                }
+            }
+        };
+        let mut int_args = Vec::with_capacity(args.len());
+        for &arg in args {
+            let VmValue::Int(value) = self.reg(caller_base + arg) else {
+                return None;
+            };
+            int_args.push(*value);
+        }
+        let lens = vec![0i64; int_args.len()];
+        let mut heap_tx = JitNativeCallFrame::begin();
+        let outcome = {
+            let native = self.native.as_ref()?;
+            native
+                .module
+                .call_with_host_ctx(id, &int_args, &lens, heap_tx.host_ctx())
+        };
+        match outcome {
+            vm_jit::NativeOutcome::Completed(bits) => {
+                heap_tx.commit_scalar_with_writebacks(&[]);
+                if let Some(native) = self.native.as_mut()
+                    && native.collect_stats
+                {
+                    native.stats.native_calls += 1;
+                }
+                Some(VmValue::Int(bits))
+            }
+            _ => {
+                heap_tx.abort();
+                None
+            }
+        }
+    }
+
     pub(super) fn run_jit_self_recursive_int(
         &mut self,
         unit: &RegUnit,
@@ -2931,12 +3061,18 @@ fn propagate_same_kind(kinds: &mut [ScalarSlotKind], dst: usize, src: usize) ->
 }
 
 fn compute_self_recursive_int_jit_candidate(unit: &RegUnit, function_id: usize) -> bool {
-    compute_self_recursive_int_jit_candidate_inner(unit, function_id).unwrap_or(false)
+    let group: std::collections::HashSet<usize> = std::iter::once(function_id).collect();
+    compute_recursive_int_member_inner(unit, function_id, &group).unwrap_or(false)
 }
 
-fn compute_self_recursive_int_jit_candidate_inner(
+/// Scalar-`Int` recursion analysis for one member of a recursive `group`: the body
+/// must be all-scalar-`Int`, return `Int`, and every `CallKnown` must target a group
+/// member (self for self-recursion, or any sibling for mutual recursion) with scalar
+/// args and matching arity. `group = {function_id}` is the self-recursive case.
+fn compute_recursive_int_member_inner(
     unit: &RegUnit,
     function_id: usize,
+    group: &std::collections::HashSet<usize>,
 ) -> Option<bool> {
     let Some(func) = unit.functions.get(function_id) else {
         return Some(false);
@@ -3011,9 +3147,9 @@ fn compute_self_recursive_int_jit_candidate_inner(
                     function,
                     args,
                     mut_args,
-                } if *function == function_id
+                } if group.contains(function)
                     && mut_args.is_empty()
-                    && args.len() == func.params =>
+                    && unit.functions.get(*function).is_some_and(|f| f.params == args.len()) =>
                 {
                     saw_self_call = true;
                     let mut local_changed =
@@ -3049,6 +3185,59 @@ fn compute_self_recursive_int_jit_candidate_inner(
     )
 }
 
+/// The mutually-recursive group (call-graph SCC) containing `function_id`, if it is
+/// a cycle of >= 2 all-scalar-`Int` functions whose every `CallKnown` targets a
+/// group member (native-call-ABI slice 4). Returned sorted; `None` for non-cyclic
+/// functions and pure self-recursion (handled by the self-recursive path).
+#[cfg(feature = "native-jit")]
+fn native_recursive_int_group(unit: &RegUnit, function_id: usize) -> Option<Vec<usize>> {
+    use std::collections::HashSet;
+    let callees = |fid: usize| -> Vec<usize> {
+        unit.functions.get(fid).map_or_else(Vec::new, |f| {
+            f.code
+                .iter()
+                .filter_map(|instr| match instr {
+                    RegInstr::CallKnown { function, .. } => Some(*function),
+                    _ => None,
+                })
+                .collect()
+        })
+    };
+    // Forward-reachable from `function_id` via CallKnown edges.
+    let mut fwd = HashSet::new();
+    let mut stack = vec![function_id];
+    while let Some(f) = stack.pop() {
+        if fwd.insert(f) {
+            stack.extend(callees(f));
+        }
+    }
+    // Backward-reachable: functions that can transitively reach `function_id`.
+    let mut bwd = HashSet::new();
+    let mut stack = vec![function_id];
+    while let Some(target) = stack.pop() {
+        if bwd.insert(target) {
+            for caller in 0..unit.functions.len() {
+                if callees(caller).contains(&target) {
+                    stack.push(caller);
+                }
+            }
+        }
+    }
+    // The SCC is the intersection (mutually reachable with `function_id`).
+    let mut scc: Vec<usize> = fwd.intersection(&bwd).copied().collect();
+    scc.sort_unstable();
+    if scc.len() < 2 {
+        return None;
+    }
+    let group: HashSet<usize> = scc.iter().copied().collect();
+    for &member in &scc {
+        if !compute_recursive_int_member_inner(unit, member, &group).unwrap_or(false) {
+            return None;
+        }
+    }
+    Some(scc)
+}
+
 fn scalar_reachable_instructions(code: &[RegInstr]) -> Vec<bool> {
     let mut reachable = vec![false; code.len()];
     let mut stack = vec![0usize];