fix(reg-vm): recursive native dispatch must honor step_budget/cancel gate

The recursive native fast paths in the CallKnown handler (self-recursive
run_jit_self_recursive_int and mutual-recursive try_native_mutual_recursive_int)
were gated only on mem_budget.is_none(), not step_budget or cancel. Cranelift
code polls neither the step budget nor the cancel flag, so a recursive native
candidate (fib, is_even/is_odd) could run to completion and silently bypass
preemption that the host armed -- exactly what the normal native-tier gate in
try_native already refuses for (it checks all three: step_budget, cancel,
mem_budget). This is a sandbox-soundness gap: a watchdog/step limit would not
stop native recursion.

Fix: factor a single `RegVm::native_limits_unarmed()` (step_budget.is_none() &&
cancel.is_none() && mem_budget.is_none()) and gate both recursive hooks on it;
dedup the try_native gate to `!native_limits_unarmed()` so there is one source of
truth. With any limit armed, recursion runs on the interpreter / tier-0 executor,
which tick()s every instruction (and tick() polls both step_budget and cancel).

Test plumbing: `eval_main_with_args_native_with_limits` threads VmLimits into the
native eval path. Regressions (jit_acceptance):
- native_self_recursion_refused_when_step_budget_armed (native_calls == 0)
- native_mutual_recursion_refused_when_step_budget_armed (native_calls == 0)
- native_recursion_refused_when_cancel_armed (present, un-triggered flag)
- native_self_recursion_step_budget_preempts (small budget actually trips)

Verified: jit_acceptance 92/0, differential 33/0, vm-jit 83/0, hostile 16/0,
static 627/0, default build clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: olwang

crates/rsscript/src/reg_vm/exec.rs

@@ -113,6 +113,18 @@ impl RegVm {
     /// `CancellationToken` remains the cooperative, per-task mechanism (it only
     /// preempts at await points); this ambient flag is the blunt host-level kill.
     #[inline]
+    /// Whether it is sound to dispatch Cranelift-native code right now: native code
+    /// polls neither the step budget nor the cancel flag and runs allocation off the
+    /// memory meter, so all three preemption/accounting limits must be unarmed (it
+    /// `tick()`s on the interpreter/tier-0 paths instead). The single source of truth
+    /// for both the native-tier gate (`try_native`) and the recursive native fast
+    /// paths (self-recursive + mutual-recursive); see execution spec §6.2 (Model A).
+    pub(super) fn native_limits_unarmed(&self) -> bool {
+        self.limits.step_budget.is_none()
+            && self.limits.cancel.is_none()
+            && self.limits.mem_budget.is_none()
+    }
+
     pub(super) fn tick(&mut self) -> Result<(), EvalError> {
         self.steps += 1;
         if let Some(limit) = self.limits.step_budget
@@ -1310,9 +1322,15 @@ impl RegVm {
                             args,
                             mut_args,
                         } => {
+                            // Recursive native fast paths run Cranelift code that polls
+                            // neither `step_budget` nor `cancel` (and allocates off the
+                            // `mem_budget` meter), so they are gated on all three limits
+                            // being unarmed — matching the native-tier gate in
+                            // `try_native`. With any limit armed, recursion runs on the
+                            // interpreter / tier-0 executor, which `tick()`s every step.
                             if self.jit_enabled
                                 && mut_args.is_empty()
-                                && self.limits.mem_budget.is_none()
+                                && self.native_limits_unarmed()
                                 && let Some(value) =
                                     self.run_jit_self_recursive_int(unit, *callee_id, base, args)?
                             {
@@ -1325,7 +1343,7 @@ impl RegVm {
                             #[cfg(feature = "native-jit")]
                             if self.jit_enabled
                                 && mut_args.is_empty()
-                                && self.limits.mem_budget.is_none()
+                                && self.native_limits_unarmed()
                                 && let Some(value) =
                                     self.try_native_mutual_recursive_int(unit, *callee_id, base, args)
                             {

crates/rsscript/src/reg_vm/mod.rs

@@ -1445,7 +1445,7 @@ impl RegVmExecutable {
         args: impl IntoIterator<Item = impl Into<String>>,
     ) -> Result<(EvalOutput, NativeStats, Vec<String>), EvalError> {
         self.eval_main_with_args_native_inner_reported(
-            args, 0, false, true, true, true, true, None, false,
+            args, 0, false, true, true, true, true, None, false, VmLimits::default(),
         )
     }
 
@@ -1471,6 +1471,23 @@ impl RegVmExecutable {
             false,
             forced_safepoint,
             force_all_safepoints_override,
+            VmLimits::default(),
+        )
+        .map(|(output, stats, _lines)| (output, stats))
+    }
+
+    /// Like [`Self::eval_main_with_args_native_with_stats`] but runs under explicit
+    /// [`VmLimits`]. With native enabled, an armed `step_budget`/`cancel`/`mem_budget`
+    /// must prevent native dispatch (Cranelift polls/accounts none of them) — used to
+    /// regression-test the recursive native fast-path limit gate.
+    #[cfg(feature = "native-jit")]
+    pub fn eval_main_with_args_native_with_limits(
+        &self,
+        args: impl IntoIterator<Item = impl Into<String>>,
+        limits: VmLimits,
+    ) -> Result<(EvalOutput, NativeStats), EvalError> {
+        self.eval_main_with_args_native_inner_reported(
+            args, 0, false, true, true, false, false, None, false, limits,
         )
         .map(|(output, stats, _lines)| (output, stats))
     }
@@ -1488,12 +1505,17 @@ impl RegVmExecutable {
         report_override: bool,
         forced_safepoint: Option<u32>,
         force_all_safepoints_override: bool,
+        limits: VmLimits,
     ) -> Result<(EvalOutput, NativeStats, Vec<String>), EvalError> {
         let mut vm = RegVm::new(
             Rc::clone(&self.unit),
             args.into_iter().map(Into::into).collect(),
             std::iter::empty::<(String, NativeInterpreterFn)>().collect(),
         );
+        // Limits gate native dispatch: when any preemption/accounting limit is armed,
+        // `native_limits_unarmed()` refuses native (incl. the recursive fast paths) so
+        // the interpreter/tier-0 path enforces it via `tick()`.
+        vm.set_limits(limits);
         // Native first, then tier-0, then interpreter.
         // `RSS_JIT_BASELINE=1` selects the Phase-2 path-B baseline tier
         // (`opt_level="none"`); default (unset) keeps the optimizing tier

crates/rsscript/src/reg_vm/tier.rs

@@ -868,10 +868,7 @@ impl RegVm {
         // `mem_budget`, so a function could grow the accounted live-set past the
         // limit without erroring. Refusing native while `mem_budget` is armed keeps
         // the limit exact (Model A; matches the tier-0 self-recursive gate).
-        if self.limits.step_budget.is_some()
-            || self.limits.cancel.is_some()
-            || self.limits.mem_budget.is_some()
-        {
+        if !self.native_limits_unarmed() {
             return NativeAttempt::Fallback;
         }
         // Cheap negative path: a function known not native-eligible never compiles,

crates/rsscript/tests/jit_acceptance.rs

@@ -4693,3 +4693,124 @@ fn main() -> Unit {
         "shallow Bool mutual recursion should run natively: {stats:?}",
     );
 }
+
+// ---------------------------------------------------------------------------
+// Recursive native fast paths must honor the same limit gate as `try_native`:
+// Cranelift code polls neither `step_budget` nor `cancel` and allocates off the
+// `mem_budget` meter, so with any of them armed the recursive paths must NOT
+// dispatch natively (they run on the interpreter / tier-0, which `tick()`s).
+// ---------------------------------------------------------------------------
+
+const FIB_SELF_SRC: &str = "\
+fn fib(n: Int) -> Int {
+    if n < 2 { return n }
+    return fib(n: read n - 1) + fib(n: read n - 2)
+}
+fn main() -> Unit {
+    Log.write(message: read String.from_int(value: fib(n: read 28)))
+    return Unit
+}
+";
+
+const IS_EVEN_MUTUAL_SRC: &str = "\
+fn is_even(n: Int) -> Int {
+    if n < 1 { return 1 }
+    return is_odd(n: read n - 1)
+}
+fn is_odd(n: Int) -> Int {
+    if n < 1 { return 0 }
+    return is_even(n: read n - 1)
+}
+fn main() -> Unit {
+    Log.write(message: read String.from_int(value: is_even(n: read 20)))
+    return Unit
+}
+";
+
+/// Self-recursive native candidate (`fib`) must NOT dispatch natively while a
+/// `step_budget` is armed (native never `tick()`s, so it would bypass the budget).
+/// The budget here is generous enough to complete on the interpreter; the proof is
+/// `native_calls == 0` (native refused) with the correct result.
+#[cfg(feature = "native-jit")]
+#[test]
+fn native_self_recursion_refused_when_step_budget_armed() {
+    let exe = rsscript::reg_vm_compile_source("limit-self.rss", FIB_SELF_SRC).expect("compile");
+    let limits = rsscript::VmLimits {
+        step_budget: Some(50_000_000),
+        ..rsscript::VmLimits::default()
+    };
+    let (out, stats) = exe
+        .eval_main_with_args_native_with_limits(std::iter::empty::<String>(), limits)
+        .expect("completes within budget");
+    assert_eq!(out.stdout.trim_end(), "317811");
+    assert_eq!(
+        stats.native_calls, 0,
+        "armed step_budget must refuse native self-recursion: {stats:?}"
+    );
+}
+
+/// Mutual-recursion native candidate (`is_even`/`is_odd`) must NOT dispatch natively
+/// while a `step_budget` is armed.
+#[cfg(feature = "native-jit")]
+#[test]
+fn native_mutual_recursion_refused_when_step_budget_armed() {
+    let exe =
+        rsscript::reg_vm_compile_source("limit-mutual.rss", IS_EVEN_MUTUAL_SRC).expect("compile");
+    let limits = rsscript::VmLimits {
+        step_budget: Some(50_000_000),
+        ..rsscript::VmLimits::default()
+    };
+    let (out, stats) = exe
+        .eval_main_with_args_native_with_limits(std::iter::empty::<String>(), limits)
+        .expect("completes within budget");
+    assert_eq!(out.stdout.trim_end(), "1");
+    assert_eq!(
+        stats.native_calls, 0,
+        "armed step_budget must refuse native mutual recursion: {stats:?}"
+    );
+}
+
+/// A present `cancel` flag (even un-triggered) must refuse recursive native dispatch,
+/// matching `try_native` — native code can never observe the flag, so the cooperative
+/// interpreter path (which polls it in `tick()`) must run instead.
+#[cfg(feature = "native-jit")]
+#[test]
+fn native_recursion_refused_when_cancel_armed() {
+    let exe = rsscript::reg_vm_compile_source("limit-cancel.rss", FIB_SELF_SRC).expect("compile");
+    let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let limits = rsscript::VmLimits {
+        cancel: Some(cancel),
+        ..rsscript::VmLimits::default()
+    };
+    let (out, stats) = exe
+        .eval_main_with_args_native_with_limits(std::iter::empty::<String>(), limits)
+        .expect("completes (flag never set)");
+    assert_eq!(out.stdout.trim_end(), "317811");
+    assert_eq!(
+        stats.native_calls, 0,
+        "an armed cancel flag must refuse native recursion: {stats:?}"
+    );
+}
+
+/// Enforcement, not just refusal: a small `step_budget` must actually PREEMPT a
+/// recursive native candidate (with the bug, native ran `fib(28)` to completion and
+/// silently bypassed the budget). Now it runs on the interpreter and trips the budget.
+#[cfg(feature = "native-jit")]
+#[test]
+fn native_self_recursion_step_budget_preempts() {
+    let exe = rsscript::reg_vm_compile_source("limit-preempt.rss", FIB_SELF_SRC).expect("compile");
+    let limits = rsscript::VmLimits {
+        step_budget: Some(1_000),
+        ..rsscript::VmLimits::default()
+    };
+    let err = exe
+        .eval_main_with_args_native_with_limits(std::iter::empty::<String>(), limits)
+        .expect_err("small step budget must preempt, not be bypassed by native");
+    match err {
+        rsscript::EvalError::Runtime(msg) => assert!(
+            msg.contains("step budget"),
+            "expected step-budget error, got: {msg}"
+        ),
+        other => panic!("expected Runtime(step budget), got {other:?}"),
+    }
+}