pulley: trap on null in 8 fused funcref-dispatch handlers

Codex review on the rebeckerspecialties wasmtime fork PR pointed out
that phase-2/3's continuation-block load absorption breaks the
lazy-init slow path's correctness: the slow path's libcall rejoins
`continuation_block` via a block param, and after absorption the
loads are gone — `call_indirect` would see uninitialized
`dst_code`/`dst_vmctx` if the slow path is ever reached.

Fusion is gated on `is_eagerly_initialized_funcref_table` so the
slow path is unreachable at runtime, but the previous handler's
`ControlFlow::Continue(())` on null was advertised as defence-in-
depth and was itself broken. Replace it with `done_trap` in the 8
affected handlers (4 forward + 4 `_not` variants across x64/x32 ×
xfuncref_dispatch/xband_funcref_dispatch). `offset` on the `_not`
variants becomes vestigial; kept for encoding-shape parity.

Author: matthargett

cranelift/codegen/src/isa/pulley_shared/inst.isle

@@ -67,16 +67,10 @@
     ;; Jump to `then` if `c` is true, otherwise to `else`.
     (BrIf (cond Cond) (taken MachLabel) (not_taken MachLabel))
 
-    ;; Fused band-immediate + brif: compute `dst = src & sign_extend(mask)`,
-    ;; then conditionally branch to `taken` if `src` is non-zero (low-32 or
-    ;; full 64-bit comparison per `size`), otherwise fall through to
-    ;; `not_taken`. The mask + dst write happen unconditionally.
-    ;;
-    ;; Emitted by the Cranelift Pulley backend at the call_indirect lazy-init
-    ;; brif site when the same funcref-pointer value feeds both the init-bit
-    ;; mask (`band v, -2`) and the null-check branch (`brif v`). The fusion
-    ;; saves one match_loop dispatch per call_indirect site. See pulley/
-    ;; `xband*_s8_br_if_*` ops for the underlying bytecode.
+    ;; Fused `band src, mask` + `brif src` emitted at the call_indirect
+    ;; lazy-init brif site. `dst = src & sign_extend(mask)` is
+    ;; unconditional; the branch test is on `src`'s low-32 or full-64 bits
+    ;; per `size`. Pulley-side: `xband*_s8_br_if_*`.
     (BandBrIf
       (dst WritableXReg)
       (src XReg)
@@ -85,19 +79,11 @@
       (taken MachLabel)
       (not_taken MachLabel))
 
-    ;; Funcref-dispatch fusion: branch-and-load fused dispatch op emitted
-    ;; at the call_indirect lazy-init brif site under the eager-init
-    ;; predicate + statically-elided sig check. Fuses
-    ;;   `band v, -2 ; brif ; xload (v_masked + offset_code) ; xload (v_masked + offset_vmctx)`
-    ;; from across the brif's predecessor and continuation blocks into one
-    ;; Pulley dispatch. The pulley-side ops are `xfuncref_dispatch_{x64,
-    ;; not_x64, x32, not_x32}`; the runtime null check is defence-in-depth
-    ;; (the predicate guarantees the funcref is non-null at runtime, but
-    ;; the handler must match the original brif's null branch as a
-    ;; correctness fallback). See `try_fuse_funcref_dispatch` in
-    ;; `pulley_shared::lower` for the recogniser + Pulley's
-    ;; `pre_lower_analysis` for the cross-block sink that lets the
-    ;; continuation-block loads be absorbed into one MachInst here.
+    ;; Funcref-dispatch fusion: `brif (band v -2) + load code + load vmctx`
+    ;; across the brif and its continuation block. Emitted at the
+    ;; call_indirect lazy-init site under
+    ;; `is_eagerly_initialized_funcref_table`. Pulley-side:
+    ;; `xfuncref_dispatch_{x64,not_x64,x32,not_x32}`.
     (FuncrefDispatch
       (dst_code WritableXReg)
       (dst_vmctx WritableXReg)
@@ -108,15 +94,10 @@
       (taken MachLabel)
       (not_taken MachLabel))
 
-    ;; Phase-3 fusion: absorbs the preceding standalone `xband_s8 dst, src,
-    ;; -2` into the same MachInst as a `FuncrefDispatch`. Operand
-    ;; structure differs: `src` here is the UNMASKED funcref (the band's
-    ;; input), and `dst_masked` is added as a third def so the brif's
-    ;; block-call-arg copy to the continuation block still has a real
-    ;; producer for the funcref-ptr block param. The underlying Pulley
-    ;; ops are `xband_funcref_dispatch_{x64,not_x64,x32,not_x32}`. See
-    ;; `try_fuse_band_into_funcref_dispatch` in `pulley_shared::lower`
-    ;; for when phase 3 fires vs phase 2.
+    ;; FuncrefDispatch + the preceding `xband_s8 -2` absorbed. `src` is
+    ;; the unmasked funcref; the fused op writes `dst_masked = src & -2`
+    ;; so the brif's block-call-arg copy still has a producer.
+    ;; Pulley-side: `xband_funcref_dispatch_{x64,not_x64,x32,not_x32}`.
     (BandFuncrefDispatch
       (dst_masked WritableXReg)
       (dst_code WritableXReg)

cranelift/codegen/src/isa/pulley_shared/inst/args.rs

@@ -577,23 +577,16 @@ pub struct PulleyCall {
     pub args: SmallVec<[XReg; 4]>,
 }
 
-/// Payload of `CallInfo` for indirect-call instructions.
-///
-/// Mirror of `PulleyCall` for `Inst::IndirectCall`: the call target is a
-/// runtime register (the loaded `wasm_call` pointer at the call_indirect
-/// dispatch tail), and the first 0–4 integer ABI args are passed as free
-/// registers so the `call_indirect1/2/3/4` opcodes can move them into
-/// `x0..x3` as part of the call (saving one `xmov` per arg on the hot
-/// dispatch path). Remaining args live in `CallInfo::uses` with fixed
-/// pregs, just as for `PulleyCall`.
+/// Payload of `CallInfo` for `Inst::IndirectCall`. Mirror of `PulleyCall`:
+/// the first 0–4 integer ABI args are tracked here so the emitted
+/// `call_indirect{1,2,3,4}` opcode moves them into `x0..x3` itself
+/// instead of regalloc synthesising `xmov`s. Remaining args use the
+/// fixed-preg path in `CallInfo::uses`.
 #[derive(Clone, Debug)]
 pub struct PulleyCallIndirect {
-    /// The register holding the call target (e.g. the `wasm_call` pointer
-    /// loaded out of a `VMFuncRef`).
+    /// The register holding the call target.
     pub target: XReg,
-    /// Up to 4 integer args destined for `x0..x3`. Tracked separately so
-    /// regalloc doesn't insert moves and the `call_indirectN` opcode moves
-    /// them itself.
+    /// Up to 4 integer args destined for `x0..x3`.
     pub args: SmallVec<[XReg; 4]>,
 }
 

cranelift/codegen/src/isa/pulley_shared/inst/emit.rs

@@ -233,10 +233,9 @@ fn pulley_emit<P>(
         }
 
         Inst::IndirectCall { info } => {
-            // If x0..xN args are already in their correct ABI register
-            // (because regalloc allocated the producer's vreg there), drop
-            // them off the end so we can use a narrower `call_indirectN`
-            // op — mirror of the direct-call shrink loop above.
+            // Drop args already in their ABI register so we can pick a
+            // narrower `call_indirectN` — mirrors the direct-call shrink
+            // above.
             let target = info.dest.target;
             let mut args = &info.dest.args[..];
             while !args.is_empty() && args.last().copied() == XReg::new(x_reg(args.len() - 1)) {
@@ -407,12 +406,20 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xband32_s8_br_if_not_x32(
-                        &mut inverted, dst_writable, src_reg, mask_imm, 0,
+                        &mut inverted,
+                        dst_writable,
+                        src_reg,
+                        mask_imm,
+                        0,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xband64_s8_br_if_not_x64(
-                        &mut inverted, dst_writable, src_reg, mask_imm, 0,
+                        &mut inverted,
+                        dst_writable,
+                        src_reg,
+                        mask_imm,
+                        0,
                     );
                 }
             }
@@ -422,12 +429,20 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xband32_s8_br_if_not_x32(
-                        &mut inverted, dst_writable, src_reg, mask_imm, inv_rel,
+                        &mut inverted,
+                        dst_writable,
+                        src_reg,
+                        mask_imm,
+                        inv_rel,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xband64_s8_br_if_not_x64(
-                        &mut inverted, dst_writable, src_reg, mask_imm, inv_rel,
+                        &mut inverted,
+                        dst_writable,
+                        src_reg,
+                        mask_imm,
+                        inv_rel,
                     );
                 }
             }
@@ -482,12 +497,24 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xfuncref_dispatch_not_x32(
-                        &mut inverted, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0,
+                        &mut inverted,
+                        dst_code_w,
+                        dst_vmctx_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        0,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xfuncref_dispatch_not_x64(
-                        &mut inverted, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0,
+                        &mut inverted,
+                        dst_code_w,
+                        dst_vmctx_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        0,
                     );
                 }
             }
@@ -497,12 +524,24 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xfuncref_dispatch_not_x32(
-                        &mut inverted, dst_code_w, dst_vmctx_w, src_reg, oc, ov, inv_rel,
+                        &mut inverted,
+                        dst_code_w,
+                        dst_vmctx_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        inv_rel,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xfuncref_dispatch_not_x64(
-                        &mut inverted, dst_code_w, dst_vmctx_w, src_reg, oc, ov, inv_rel,
+                        &mut inverted,
+                        dst_code_w,
+                        dst_vmctx_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        inv_rel,
                     );
                 }
             }
@@ -513,12 +552,12 @@ fn pulley_emit<P>(
             sink.use_label_at_offset(taken_end - 4, *taken, LabelUse::PcRel);
             sink.add_cond_branch(*start_offset, taken_end, *taken, &inverted);
             patch_pc_rel_offset(sink, |sink| match size {
-                OperandSize::Size32 => enc::xfuncref_dispatch_x32(
-                    sink, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0,
-                ),
-                OperandSize::Size64 => enc::xfuncref_dispatch_x64(
-                    sink, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0,
-                ),
+                OperandSize::Size32 => {
+                    enc::xfuncref_dispatch_x32(sink, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0)
+                }
+                OperandSize::Size64 => {
+                    enc::xfuncref_dispatch_x64(sink, dst_code_w, dst_vmctx_w, src_reg, oc, ov, 0)
+                }
             });
             debug_assert_eq!(sink.cur_offset(), taken_end);
 
@@ -559,12 +598,26 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xband_funcref_dispatch_not_x32(
-                        &mut inverted, dm_w, dc_w, dv_w, src_reg, oc, ov, 0,
+                        &mut inverted,
+                        dm_w,
+                        dc_w,
+                        dv_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        0,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xband_funcref_dispatch_not_x64(
-                        &mut inverted, dm_w, dc_w, dv_w, src_reg, oc, ov, 0,
+                        &mut inverted,
+                        dm_w,
+                        dc_w,
+                        dv_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        0,
                     );
                 }
             }
@@ -574,12 +627,26 @@ fn pulley_emit<P>(
             match size {
                 OperandSize::Size32 => {
                     enc::xband_funcref_dispatch_not_x32(
-                        &mut inverted, dm_w, dc_w, dv_w, src_reg, oc, ov, inv_rel,
+                        &mut inverted,
+                        dm_w,
+                        dc_w,
+                        dv_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        inv_rel,
                     );
                 }
                 OperandSize::Size64 => {
                     enc::xband_funcref_dispatch_not_x64(
-                        &mut inverted, dm_w, dc_w, dv_w, src_reg, oc, ov, inv_rel,
+                        &mut inverted,
+                        dm_w,
+                        dc_w,
+                        dv_w,
+                        src_reg,
+                        oc,
+                        ov,
+                        inv_rel,
                     );
                 }
             }
@@ -589,12 +656,12 @@ fn pulley_emit<P>(
             sink.use_label_at_offset(taken_end - 4, *taken, LabelUse::PcRel);
             sink.add_cond_branch(*start_offset, taken_end, *taken, &inverted);
             patch_pc_rel_offset(sink, |sink| match size {
-                OperandSize::Size32 => enc::xband_funcref_dispatch_x32(
-                    sink, dm_w, dc_w, dv_w, src_reg, oc, ov, 0,
-                ),
-                OperandSize::Size64 => enc::xband_funcref_dispatch_x64(
-                    sink, dm_w, dc_w, dv_w, src_reg, oc, ov, 0,
-                ),
+                OperandSize::Size32 => {
+                    enc::xband_funcref_dispatch_x32(sink, dm_w, dc_w, dv_w, src_reg, oc, ov, 0)
+                }
+                OperandSize::Size64 => {
+                    enc::xband_funcref_dispatch_x64(sink, dm_w, dc_w, dv_w, src_reg, oc, ov, 0)
+                }
             });
             debug_assert_eq!(sink.cur_offset(), taken_end);
 

cranelift/codegen/src/isa/pulley_shared/inst/mod.rs

@@ -215,11 +215,9 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
                 ..
             } = &mut **info;
 
-            // Phase-4: the target and the first up-to-4 integer args live
-            // in `dest` and are passed as free reg uses; the emitted
-            // `call_indirect{1,2,3,4}` op moves the args into x0..x3 at
-            // call time. Remaining args still flow through `uses` with
-            // fixed pregs as before.
+            // First 0–4 integer args are passed as free reg uses; the
+            // emitted `call_indirect{1,2,3,4}` op moves them into x0..x3.
+            // Remaining args use the fixed-preg path in `uses`.
             let PulleyCallIndirect { target, args } = dest;
             collector.reg_use(target);
             for arg in args {