adapt to the split table-mutability base

The eager-table-initialization commits moved out of the base PR to land
separately, and upstream moved MemFlags behind an interner. Two
adaptations:

- sink_pure_inst: resolve the MemFlags handle through dfg.mem_flags
  before checking readonly/notrap.
- call_indirect lazy-init site: test the raw funcref value again
  (upstream behavior). Testing the masked value, which lets the Pulley
  backend absorb the band into the dispatch op, is only sound when every
  reachable slot is eagerly initialized; that variant returns with the
  eager-init PR.

Goldens re-blessed accordingly: the xband*_s8_brif and
band_funcref_dispatch families are dormant until then. The dispatch
tail at the lazy-init site is xband ; xfuncref_dispatch ;
call_indirectN -- three interpreter dispatches, down from five unfused.

Author: matthargett

cranelift/codegen/src/machinst/lower.rs

@@ -1724,16 +1724,18 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
     /// unnecessary because we're not moving a side-effecting op — we're
     /// telling the lowerer it has been handled elsewhere.
     pub fn sink_pure_inst(&mut self, ir_inst: Inst) {
-        let dfg_inst = &self.f.dfg.insts[ir_inst];
         let is_pure = !has_lowering_side_effect(self.f, ir_inst);
-        let is_safe_load = matches!(
-            dfg_inst,
+        let is_safe_load = match &self.f.dfg.insts[ir_inst] {
             InstructionData::Load {
                 opcode: crate::ir::Opcode::Load,
                 flags,
                 ..
-            } if flags.readonly() && flags.notrap()
-        );
+            } => {
+                let flags = self.f.dfg.mem_flags[*flags];
+                flags.readonly() && flags.notrap()
+            }
+            _ => false,
+        };
         assert!(is_pure || is_safe_load);
         self.inst_absorbed_pure.insert(ir_inst);
     }

crates/cranelift/src/func_environ.rs

@@ -1074,20 +1074,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
         let result_param = builder.append_block_param(continuation_block, pointer_type);
         builder.set_cold_block(null_block);
 
-        // Under `is_eagerly_initialized_funcref_table`, `value != 0` and
-        // `value_masked != 0` agree on every reachable slot, so we can
-        // test the masked result. The Pulley backend then fuses the
-        // `band + brif` pair.
-        let brif_cond = if self
-            .module
-            .is_eagerly_initialized_funcref_table(table_index)
-        {
-            value_masked
-        } else {
-            value
-        };
+        // Branching on `value_masked` instead (letting the Pulley backend
+        // fuse the `band + brif` pair) requires a table whose slots are
+        // all eagerly initialized; that variant comes with eager
+        // initialization support.
         builder.ins().brif(
-            brif_cond,
+            value,
             continuation_block,
             &[value_masked.into()],
             null_block,

tests/disas/call-indirect-immutable-elide-null.wat

@@ -27,8 +27,9 @@
   ;; Fully cover the table — no null slot anywhere.
   (elem (i32.const 0) func $f1 $f2 $f3))
 ;; function u0:0(i64 vmctx, i64) -> i32 tail {
+;;     region0 = 8 "VMContext+0x8"
 ;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
+;;     gv1 = load.i64 notrap aligned readonly region0 gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1+24
 ;;     stack_limit = gv2
 ;;
@@ -41,8 +42,9 @@
 ;; }
 ;;
 ;; function u0:1(i64 vmctx, i64) -> i32 tail {
+;;     region0 = 8 "VMContext+0x8"
 ;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
+;;     gv1 = load.i64 notrap aligned readonly region0 gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1+24
 ;;     stack_limit = gv2
 ;;
@@ -55,8 +57,9 @@
 ;; }
 ;;
 ;; function u0:2(i64 vmctx, i64) -> i32 tail {
+;;     region0 = 8 "VMContext+0x8"
 ;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
+;;     gv1 = load.i64 notrap aligned readonly region0 gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1+24
 ;;     stack_limit = gv2
 ;;
@@ -69,43 +72,45 @@
 ;; }
 ;;
 ;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
+;;     region0 = 8 "VMContext+0x8"
+;;     region1 = 1342177280 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))"
 ;;     gv0 = vmctx
-;;     gv1 = load.i64 notrap aligned readonly gv0+8
+;;     gv1 = load.i64 notrap aligned readonly region0 gv0+8
 ;;     gv2 = load.i64 notrap aligned gv1+24
 ;;     gv3 = vmctx
 ;;     gv4 = load.i64 notrap aligned readonly can_move gv3+48
 ;;     sig0 = (i64 vmctx, i64) -> i32 tail
 ;;     sig1 = (i64 vmctx, i32, i64) -> i64 tail
-;;     fn0 = colocated u805306368:9 sig1
+;;     fn0 = colocated u805306368:7 sig1
 ;;     stack_limit = gv2
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32):
 ;; @0050                               v4 = iconst.i32 3
 ;; @0050                               v5 = icmp uge v2, v4  ; v4 = 3
 ;; @0050                               v6 = uextend.i64 v2
 ;; @0050                               v7 = load.i64 notrap aligned readonly can_move v0+48
-;;                                     v23 = iconst.i64 3
-;; @0050                               v8 = ishl v6, v23  ; v23 = 3
-;; @0050                               v9 = iadd v7, v8
-;; @0050                               v10 = iconst.i64 0
-;; @0050                               v11 = select_spectre_guard v5, v10, v9  ; v10 = 0
-;; @0050                               v12 = load.i64 user5 aligned table v11
-;;                                     v22 = iconst.i64 -2
-;; @0050                               v13 = band v12, v22  ; v22 = -2
-;; @0050                               brif v13, block3(v13), block2
+;; @0050                               v8 = iconst.i64 3
+;; @0050                               v9 = ishl v6, v8  ; v8 = 3
+;; @0050                               v10 = iadd v7, v9
+;; @0050                               v11 = iconst.i64 0
+;; @0050                               v12 = select_spectre_guard v5, v11, v10  ; v11 = 0
+;; @0050                               v13 = load.i64 user6 aligned region1 v12
+;; @0050                               v14 = iconst.i64 -2
+;; @0050                               v15 = band v13, v14  ; v14 = -2
+;; @0050                               brif v13, block3(v15), block2
 ;;
 ;;                                 block2 cold:
-;; @0050                               v15 = iconst.i32 0
-;; @0050                               v17 = uextend.i64 v2
-;; @0050                               v18 = call fn0(v0, v15, v17)  ; v15 = 0
-;; @0050                               jump block3(v18)
+;; @0050                               v17 = iconst.i32 0
+;; @0050                               v18 = uextend.i64 v2
+;; @0050                               v19 = call fn0(v0, v17, v18)  ; v17 = 0
+;; @0050                               jump block3(v19)
 ;;
-;;                                 block3(v14: i64):
-;; @0050                               v19 = load.i64 notrap aligned readonly v14+8
-;; @0050                               v20 = load.i64 notrap aligned readonly v14+24
-;; @0050                               v21 = call_indirect sig0, v19(v20, v0)
+;;                                 block3(v16: i64):
+;; @0050                               v20 = load.i64 notrap aligned readonly v16+8
+;; @0050                               v21 = load.i64 notrap aligned readonly v16+24
+;; @0050                               v22 = call_indirect sig0, v20(v21, v0)
 ;; @0053                               jump block1
 ;;
 ;;                                 block1:
-;; @0053                               return v21
+;; @0053                               return v22
 ;; }

tests/disas/pulley-call-indirect-band-brif-fusion.wat

@@ -53,25 +53,28 @@
 ;;       ret
 ;;
 ;; wasm[0]::function[3]:
-;;       push_frame_save 32, x16, x17, x24
+;;       push_frame_save 16, x25
 ;;       xmov x3, x0
-;;       br_if_xugteq32_u8 x2, 3, 0x4b    // target = 0x64
+;;       br_if_xugteq32_u8 x2, 3, 0x59    // target = 0x72
 ;;   20: xmov x1, x3
 ;;       xload64le_o32 x0, x1, 48
 ;;       zext32 x15, x2
 ;;       xshl64_u6 x1, x15, 3
 ;;       xadd64 x0, x0, x1
-;;       xload64le_o32 x0, x0, 0
-;;       xband_funcref_dispatch_not_x64 x0, x17, x16, x0, 8, 24, 0x18    // target = 0x52
-;;       xmov x24, x3
-;;       call_indirect2 x17, x16, x24
-;;       pop_frame_restore 32, x16, x17, x24
+;;       xload64le_o32 x1, x0, 0
+;;       xband64_s8 x0, x1, -2
+;;       br_if_xeq64_i8 x1, 0, 0x22    // target = 0x60
+;;   45: xmov x25, x3
+;;       xload64le_o32 x1, x0, 8
+;;       xload64le_o32 x0, x0, 24
+;;       call_indirect2 x1, x0, x25
+;;       pop_frame_restore 16, x25
 ;;       ret
-;;   52: xzero x0
-;;   54: xmov x24, x3
-;;   57: call3 x24, x0, x15, 0x267    // target = 0x2be
-;;   5f: jump -0x17    // target = 0x48
-;;   64: trap
+;;   60: xzero x0
+;;   62: xmov x25, x3
+;;   65: call3 x25, x0, x15, 0x267    // target = 0x2cc
+;;   6d: jump -0x25    // target = 0x48
+;;   72: trap
 ;;       ╰─╼ trap: Normal(TableOutOfBounds)
 ;;
 ;; wasm[0]::array_to_wasm_trampoline[0]:
@@ -82,19 +85,19 @@
 ;;       xstore64le_o32 x13, 72, x14
 ;;       xmov x14, sp
 ;;       xstore64le_o32 x13, 64, x14
-;;       xpcadd x15, 0x2a    // target = 0xb9
+;;       xpcadd x15, 0x2a    // target = 0xc7
 ;;       xstore64le_o32 x13, 80, x15
-;;       call -0x9e    // target = 0x0
+;;       call -0xac    // target = 0x0
 ;;       ├─╼ exception frame offset: SP = FP - 0x90
-;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0xb9
+;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0xc7
 ;;       xload64le_o32 x2, sp, 0
 ;;       xstore32le_o32 x2, 0, x0
 ;;       xone x0
 ;;       pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
 ;;       ret
-;;   b9: xzero x0
-;;   bb: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;;   c0: ret
+;;   c7: xzero x0
+;;   c9: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;;   ce: ret
 ;;
 ;; wasm[0]::array_to_wasm_trampoline[1]:
 ;;       push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
@@ -104,19 +107,19 @@
 ;;       xstore64le_o32 x13, 72, x14
 ;;       xmov x14, sp
 ;;       xstore64le_o32 x13, 64, x14
-;;       xpcadd x15, 0x2a    // target = 0x113
+;;       xpcadd x15, 0x2a    // target = 0x121
 ;;       xstore64le_o32 x13, 80, x15
-;;       call -0xf3    // target = 0x5
+;;       call -0x101    // target = 0x5
 ;;       ├─╼ exception frame offset: SP = FP - 0x90
-;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x113
+;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x121
 ;;       xload64le_o32 x2, sp, 0
 ;;       xstore32le_o32 x2, 0, x0
 ;;       xone x0
 ;;       pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
 ;;       ret
-;;  113: xzero x0
-;;  115: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;;  11a: ret
+;;  121: xzero x0
+;;  123: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;;  128: ret
 ;;
 ;; wasm[0]::array_to_wasm_trampoline[2]:
 ;;       push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
@@ -126,19 +129,19 @@
 ;;       xstore64le_o32 x13, 72, x14
 ;;       xmov x14, sp
 ;;       xstore64le_o32 x13, 64, x14
-;;       xpcadd x15, 0x2a    // target = 0x16d
+;;       xpcadd x15, 0x2a    // target = 0x17b
 ;;       xstore64le_o32 x13, 80, x15
-;;       call -0x147    // target = 0xb
+;;       call -0x155    // target = 0xb
 ;;       ├─╼ exception frame offset: SP = FP - 0x90
-;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x16d
+;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x17b
 ;;       xload64le_o32 x2, sp, 0
 ;;       xstore32le_o32 x2, 0, x0
 ;;       xone x0
 ;;       pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
 ;;       ret
-;;  16d: xzero x0
-;;  16f: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;;  174: ret
+;;  17b: xzero x0
+;;  17d: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;;  182: ret
 ;;
 ;; wasm[0]::array_to_wasm_trampoline[3]:
 ;;       push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
@@ -149,19 +152,19 @@
 ;;       xstore64le_o32 x15, 72, x2
 ;;       xmov x2, sp
 ;;       xstore64le_o32 x15, 64, x2
-;;       xpcadd x2, 0x2d    // target = 0x1d1
+;;       xpcadd x2, 0x2d    // target = 0x1df
 ;;       xstore64le_o32 x15, 80, x2
-;;       call3 x0, x1, x14, -0x1a2    // target = 0x11
+;;       call3 x0, x1, x14, -0x1b0    // target = 0x11
 ;;       ├─╼ exception frame offset: SP = FP - 0x90
-;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x1d1
+;;       ╰─╼ exception handler: default handler, no dynamic context, handler=0x1df
 ;;       xload64le_o32 x2, sp, 0
 ;;       xstore32le_o32 x2, 0, x0
 ;;       xone x0
 ;;       pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
 ;;       ret
-;;  1d1: xzero x0
-;;  1d3: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;;  1d8: ret
+;;  1df: xzero x0
+;;  1e1: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;;  1e6: ret
 ;;
 ;; signatures[0]::wasm_to_array_trampoline:
 ;;       push_frame_save 32, x16, x17
@@ -180,15 +183,15 @@
 ;;       xmov x3, x16
 ;;       call_indirect_host 0
 ;;       zext8 x15, x0
-;;       br_if_not32 x15, 0x13    // target = 0x230
-;;  223: xload32le_o32 x0, x16, 0
+;;       br_if_not32 x15, 0x13    // target = 0x23e
+;;  231: xload32le_o32 x0, x16, 0
 ;;       pop_frame_restore 32, x16, x17
 ;;       ret
-;;  230: xmov x1, x17
-;;  233: xload64le_o32 x0, x1, 16
-;;  23a: xload64le_o32 x0, x0, 328
-;;  241: call_indirect_host 42
-;;  245: trap
+;;  23e: xmov x1, x17
+;;  241: xload64le_o32 x0, x1, 16
+;;  248: xload64le_o32 x0, x0, 328
+;;  24f: call_indirect_host 42
+;;  253: trap
 ;;
 ;; signatures[1]::wasm_to_array_trampoline:
 ;;       push_frame_save 32, x16, x17
@@ -208,15 +211,15 @@
 ;;       xmov x3, x16
 ;;       call_indirect_host 0
 ;;       zext8 x0, x0
-;;       br_if_not32 x0, 0x13    // target = 0x2a6
-;;  299: xload32le_o32 x0, x16, 0
+;;       br_if_not32 x0, 0x13    // target = 0x2b4
+;;  2a7: xload32le_o32 x0, x16, 0
 ;;       pop_frame_restore 32, x16, x17
 ;;       ret
-;;  2a6: xmov x1, x17
-;;  2a9: xload64le_o32 x0, x1, 16
-;;  2b0: xload64le_o32 x0, x0, 328
-;;  2b7: call_indirect_host 42
-;;  2bb: trap
+;;  2b4: xmov x1, x17
+;;  2b7: xload64le_o32 x0, x1, 16
+;;  2be: xload64le_o32 x0, x0, 328
+;;  2c5: call_indirect_host 42
+;;  2c9: trap
 ;;
 ;; wasmtime_builtin_table_get_lazy_init_func_ref:
 ;;       push_frame