Merge pull request #234 from alexcrichton/merge

Merge with upstream

Author: alexcrichton

.github/workflows/main.yml

@@ -327,6 +327,7 @@ jobs:
               -p wasmtime --no-default-features --features pooling-allocator
               -p wasmtime --no-default-features --features cranelift
               -p wasmtime --no-default-features --features component-model
+              -p wasmtime --no-default-features --features component-model-async
               -p wasmtime --no-default-features --features runtime,component-model
               -p wasmtime --no-default-features --features cranelift,wat,async,std,cache
               -p wasmtime --no-default-features --features winch

cranelift/codegen/src/isa/x64/lower.isle

@@ -3618,10 +3618,15 @@
   (lower_select (is_nonzero_cmp cond) x y))
 
 ;; Note that for GPR-based spectre guards everything is forced into a register
-;; so go straight to the `lower_select_gpr` helper forcing `x` to be in a `Gpr`
-;; not `GprMem`.
+;; not `GprMem`. The `lower_select_spectre_gpr` helper below handles "and"
+;; conditions which the `lower_select_gpr` helper does not.
 (rule 1 (lower (has_type (is_single_register_gpr_type ty) (select_spectre_guard cond x y)))
-  (lower_select_gpr ty (is_nonzero_cmp cond) (put_in_gpr x) y))
+  (lower_select_spectre_gpr ty (is_nonzero_cmp cond) (put_in_gpr x) y))
+
+(decl lower_select_spectre_gpr (Type CondResult Gpr Gpr) Gpr)
+(rule 0 (lower_select_spectre_gpr ty cond a b) (lower_select_gpr ty cond a b))
+(rule 1 (lower_select_spectre_gpr ty cond @ (CondResult.And _ _ _) a b)
+  (lower_select_gpr ty (cond_invert cond) b a))
 
 ;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/entity/src/set.rs

@@ -148,6 +148,14 @@ where
         self.bitset.insert(index)
     }
 
+    /// Remove `k` from this bitset.
+    ///
+    /// Returns whether `k` was previously in this set or not.
+    pub fn remove(&mut self, k: K) -> bool {
+        let index = k.index();
+        self.bitset.remove(index)
+    }
+
     /// Removes and returns the entity from the set if it exists.
     pub fn pop(&mut self) -> Option<K> {
         let index = self.bitset.pop()?;

cranelift/filetests/filetests/isa/x64/select.clif

@@ -193,3 +193,67 @@ block0(v0: i8, v1: f128, v2: f128):
 ;   popq %rbp
 ;   retq
 
+function %select_gpr_with_and_condition(f32, f32, i64, i64) -> i64 {
+block0(v0: f32, v1: f32, v2: i64, v3: i64):
+    v4 = fcmp eq v0, v1
+    v5 = select v4, v2, v3
+    return v5
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   ucomiss %xmm1, %xmm0
+;   cmovpq %rsi, %rdi
+;   movq %rdi, %rax
+;   cmovneq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   ucomiss %xmm1, %xmm0
+;   cmovpq %rsi, %rdi
+;   movq %rdi, %rax
+;   cmovneq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %select_spectre_gpr_with_and_condition(f32, f32, i64, i64) -> i64 {
+block0(v0: f32, v1: f32, v2: i64, v3: i64):
+    v4 = fcmp eq v0, v1
+    v5 = select_spectre_guard v4, v2, v3
+    return v5
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   ucomiss %xmm1, %xmm0
+;   cmovpq %rsi, %rdi
+;   movq %rdi, %rax
+;   cmovneq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   ucomiss %xmm1, %xmm0
+;   cmovpq %rsi, %rdi
+;   movq %rdi, %rax
+;   cmovneq %rsi, %rax
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+

crates/c-api/src/val.rs

@@ -96,6 +96,7 @@ impl wasm_val_t {
             },
             Val::AnyRef(_) => crate::abort("creating a wasm_val_t from an anyref"),
             Val::ExternRef(_) => crate::abort("creating a wasm_val_t from an externref"),
+            Val::ExnRef(_) => crate::abort("creating a wasm_val_t from  an exnref"),
             Val::V128(_) => crate::abort("creating a wasm_val_t from a v128"),
         }
     }
@@ -251,6 +252,7 @@ impl wasmtime_val_t {
                     funcref: func.into(),
                 },
             },
+            Val::ExnRef(_) => crate::abort("exnrefs not yet supported in C API"),
             Val::V128(val) => wasmtime_val_t {
                 kind: crate::WASMTIME_V128,
                 of: wasmtime_val_union {

crates/cranelift/src/func_environ.rs

@@ -118,7 +118,6 @@ pub struct FuncEnvironment<'module_environment> {
     #[cfg(feature = "gc")]
     gc_heap_bound: Option<ir::GlobalValue>,
 
-    #[cfg(feature = "wmemcheck")]
     translation: &'module_environment ModuleTranslation<'module_environment>,
 
     /// Heaps implementing WebAssembly linear memories.
@@ -224,7 +223,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
             // functions should consume at least some fuel.
             fuel_consumed: 1,
 
-            #[cfg(feature = "wmemcheck")]
             translation,
 
             stack_limit_at_function_entry: None,
@@ -1321,26 +1319,34 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                 .vmctx_vmfunction_import_wasm_call(callee_index),
         )
         .unwrap();
-        let func_addr = self
-            .builder
-            .ins()
-            .load(pointer_type, mem_flags, base, body_offset);
 
         // First append the callee vmctx address.
         let vmctx_offset =
             i32::try_from(self.env.offsets.vmctx_vmfunction_import_vmctx(callee_index)).unwrap();
-        let vmctx = self
+        let callee_vmctx = self
             .builder
             .ins()
             .load(pointer_type, mem_flags, base, vmctx_offset);
-        real_call_args.push(vmctx);
+        real_call_args.push(callee_vmctx);
         real_call_args.push(caller_vmctx);
 
         // Then append the regular call arguments.
         real_call_args.extend_from_slice(call_args);
 
-        // Finally, make the indirect call!
-        Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+        // If we statically know the imported function (e.g. this is a
+        // component-to-component call where we statically know both components)
+        // then we can actually still make a direct call (although we do have to
+        // pass the callee's vmctx that we just loaded, not our own). Otherwise,
+        // we really do an indirect call.
+        if self.env.translation.known_imported_functions[callee_index].is_some() {
+            Ok(self.direct_call_inst(callee, &real_call_args))
+        } else {
+            let func_addr = self
+                .builder
+                .ins()
+                .load(pointer_type, mem_flags, base, body_offset);
+            Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+        }
     }
 
     /// Do an indirect call through the given funcref table.
@@ -1514,6 +1520,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             | WasmHeapType::ConcreteArray(_)
             | WasmHeapType::Struct
             | WasmHeapType::ConcreteStruct(_)
+            | WasmHeapType::Exn
+            | WasmHeapType::ConcreteExn(_)
+            | WasmHeapType::NoExn
             | WasmHeapType::None => {
                 unreachable!()
             }
@@ -1741,7 +1750,7 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm
     fn reference_type(&self, wasm_ty: WasmHeapType) -> (ir::Type, bool) {
         let ty = crate::reference_type(wasm_ty, self.pointer_type());
         let needs_stack_map = match wasm_ty.top() {
-            WasmHeapTopType::Extern | WasmHeapTopType::Any => true,
+            WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => true,
             WasmHeapTopType::Func => false,
             WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
         };
@@ -1838,7 +1847,7 @@ impl FuncEnvironment<'_> {
         let heap_ty = table.ref_type.heap_type;
         match heap_ty.top() {
             // GC-managed types.
-            WasmHeapTopType::Any | WasmHeapTopType::Extern => {
+            WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
                 let (src, flags) = table_data.prepare_table_addr(self, builder, index);
                 gc::gc_compiler(self)?.translate_read_gc_reference(
                     self,
@@ -1872,7 +1881,7 @@ impl FuncEnvironment<'_> {
         let heap_ty = table.ref_type.heap_type;
         match heap_ty.top() {
             // GC-managed types.
-            WasmHeapTopType::Any | WasmHeapTopType::Extern => {
+            WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
                 let (dst, flags) = table_data.prepare_table_addr(self, builder, index);
                 gc::gc_compiler(self)?.translate_write_gc_reference(
                     self,
@@ -2254,7 +2263,9 @@ impl FuncEnvironment<'_> {
         Ok(match ht.top() {
             WasmHeapTopType::Func => pos.ins().iconst(self.pointer_type(), 0),
             // NB: null GC references don't need to be in stack maps.
-            WasmHeapTopType::Any | WasmHeapTopType::Extern => pos.ins().iconst(types::I32, 0),
+            WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
+                pos.ins().iconst(types::I32, 0)
+            }
             WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
         })
     }
@@ -2646,7 +2657,8 @@ impl FuncEnvironment<'_> {
             // wasm module (e.g. imports or libcalls) are either encoded through
             // the `vmcontext` as relative jumps (hence no relocations) or
             // they're libcalls with absolute relocations.
-            colocated: self.module.defined_func_index(index).is_some(),
+            colocated: self.module.defined_func_index(index).is_some()
+                || self.translation.known_imported_functions[index].is_some(),
         }))
     }
 

crates/cranelift/src/func_environ/gc/enabled.rs

@@ -117,8 +117,10 @@ fn read_field_at_addr(
             WasmValType::F64 => builder.ins().load(ir::types::F64, flags, addr, 0),
             WasmValType::V128 => builder.ins().load(ir::types::I8X16, flags, addr, 0),
             WasmValType::Ref(r) => match r.heap_type.top() {
-                WasmHeapTopType::Any | WasmHeapTopType::Extern => gc_compiler(func_env)?
-                    .translate_read_gc_reference(func_env, builder, r, addr, flags)?,
+                WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
+                    gc_compiler(func_env)?
+                        .translate_read_gc_reference(func_env, builder, r, addr, flags)?
+                }
                 WasmHeapTopType::Func => {
                     let expected_ty = match r.heap_type {
                         WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),
@@ -1042,6 +1044,8 @@ pub fn translate_ref_test(
         | WasmHeapType::NoFunc
         | WasmHeapType::Cont
         | WasmHeapType::NoCont
+        | WasmHeapType::Exn
+        | WasmHeapType::NoExn
         | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"),
 
         // For these abstract but non-top and non-bottom types, we check the
@@ -1057,7 +1061,9 @@ pub fn translate_ref_test(
         // TODO: This check should ideally be done inline, but we don't have a
         // good way to access the `TypeRegistry`'s supertypes arrays from Wasm
         // code at the moment.
-        WasmHeapType::ConcreteArray(ty) | WasmHeapType::ConcreteStruct(ty) => {
+        WasmHeapType::ConcreteArray(ty)
+        | WasmHeapType::ConcreteStruct(ty)
+        | WasmHeapType::ConcreteExn(ty) => {
             let expected_interned_ty = ty.unwrap_module_type_index();
             let expected_shared_ty =
                 func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
@@ -1417,6 +1423,8 @@ impl FuncEnvironment<'_> {
             // Can only ever be `null`.
             WasmHeapType::NoExtern => false,
 
+            WasmHeapType::Exn | WasmHeapType::ConcreteExn(_) | WasmHeapType::NoExn => false,
+
             // Wrong type hierarchy, and also funcrefs are not GC-managed
             // types. Should have been caught by the assertion at the start of
             // the function.

crates/cranelift/src/lib.rs

@@ -212,7 +212,7 @@ fn wasm_call_signature(
 fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type {
     match wasm_ht.top() {
         WasmHeapTopType::Func => pointer_type,
-        WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32,
+        WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => ir::types::I32,
         WasmHeapTopType::Cont =>
         // TODO(10248) This is added in a follow-up PR
         {

crates/environ/src/compile/module_environ.rs

@@ -2,15 +2,16 @@ use crate::module::{
     FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment,
     TableSegmentElements,
 };
-use crate::prelude::*;
 use crate::{
     ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex,
     EntityIndex, EntityType, FuncIndex, GlobalIndex, IndexType, InitMemory, MemoryIndex,
     ModuleInternedTypeIndex, ModuleTypesBuilder, PrimaryMap, SizeOverflow, StaticMemoryInitializer,
     TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, Unsigned,
     WasmError, WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,
 };
+use crate::{StaticModuleIndex, prelude::*};
 use anyhow::{Result, bail};
+use cranelift_entity::SecondaryMap;
 use cranelift_entity::packed_option::ReservedValue;
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -54,6 +55,13 @@ pub struct ModuleTranslation<'data> {
     /// References to the function bodies.
     pub function_body_inputs: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'data>>,
 
+    /// For each imported function, the single statically-known defined function
+    /// that satisfies that import, if any. This is used to turn what would
+    /// otherwise be indirect calls through the imports table into direct calls,
+    /// when possible.
+    pub known_imported_functions:
+        SecondaryMap<FuncIndex, Option<(StaticModuleIndex, DefinedFuncIndex)>>,
+
     /// A list of type signatures which are considered exported from this
     /// module, or those that can possibly be called. This list is sorted, and
     /// trampolines for each of these signatures are required.
@@ -1220,7 +1228,10 @@ impl ModuleTranslation<'_> {
                 // initializer won't trap so we could continue processing
                 // segments, but that's left as a future optimization if
                 // necessary.
-                WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Cont => break,
+                WasmHeapTopType::Any
+                | WasmHeapTopType::Extern
+                | WasmHeapTopType::Cont
+                | WasmHeapTopType::Exn => break,
             }
 
             // Function indices can be optimized here, but fully general

crates/environ/src/compile/module_types.rs

@@ -441,6 +441,7 @@ where
                         WasmCompositeInnerType::Func(_) => WasmHeapType::ConcreteFunc(index),
                         WasmCompositeInnerType::Struct(_) => WasmHeapType::ConcreteStruct(index),
                         WasmCompositeInnerType::Cont(_) => WasmHeapType::ConcreteCont(index),
+                        WasmCompositeInnerType::Exn(_) => WasmHeapType::ConcreteExn(index),
                     }
                 } else if let Some((wasmparser_types, _)) = self.rec_group_context.as_ref() {
                     let wasmparser_ty = &wasmparser_types[id].composite_type;
@@ -481,6 +482,7 @@ where
                         WasmCompositeInnerType::Func(_) => WasmHeapType::ConcreteFunc(index),
                         WasmCompositeInnerType::Struct(_) => WasmHeapType::ConcreteStruct(index),
                         WasmCompositeInnerType::Cont(_) => WasmHeapType::ConcreteCont(index),
+                        WasmCompositeInnerType::Exn(_) => WasmHeapType::ConcreteExn(index),
                     }
                 } else if let Some((parser_types, rec_group)) = self.rec_group_context.as_ref() {
                     let rec_group_index = interned.index() - self.types.types.len_types();