Only allow 'async' ABI options for 'async'-typed function imports/exports (#646)

Author: lukewagner

design/mvp/Binary.md

@@ -216,8 +216,6 @@ label'        ::= len:<u32> l:<label>                     => l    (if len = |l|)
 valtype       ::= i:<typeidx>                             => i
                 | pvt:<primvaltype>                       => pvt
 resourcetype  ::= 0x3f v:<valtype> f?:<core:funcidx>?     => (resource (rep v) (dtor f)?)
-                | 0x3e v:<valtype> f:<core:funcidx>
-                                   cb?:<core:funcidx>?    => (resource (rep v) (dtor async f (callback cb)?)) 🚝
 functype      ::= 0x40 ps:<paramlist> rs:<resultlist>     => (func ps rs)
                 | 0x43 ps:<paramlist> rs:<resultlist>     => (func async ps rs)
 paramlist     ::= lt*:vec(<labelvaltype>)                 => (param lt)*

design/mvp/CanonicalABI.md

@@ -777,15 +777,22 @@ class Task(Supertask):
     self.threads = []
 ```
 
-The `Task.needs_exclusive` predicate returns whether the Canonical ABI options
-indicate that the core wasm being executed does not expect to be reentered
-(e.g., because the code is using a single global linear memory shadow stack).
-Concretely, this is assumed to be the case when core wasm is lifted
-synchronously or with `async callback`. This predicate is used by the other
-`Task` methods to determine whether to acquire/release the component instance's
-`exclusive` lock.
+The `Task.needs_exclusive` method returns whether an `async`-typed function's
+ABI options indicate that the Core WebAssembly code requires serialized
+execution (with the common reason being that there is a single, global linear
+memory shadow stack). This serialized execution is implemented by
+acquiring/releasing the component-instance-wide `exclusive` lock before/after
+executing Core WebAssembly code executing on the task's *implicit thread*
+(explicit threads created by `thread.new-indirect` ignore the `exclusive` lock).
+Specifically, sync- and stackless-async-lifted (`async callback`) functions
+require the `exclusive` lock and stackful-async-lifted (`async`) functions
+ignore the `exclusive` lock (just like explicit threads). Note that
+non-`async`-typed functions' implicit threads also ignore the `exclusive` lock
+since they must complete synchronously without blocking and thus don't have to
+worry about non-LIFO stack interleaving.
 ```python
   def needs_exclusive(self):
+    assert(self.ft.async_)
     return not self.opts.async_ or self.opts.callback
 ```
 
@@ -1283,14 +1290,10 @@ the `rt` field of `ResourceHandle` (above) and thus resource type equality is
 class ResourceType(Type):
   impl: ComponentInstance
   dtor: Optional[Callable]
-  dtor_async: bool
-  dtor_callback: Optional[Callable]
 
-  def __init__(self, impl, dtor = None, dtor_async = False, dtor_callback = None):
+  def __init__(self, impl, dtor = None):
     self.impl = impl
     self.dtor = dtor
-    self.dtor_async = dtor_async
-    self.dtor_callback = dtor_callback
 ```
 
 
@@ -3470,7 +3473,9 @@ present, is validated as such:
 * if `realloc` is present then `memory` must be present
 * `post-return` - only allowed on [`canon lift`](#canon-lift), which has rules
   for validation
-* 🔀 `async` - cannot be present with `post-return`
+* 🔀 `async` - is only allowed when used with an `async` function type in
+  [`canon lift`](#canon-lift) or [`canon lower`](#canon-lower) and cannot be
+  present with `post-return`
 * 🔀,not(🚟) `async` - `callback` must also be present. Note that with the 🚟
   feature (the "stackful" ABI), this restriction is lifted.
 * 🔀 `callback` - the function has type `(func (param i32 i32 i32) (result i32))`
@@ -3598,7 +3603,7 @@ function (specified as a `funcidx` immediate in `canon lift`) until the
     [packed] = call_and_trap_on_throw(callee, flat_args)
     code,si = unpack_callback_result(packed)
     while code != CallbackCode.EXIT:
-      assert(inst.exclusive is task)
+      assert(task.needs_exclusive() and inst.exclusive is task)
       inst.exclusive = None
       match code:
         case CallbackCode.YIELD:
@@ -3921,22 +3926,22 @@ def canon_resource_drop(rt, i):
       if rt.dtor:
         rt.dtor(h.rep)
     else:
-      caller_opts = CanonicalOptions(async_ = False)
-      callee_opts = CanonicalOptions(async_ = rt.dtor_async, callback = rt.dtor_callback)
-      ft = FuncType([U32Type()],[], async_ = False)
+      ft = FuncType([U32Type()], [], async_ = False)
       dtor = rt.dtor or (lambda rep: [])
-      callee = inst.store.lift(dtor, ft, callee_opts, rt.impl)
-      caller = inst.store.lower(callee, ft, caller_opts, inst)
+      opts = CanonicalOptions(async_ = False)
+      callee = inst.store.lift(dtor, ft, opts, rt.impl)
+      caller = inst.store.lower(callee, ft, opts, inst)
       caller([h.rep])
   else:
     h.borrow_scope.num_borrows -= 1
   return []
 ```
-The call to a resource's destructor is defined as a non-`async`-lowered,
-non-`async`-typed function call to a possibly-`async`-lifted callee, passing
-the private `i32` representation as a parameter. Thus, destructors *may* block
-on I/O, but only after they `task.return`, ensuring that `resource.drop` never
-blocks.
+The call to a resource's destructor passes the `i32` representation value that
+was previously supplied to `resource.new`. The call works like a normal
+non-`async` cross-component call, using the same `canon_lift` and `canon_lower`
+rules to, for example, catch reentrance. Because the type, lifting and
+lowering are all non-`async`, the destructor may not block. However, the
+destructor may spawn a cooperative thread that does.
 
 Since there are valid reasons to call `resource.drop` in the same component
 instance that defined the resource, which would otherwise trap at the

design/mvp/Concurrency.md

@@ -74,16 +74,6 @@ the same way that they already bind to various OS's concurrent I/O APIs (such
 as `select`, `epoll`, `io_uring`, `kqueue` and Overlapped I/O) making the
 Component Model "just another OS" from the language toolchain's perspective.
 
-The new async ABI can be used alongside or instead of the existing Preview 2
-"sync ABI" to call or implement *any* WIT function type. When *calling* an
-imported function via the async ABI, if the callee [blocks](#blocking), control
-flow is returned immediately to the caller, and the callee continues executing
-concurrently. When *implementing* an exported function via the async ABI,
-multiple concurrent export calls are allowed to be made by the caller.
-Critically, both sync-ABI-calls-async-ABI and async-ABI-calls-sync-ABI pairings
-have well-defined, composable behavior for both inter-component and
-intra-component calls.
-
 In addition to adding a new async *ABI* for use by the language's compiler and
 runtime, the Component Model also adds a new `async` [effect type] that can be
 added to function types (in both WIT and raw component function type
@@ -103,6 +93,16 @@ invariant is necessary to allow non-`async` component exports to be called in
 synchronous contexts (like event listeners, callbacks, getters, setters and
 constructors).
 
+The new async ABI can be used alongside or instead of the existing Preview 2
+"sync ABI" to call or implement any `async`-typed functions. When *calling* an
+imported function via the async ABI, if the `async` callee [blocks](#blocking),
+control flow is returned immediately to the caller, and the callee continues
+executing concurrently. When *implementing* an `async` function via the async
+ABI, multiple concurrent export calls are allowed to be made by the caller.
+Critically, both sync-ABI-calls-async-ABI and async-ABI-calls-sync-ABI pairings
+have well-defined, composable behavior for both inter-component and
+intra-component calls.
+
 Because `async` function exports may be implemented with the *sync* ABI and
 then call `async` function imports using the *sync* ABI, traditional sync code
 can compile directly to components exporting `async` functions without having
@@ -685,30 +685,31 @@ the "started" state.
 
 ### Returning
 
-The way an async export call returns its value is by calling [`task.return`],
-passing the core values that are to be lifted as *parameters*.
+The way an `async` export returns its value using the async ABI is by calling
+[`task.return`], passing the core values that are to be lifted as *parameters*.
+When using the async ABI, *any* of the threads contained by a task can call
+`task.return`; there is no "main thread" of a task. When the last thread of a
+task returns, there is a trap if `task.return` has not been called. Thus, *some*
+thread (either the thread created implicitly for the initial export call or some
+thread transitively created by that thread) must call `task.return`.
 
 Returning values by calling `task.return` allows a task to continue executing
-even after it has passed its initial results to the caller. This can be useful
-for various finalization tasks (freeing memory or performing logging, billing
-or metrics operations) that don't need to be on the critical path of returning
-a value to the caller, but the major use of executing code after `task.return`
-is to continue to read and write from streams and futures. For example, a
-stream transformer function of type `func(in: stream<T>) -> stream<U>` will
-immediately `task.return` a stream created via `stream.new` and then sit in a
-loop interleaving `stream.read`s (of the readable end passed for `in`) and
-`stream.write`s (of the writable end it `stream.new`ed) before exiting the
-task.
-
-*Any* of the threads contained by a task can call `task.return`; there is no
-"main thread" of a task. When the last thread of a task returns, there is a
-trap if `task.return` has not been called. Thus, *some* thread (either the
-thread created implicitly for the initial export call or some thread
-transitively created by that thread) must call `task.return`.
+even after it has passed its initial results to the caller. This is also
+possible even with the sync ABI by using cooperative threads. Continuing
+to execute after returning a value can be useful for various finalization tasks
+(freeing memory or performing logging, billing or metrics operations) that don't
+need to be on the critical path of returning a value to the caller, but the
+major use of executing code after `task.return` is to continue to read and write
+from streams and futures. For example, a stream transformer function of type
+`func(in: stream<T>) -> stream<U>` will immediately `task.return` a stream
+created via `stream.new` and then sit in a loop interleaving `stream.read`s (of
+the readable end passed for `in`) and `stream.write`s (of the writable end it
+`stream.new`ed) before exiting the task.
 
 Once `task.return` is called, the task is in the "returned" state. Calling
-`task.return` when not in the "started" state traps. Once in a "returned"
-state, non-`async` functions are allowed to block.
+`task.return` when not in the "started" state traps. Once in a "returned" state,
+non-`async` functions may block using cooperative threads that were created
+before the synchronous task's implicit thread returned.
 
 ### Borrows
 
@@ -873,19 +874,12 @@ JS [top-level `await`] or I/O in C++ constructors executing during `start`.
 
 ## Async ABI
 
-At an ABI level, native async in the Component Model defines for every WIT
-function an async-oriented core function signature that can be used instead of
-or in addition to the existing (Preview-2-defined) synchronous core function
-signature. This async-oriented core function signature is intended to be called
-or implemented by generated bindings which then map the low-level core async
-protocol to the languages' higher-level native concurrency features.
-
-Note that *every* WIT-level function type can be lifted and lowered using the
-async (or sync) ABI. While calling a non-`async`-typed function import using
-the async ABI will never returned that the call "blocked" (as guaranteed by the
-Component Model trapping if the callee would have blocked), the async ABI is
-still allowed to be used (for the benefit of code generators that only want
-to think about one ABI).
+At an ABI level, native async in the Component Model defines for every
+`async`-typed function a non-blocking core function signature that can be
+used instead of or in addition to the existing (Preview-2-defined) synchronous
+core function signature. This non-blocking core function signature is intended
+to be called or implemented by generated bindings which then map the low-level
+core async protocol to the languages' higher-level native concurrency features.
 
 ### Async Import ABI
 

design/mvp/Explainer.md

@@ -573,9 +573,7 @@ valtype       ::= <typeidx>
                 | <defvaltype>
 keytype       ::= bool | s8 | u8 | s16 | u16 | s32 | u32 | s64 | u64 | char | string 🗺️
 resourcetype  ::= (resource (rep i32) (dtor <core:funcidx>)?)
-                | (resource (rep i32) (dtor async <core:funcidx> (callback <core:funcidx>)?)?) 🚝
                 | (resource (rep i64) (dtor <core:funcidx>)?) 🐘
-                | (resource (rep i64) (dtor async <core:funcidx> (callback <core:funcidx>)?)?) 🚝🐘
 functype      ::= (func async? (param "<label>" <valtype>)* (result <valtype>)?)
 componenttype ::= (component <componentdecl>*)
 instancetype  ::= (instance <instancedecl>*)
@@ -828,9 +826,8 @@ is currently fixed to `i32` or `i64`, but will potentially be relaxed to include
 other types. When the last handle to a resource is dropped, the resource's
 destructor function specified by the `dtor` immediate will be called (if
 present), allowing the implementing component to perform clean-up like freeing
-linear memory allocations. Destructors can be declared `async`, with the same
-meaning for the `async` and `callback` immediates as described below for `canon
-lift`. A destructor for a `resource (rep $T)` must have type `($T) -> ()`.
+linear memory allocations. A destructor for a `resource (rep $T)` must have type
+`($T) -> ()`.
 
 The `instance` type constructor describes a list of named, typed definitions
 that can be imported or exported by a component. Informally, instance types
@@ -1342,13 +1339,10 @@ be deallocated and destructors called. This immediate is always optional but,
 if present, is validated to have parameters matching the callee's return type
 and empty results.
 
-🔀 The `async` option specifies that the component wants to make (for imports)
-or support (for exports) multiple concurrent (asynchronous) calls. This option
-can be applied to any component-level function type and changes the derived
-Canonical ABI significantly. See the [concurrency explainer] for more details.
-When a function signature contains a `future` or `stream`, validation of `canon
-lower` requires the `async` option to be set (since a synchronous call to a
-function using these types is highly likely to deadlock).
+🔀 The `async` option may only be used with `async` function types and specifies
+that the component wants to make (for imports) or support (for exports) multiple
+concurrent (asynchronous) calls. This option changes the derived Canonical ABI
+significantly; see the [concurrency explainer] for more details.
 
 🔀 The `(callback ...)` option may only be present in `canon lift` when the
 `async` option has also been set and specifies a core function that is

design/mvp/canonical-abi/definitions.py

@@ -454,6 +454,7 @@ def __init__(self, ft, opts, inst, on_start, on_resolve, supertask):
     self.threads = []
 
   def needs_exclusive(self):
+    assert(self.ft.async_)
     return not self.opts.async_ or self.opts.callback
 
   def may_block(self):
@@ -690,14 +691,10 @@ def __init__(self, rt, rep, own, borrow_scope = None):
 class ResourceType(Type):
   impl: ComponentInstance
   dtor: Optional[Callable]
-  dtor_async: bool
-  dtor_callback: Optional[Callable]
 
-  def __init__(self, impl, dtor = None, dtor_async = False, dtor_callback = None):
+  def __init__(self, impl, dtor = None):
     self.impl = impl
     self.dtor = dtor
-    self.dtor_async = dtor_async
-    self.dtor_callback = dtor_callback
 
 ### Waitable State
 
@@ -2122,7 +2119,7 @@ def thread_func():
     [packed] = call_and_trap_on_throw(callee, flat_args)
     code,si = unpack_callback_result(packed)
     while code != CallbackCode.EXIT:
-      assert(inst.exclusive is task)
+      assert(task.needs_exclusive() and inst.exclusive is task)
       inst.exclusive = None
       match code:
         case CallbackCode.YIELD:
@@ -2266,12 +2263,11 @@ def canon_resource_drop(rt, i):
       if rt.dtor:
         rt.dtor(h.rep)
     else:
-      caller_opts = CanonicalOptions(async_ = False)
-      callee_opts = CanonicalOptions(async_ = rt.dtor_async, callback = rt.dtor_callback)
-      ft = FuncType([U32Type()],[], async_ = False)
+      ft = FuncType([U32Type()], [], async_ = False)
       dtor = rt.dtor or (lambda rep: [])
-      callee = inst.store.lift(dtor, ft, callee_opts, rt.impl)
-      caller = inst.store.lower(callee, ft, caller_opts, inst)
+      opts = CanonicalOptions(async_ = False)
+      callee = inst.store.lift(dtor, ft, opts, rt.impl)
+      caller = inst.store.lower(callee, ft, opts, inst)
       caller([h.rep])
   else:
     h.borrow_scope.num_borrows -= 1