Debugging: add the debug-main world. (bytecodealliance#12756)

* Debugging: add the debug-main world.

This PR "draws the rest of the owl" for the debug-main
world (bytecodealliance/rfcs#45). This includes a WIT world that hosts
debug components that have access to "host debug powers" via a
debugging API, and the ability to load such a debug-component and give
it control of the main program as a debuggee when using `wasmtime
run`.

The WIT is namespaced to `bytecodealliance:wasmtime` and is slightly
aspirational in places: for example, the host does not yet implement
injection of early return values or exception-throws. I intend to fill
out a series of TODO issues once this all lands to track followup
("post-MVP") work.

This PR does not include any debug components. I separately have a
gdbstub component, with which I tested and co-developed this host-side
implementation. My plan is to land it in a followup PR as a component
that will be embedded in/shipped with the Wasmtime CLI and available
under an easy-to-use CLI option. Once we have that gdbstub component,
we can also implement end-to-end integration tests that boot up LLDB
and run through an expected interaction. (Separately, those
integration tests will require a release of wasi-sdk to ship an LLDB
binary that we can use.) As such, there are no real tests in this PR:
interesting behaviors only really occur with a full end-to-end flow.

The integration with the CLI is a little awkward (we internally build
another `wasmtime run` command that invokes the debug component, and
tie it together with the debuggee via a special `invoke_debugger` API;
this seemed less bad than reworking all of the WASI setup to be more
reusable). Happy to take more ideas here.

* Review feedback.

* Review feedback.

* Review feedback: update vendor-wit.sh.

* Review feedback: -Ddebugger-arg= -> -Darg=.

* Review feedback.

* Review feedback.

* Review feedback: factor host.rs into several submodules.

* Review feedback: rename Debugger to Debuggee on host side.

* Review feedback: split inherit_stdin_stdout, and add corresponding options for the debug component.

* Review feedback.

* Review feedback.

* Add simple debug-component tests.

* Add wasm32-wasip2 target in a few places in CI

* Cargo vets for wstd dependency.

* Add wasm32-wasip2 in more places

* fix debug-component test dependence on componentization byte offsets

* Review feedback.

* Fix cancel-safety of EventFuture.

* Fix: Interrupted events should only occur after interrupt(), not on every epoch yield.

* Review feedback.

* Review feedback: strip down WASI imports in debugger world.

* fold debugger test component back into wasip1 + adapter test artifact compilation flow

Author: cfallin

.github/actions/install-rust/action.yml

@@ -79,4 +79,4 @@ runs:
 
     - name: Install the WASI target
       shell: bash
-      run: rustup target add wasm32-wasip1 wasm32-unknown-unknown
+      run: rustup target add wasm32-wasip2 wasm32-wasip1 wasm32-unknown-unknown

.github/workflows/main.yml

@@ -739,7 +739,7 @@ jobs:
         toolchain: ${{ matrix.rust }}
 
     # Install targets in order to build various tests throughout the repo
-    - run: rustup target add wasm32-wasip1 wasm32-unknown-unknown ${{ matrix.target }}
+    - run: rustup target add wasm32-wasip1 wasm32-unknown-unknown wasm32-wasip2 ${{ matrix.target }}
     - run: echo CARGO_BUILD_TARGET=${{ matrix.target }} >> $GITHUB_ENV
       if: matrix.target != ''
 
@@ -937,7 +937,7 @@ jobs:
       if: (runner.os == 'Windows') && (matrix.feature == 'winml')
 
     # Install Rust targets.
-    - run: rustup target add wasm32-wasip1
+    - run: rustup target add wasm32-wasip1 wasm32-wasip2
 
     # Run the tests!
     - run: cargo test -p wasmtime-wasi-nn --features ${{ matrix.feature }}
@@ -1009,7 +1009,7 @@ jobs:
         submodules: true
     - uses: ./.github/actions/install-rust
     - run: |
-        rustup target add wasm32-wasip1 wasm32-unknown-unknown
+        rustup target add wasm32-wasip2 wasm32-wasip1 wasm32-unknown-unknown
         cd /tmp
         curl --retry 5 --retry-all-errors -OL https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-30/wasi-sdk-30.0-x86_64-linux.tar.gz
         tar -xzf wasi-sdk-30.0-x86_64-linux.tar.gz
@@ -1038,7 +1038,7 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/actions/install-rust
-    - run: rustup target add wasm32-wasip1 wasm32-unknown-unknown
+    - run: rustup target add wasm32-wasip2 wasm32-wasip1 wasm32-unknown-unknown
 
     - name: Install wasm-tools
       run: |
@@ -1118,7 +1118,7 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/actions/install-rust
-    - run: rustup target add wasm32-wasip1
+    - run: rustup target add wasm32-wasip2 wasm32-wasip1
     - run: cargo test --benches --release
 
   # Verify that cranelift's code generation is deterministic

Cargo.lock

@@ -60,6 +60,7 @@ wasmtime-wasi-threads = { workspace = true, optional = true }
 wasmtime-wasi-http = { workspace = true, optional = true }
 wasmtime-unwinder = { workspace = true }
 wasmtime-wizer = { workspace = true, optional = true, features = ['clap', 'wasmtime'] }
+wasmtime-debugger = { workspace = true, optional = true }
 clap = { workspace = true }
 clap_complete = { workspace = true, optional = true }
 target-lexicon = { workspace = true }
@@ -565,7 +566,7 @@ gc-drc = ["gc", "wasmtime/gc-drc", "wasmtime-cli-flags/gc-drc"]
 gc-null = ["gc", "wasmtime/gc-null", "wasmtime-cli-flags/gc-null"]
 pulley = ["wasmtime-cli-flags/pulley"]
 stack-switching = ["wasmtime/stack-switching", "wasmtime-cli-flags/stack-switching"]
-debug = ["wasmtime-cli-flags/debug", "wasmtime/debug"]
+debug = ["wasmtime-cli-flags/debug", "wasmtime/debug", "component-model"]
 
 # CLI subcommands for the `wasmtime` executable. See `wasmtime $cmd --help`
 # for more information on each subcommand.
@@ -591,6 +592,7 @@ run = [
   "dep:tokio",
   "wasmtime-cli-flags/async",
   "wasmtime-wasi-http?/p2",
+  "dep:wasmtime-debugger",
 ]
 completion = ["dep:clap_complete"]
 objdump = [

Cargo.toml

@@ -90,6 +90,15 @@ wkg get --format wit --overwrite "wasi:random@$p3" -o "crates/wasi-http/src/p3/w
 wkg get --format wit --overwrite "wasi:sockets@$p3" -o "crates/wasi-http/src/p3/wit/deps/sockets.wit"
 wkg get --format wit --overwrite "wasi:http@$p3" -o "crates/wasi-http/src/p3/wit/deps/http.wit"
 
+rm -rf crates/debugger/wit/deps
+mkdir -p crates/debugger/wit/deps
+wkg get --format wit --overwrite "wasi:io@$p2" -o "crates/debugger/wit/deps/io.wit"
+wkg get --format wit --overwrite "wasi:clocks@$p2" -o "crates/debugger/wit/deps/clocks.wit"
+wkg get --format wit --overwrite "wasi:cli@$p2" -o "crates/debugger/wit/deps/cli.wit"
+wkg get --format wit --overwrite "wasi:filesystem@$p2" -o "crates/debugger/wit/deps/filesystem.wit"
+wkg get --format wit --overwrite "wasi:random@$p2" -o "crates/debugger/wit/deps/random.wit"
+wkg get --format wit --overwrite "wasi:sockets@$p2" -o "crates/debugger/wit/deps/sockets.wit"
+
 # wasi-nn is fetched separately since it's not in the standard WASI registry
 repo=https://raw.githubusercontent.com/WebAssembly/wasi-nn
 revision=0.2.0-rc-2024-10-28

ci/vendor-wit.sh

@@ -273,6 +273,22 @@ wasmtime_option_group! {
         pub log_to_files: Option<bool>,
         /// Enable coredump generation to this file after a WebAssembly trap.
         pub coredump: Option<String>,
+        /// Load the given debugger component and attach it to the
+        /// main module or component.
+        pub debugger: Option<PathBuf>,
+        /// Pass the given command-line arguments to the debugger
+        /// component. May be specified multiple times.
+        #[serde(default)]
+        pub arg: Vec<String>,
+        /// Allow the debugger component to inherit stdin.Off by
+        /// default.
+        pub inherit_stdin: Option<bool>,
+        /// Allow the debugger component to inherit stdout. Off by
+        /// default.
+        pub inherit_stdout: Option<bool>,
+        /// Allow the debugger component to inherit stderr. Off by
+        /// default.
+        pub inherit_stderr: Option<bool>,
     }
 
     enum Debug {
@@ -491,6 +507,12 @@ wasmtime_option_group! {
         ///
         /// This option can be further overwritten with `--env` flags.
         pub inherit_env: Option<bool>,
+        /// Inherit stdin from the parent process. On by default.
+        pub inherit_stdin: Option<bool>,
+        /// Inherit stdout from the parent process. On by default.
+        pub inherit_stdout: Option<bool>,
+        /// Inherit stderr from the parent process. On by default.
+        pub inherit_stderr: Option<bool>,
         /// Pass a wasi config variable to the program.
         #[serde(skip)]
         pub config_var: Vec<KeyValuePair>,

crates/cli-flags/src/lib.rs

@@ -9,6 +9,8 @@ use crate::{KeyValuePair, WasiNnGraph};
 use clap::builder::{StringValueParser, TypedValueParser, ValueParserFactory};
 use clap::error::{Error, ErrorKind};
 use serde::de::{self, Visitor};
+use std::path::PathBuf;
+use std::str::FromStr;
 use std::time::Duration;
 use std::{fmt, marker};
 use wasmtime::{Result, bail};
@@ -344,6 +346,20 @@ impl WasmtimeOptionValue for String {
     }
 }
 
+impl WasmtimeOptionValue for PathBuf {
+    const VAL_HELP: &'static str = "=path";
+    fn parse(val: Option<&str>) -> Result<Self> {
+        match val {
+            Some(val) => Ok(PathBuf::from_str(val)?),
+            None => bail!("value must be specified with key=val syntax"),
+        }
+    }
+
+    fn display(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{self:?}")
+    }
+}
+
 impl WasmtimeOptionValue for u32 {
     const VAL_HELP: &'static str = "=N";
     fn parse(val: Option<&str>) -> Result<Self> {

crates/cli-flags/src/opt.rs

@@ -12,9 +12,12 @@ rust-version.workspace = true
 workspace = true
 
 [dependencies]
-wasmtime = { workspace = true, features = ["debug", "std", "async"] }
+wasmtime = { workspace = true, features = ["debug", "std", "async", "component-model", "gc", "gc-drc"] }
+wasmtime-wasi = { workspace = true }
+wasmtime-wasi-io = { workspace = true }
 tokio = { workspace = true, features = ["rt", "sync", "macros"] }
 log = { workspace = true }
+async-trait = { workspace = true }
 
 [dev-dependencies]
 # Depend on `wasmtime` again to get `cranelift` and `wat` so we can

crates/debugger/Cargo.toml

@@ -0,0 +1,38 @@
+//! Host implementation for the debugger world.
+
+use wasmtime::{
+    Result,
+    component::{Resource, ResourceTable},
+};
+
+mod api;
+mod bindings;
+mod opaque;
+
+pub use api::Debuggee;
+pub use bindings::DebugMain as DebuggerComponent;
+pub use bindings::bytecodealliance::wasmtime::debuggee as wit;
+use opaque::OpaqueDebugger;
+
+/// Register a debuggee in a resource table.
+pub fn add_debuggee<T: Send + 'static>(
+    table: &mut ResourceTable,
+    debuggee: crate::Debuggee<T>,
+) -> Result<Resource<Debuggee>> {
+    let engine = debuggee.engine().clone();
+    let interrupt_pending = debuggee.interrupt_pending().clone();
+    let inner: Option<Box<dyn OpaqueDebugger + Send + 'static>> = Some(Box::new(debuggee));
+    Ok(table.push(Debuggee {
+        inner,
+        engine,
+        interrupt_pending,
+    })?)
+}
+
+/// Add the debugger world's host functions to a [`wasmtime::component::Linker`].
+pub fn add_to_linker<T: Send + 'static>(
+    linker: &mut wasmtime::component::Linker<T>,
+    f: fn(&mut T) -> &mut ResourceTable,
+) -> wasmtime::Result<()> {
+    wit::add_to_linker::<_, wasmtime::component::HasSelf<ResourceTable>>(linker, f)
+}