Change wasmtime wast to be powered from JSON AST (#11113)

* Change `wasmtime wast` to be powered from JSON AST

This commit is a refactoring of the `wasmtime-wast` crate to use the
`json-from-wast` crate added in bytecodealliance/wasm-tools#2247 instead
of the `wast` crate directly. The primary motivation for this PR is to
make spec tests more suitable for running inside of Miri. There are two
primary factors in how Miri is slow with wast tests today:

* Compiling WebAssembly modules. These are all embedded throughout
  `*.wast` files and basically need to be precompiled to run in Miri.

* Parsing the `*.wast` script itself. The scripts are generally quite
  large in the upstream spec test suite and parsing the script requires
  parsing all modules as well, so this can take quite some time.

The goal of this commit was to leverage `json-from-wast` to perform much
of the heavy lifting on the host and then have a "cheap" parse step in
Miri which deserializes the JSON and runs precompiled `*.cwasm` files.
The main change then required of `wasmtime-wast` was to use the JSON AST
as its "IR" instead of `wast` directly. The actual transformation of the
`wasmtime-wast` crate isn't too bad, mostly just some refactorings here
and there.

A new `./ci/miri-wast.sh` script is added which first runs on native
with `wasmtime wast --precompile-save` and then runs in Miri with
`wasmtime wast --precompile-load`. In this manner I was able to get a
number of spec test `*.wast` files running in Miri.

Unfortunately, though, Miri is still far too slow to run in CI. One
major bottleneck is that the `*.json` files are pretty large and take a
nontrivial amount of time to parse. Another issue is that these tests
run a fair bit of code which with Miri's ~million-x slowdown. For the
first problem I tried using other more-binary serialization formats but
the JSON AST is unfortunately not compatible with many of them (e.g.
`postcard`, which we use for Wasmtime, is not compatible with the JSON
AST's structure in Rust types). For the latter I'm not sure what to
do...

In the end I figured this might be a reasonable refactoring to go ahead
and land anyway. It at least enables running `*.wast` tests in Miri
while removing a large part of the runtime slowdown. We can in theory
still allow-list some tests to run as they don't all take forever. Some
future breakthrough is still going to be required though to run
everything through Miri.

* Document usage

* Adjust lint

* Fix CI issues

Author: alexcrichton

.github/workflows/main.yml

@@ -1026,6 +1026,7 @@ jobs:
           - crate: "wasmtime-environ --all-features"
           - crate: "pulley-interpreter --all-features"
           - script: ./ci/miri-provenance-test.sh
+          - script: ./ci/miri-wast.sh ./tests/spec_testsuite/table.wast
     needs: determine
     if: needs.determine.outputs.test-miri && github.repository == 'bytecodealliance/wasmtime'
     name: Miri

Cargo.lock

@@ -339,6 +339,7 @@ wit-parser = "0.236.0"
 wit-component = "0.236.0"
 wasm-wave = "0.236.0"
 wasm-compose = "0.236.0"
+json-from-wast = "0.236.0"
 
 # Non-Bytecode Alliance maintained dependencies:
 # --------------------------

Cargo.toml

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Helper script to execute a `*.wast` test in Miri. This is only lightly used on
+# CI and is provided here to assist with development of anything that ends up
+# using unsafe for example.
+#
+# Example usage is:
+#
+#   ./ci/miri-wast.sh ./tests/spec_testsuite/br_if.wast
+#
+# extra flags to this script are passed to `cargo run wast` which means they
+# must be suitable flags for the `wast` subcommand.
+
+set -ex
+
+rm -rf ./miri-wast
+mkdir ./miri-wast
+cargo run -- wast --target pulley64 --precompile-save ./miri-wast "$@" \
+  -O memory-reservation=$((1 << 20)) \
+  -O memory-guard-size=0 \
+  -O signals-based-traps=n \
+  -O memory-init-cow=n
+
+MIRIFLAGS="$MIRIFLAGS -Zmiri-disable-isolation -Zmiri-permissive-provenance" \
+  cargo miri run -- wast -Ccache=n --target pulley64 --precompile-load ./miri-wast "$@" \
+  -O memory-init-cow=n

ci/miri-wast.sh

@@ -26,7 +26,7 @@ pub use imp::*;
 
 /// How to initialize a newly-allocated array's elements.
 #[derive(Clone, Copy)]
-#[cfg_attr(not(any(feature = "gc-null", feature = "gc-drc")), allow(dead_code))]
+#[cfg_attr(not(feature = "gc"), expect(dead_code))]
 pub enum ArrayInit<'a> {
     /// Initialize the array's elements with the given values.
     Elems(&'a [ir::Value]),

crates/cranelift/src/func_environ/gc.rs

@@ -58,7 +58,10 @@ pub fn gc_compiler(func_env: &mut FuncEnvironment<'_>) -> WasmResult<Box<dyn GcC
     }
 }
 
-#[cfg_attr(not(feature = "gc-drc"), allow(dead_code))]
+#[cfg_attr(
+    not(feature = "gc-drc"),
+    expect(dead_code, reason = "easier to define")
+)]
 fn unbarriered_load_gc_ref(
     builder: &mut FunctionBuilder,
     ty: WasmHeapType,
@@ -73,7 +76,10 @@ fn unbarriered_load_gc_ref(
     Ok(gc_ref)
 }
 
-#[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))]
+#[cfg_attr(
+    not(any(feature = "gc-drc", feature = "gc-null")),
+    expect(dead_code, reason = "easier to define")
+)]
 fn unbarriered_store_gc_ref(
     builder: &mut FunctionBuilder,
     ty: WasmHeapType,
@@ -461,7 +467,10 @@ pub fn translate_array_new_fixed(
 
 impl ArrayInit<'_> {
     /// Get the length (as an `i32`-typed `ir::Value`) of these array elements.
-    #[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))]
+    #[cfg_attr(
+        not(any(feature = "gc-drc", feature = "gc-null")),
+        expect(dead_code, reason = "easier to define")
+    )]
     fn len(self, pos: &mut FuncCursor) -> ir::Value {
         match self {
             ArrayInit::Fill { len, .. } => len,
@@ -473,7 +482,10 @@ impl ArrayInit<'_> {
     }
 
     /// Initialize a newly-allocated array's elements.
-    #[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))]
+    #[cfg_attr(
+        not(any(feature = "gc-drc", feature = "gc-null")),
+        expect(dead_code, reason = "easier to define")
+    )]
     fn initialize(
         self,
         func_env: &mut FuncEnvironment<'_>,
@@ -1140,7 +1152,10 @@ fn uextend_i32_to_pointer_type(
 /// in its initialization.
 ///
 /// Traps if the size overflows.
-#[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))]
+#[cfg_attr(
+    not(any(feature = "gc-drc", feature = "gc-null")),
+    expect(dead_code, reason = "easier to define")
+)]
 fn emit_array_size(
     func_env: &mut FuncEnvironment<'_>,
     builder: &mut FunctionBuilder<'_>,
@@ -1185,7 +1200,10 @@ fn emit_array_size(
 
 /// Common helper for struct-field initialization that can be reused across
 /// collectors.
-#[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))]
+#[cfg_attr(
+    not(any(feature = "gc-drc", feature = "gc-null")),
+    expect(dead_code, reason = "easier to define")
+)]
 fn initialize_struct_fields(
     func_env: &mut FuncEnvironment<'_>,
     builder: &mut FunctionBuilder<'_>,
@@ -1386,7 +1404,10 @@ impl FuncEnvironment<'_> {
     /// reference is null or is an `i31ref`; otherwise, it will be zero.
     ///
     /// This method is collector-agnostic.
-    #[cfg_attr(not(feature = "gc-drc"), allow(dead_code))]
+    #[cfg_attr(
+        not(feature = "gc-drc"),
+        expect(dead_code, reason = "easier to define")
+    )]
     fn gc_ref_is_null_or_i31(
         &mut self,
         builder: &mut FunctionBuilder,

crates/cranelift/src/func_environ/gc/enabled.rs

@@ -764,7 +764,7 @@ pub fn wast_test(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<()> {
         })
         .unwrap();
     wast_context
-        .run_buffer(test.path.to_str().unwrap(), test.contents.as_bytes())
+        .run_wast(test.path.to_str().unwrap(), test.contents.as_bytes())
         .unwrap();
     Ok(())
 }

crates/fuzzing/src/oracles.rs

@@ -19,6 +19,10 @@ wasmtime = { workspace = true, features = ['cranelift', 'wat', 'runtime', 'gc',
 wast = { workspace = true, features = ['dwarf'] }
 log = { workspace = true }
 tokio = { workspace = true, features = ['rt'] }
+json-from-wast = { workspace = true }
+wasmparser = { workspace = true }
+serde_json = { workspace = true }
+object = { workspace = true, features = ['unaligned'] }
 
 [features]
 component-model = ['wasmtime/component-model']