Add support for the Linux PAGEMAP_SCAN ioctl (#11372)

* WIP: use the pagemap_scan ioctl to selectively reset an instance's dirty pages

* Less hacky, and supporting tables, too

* Bugfixes

* WIP: memcpy instead of pread

* Refactor support for pagemap

* Don't hold a raw pointer to the original data, plumb through an `Arc`
  to have a safe reference instead.
* Update pagemap bindings to latest version of abstraction written.
* Include pagemap-specific tests.
* Use a `PageMap` structure created once-per-pool instead of a
  static-with-a-file.
* Refactor to use the "pagemap path" unconditionally which blends in the
  keep_resident bits.

* Improve safety documentation

prtest:full

* Fix some lints

* Skip ioctl tests when it's not supported

* Fix a memory leak by moving impls around

* Fix no vm build

* Review comments

* Add more pagemap-specific documentation

* Add more docs, refactor implementation slightly

* Improve `category_*` docs

Basically forward to the Linux kernel source itself.

* Fix compile

* Make pagemap integration resilient across forks

* Fix non-pooling-allocator-build

* Fix portability issues of new test

* Actually use config on macos

---------

Co-authored-by: Till Schneidereit <till@tillschneidereit.net>

Author: alexcrichton

crates/wasmtime/Cargo.toml

@@ -112,6 +112,10 @@ wasmtime-versioned-export-macros = { workspace = true, optional = true }
 name = "host_segfault"
 harness = false
 
+[[test]]
+name = "engine_across_forks"
+harness = false
+
 # =============================================================================
 #
 # Features for the Wasmtime crate.

crates/wasmtime/src/runtime/module.rs

@@ -1,7 +1,7 @@
 use crate::prelude::*;
 #[cfg(feature = "std")]
 use crate::runtime::vm::open_file_for_mmap;
-use crate::runtime::vm::{CompiledModuleId, ModuleMemoryImages, VMWasmCallFunction};
+use crate::runtime::vm::{CompiledModuleId, MmapVec, ModuleMemoryImages, VMWasmCallFunction};
 use crate::sync::OnceLock;
 use crate::{
     Engine,
@@ -1114,7 +1114,7 @@ impl Module {
         let images = self
             .inner
             .memory_images
-            .get_or_try_init(|| memory_images(&self.inner.engine, &self.inner.module))?
+            .get_or_try_init(|| memory_images(&self.inner))?
             .as_ref();
         Ok(images)
     }
@@ -1164,21 +1164,30 @@ fn _assert_send_sync() {
 
 /// Helper method to construct a `ModuleMemoryImages` for an associated
 /// `CompiledModule`.
-fn memory_images(engine: &Engine, module: &CompiledModule) -> Result<Option<ModuleMemoryImages>> {
+fn memory_images(inner: &Arc<ModuleInner>) -> Result<Option<ModuleMemoryImages>> {
     // If initialization via copy-on-write is explicitly disabled in
     // configuration then this path is skipped entirely.
-    if !engine.tunables().memory_init_cow {
+    if !inner.engine.tunables().memory_init_cow {
         return Ok(None);
     }
 
     // ... otherwise logic is delegated to the `ModuleMemoryImages::new`
     // constructor.
-    let mmap = if engine.config().force_memory_init_memfd {
-        None
-    } else {
-        Some(module.mmap())
-    };
-    ModuleMemoryImages::new(module.module(), module.code_memory().wasm_data(), mmap)
+    ModuleMemoryImages::new(
+        &inner.engine,
+        inner.module.module(),
+        inner.code.code_memory(),
+    )
+}
+
+impl crate::vm::ModuleMemoryImageSource for CodeMemory {
+    fn wasm_data(&self) -> &[u8] {
+        <Self>::wasm_data(self)
+    }
+
+    fn mmap(&self) -> Option<&MmapVec> {
+        Some(<Self>::mmap(self))
+    }
 }
 
 #[cfg(test)]

crates/wasmtime/src/runtime/vm.rs

@@ -60,6 +60,8 @@ mod imports;
 mod instance;
 mod memory;
 mod mmap_vec;
+#[cfg(has_virtual_memory)]
+mod pagemap_disabled;
 mod provenance;
 mod send_sync_ptr;
 mod stack_switching;
@@ -161,6 +163,17 @@ cfg_if::cfg_if! {
     }
 }
 
+/// Source of data used for [`MemoryImage`]
+pub trait ModuleMemoryImageSource: Send + Sync + 'static {
+    /// Returns this image's slice of all wasm data for a module which is then
+    /// further sub-sliced for a particular initialization segment.
+    fn wasm_data(&self) -> &[u8];
+
+    /// Optionally returns the backing mmap. Used for using the backing mmap's
+    /// file to perform other mmaps, for example.
+    fn mmap(&self) -> Option<&MmapVec>;
+}
+
 /// Dynamic runtime functionality needed by this crate throughout the execution
 /// of a wasm instance.
 ///