feat: allow extending hyperlight-js-runtime with custom native modules

Resolves #48

Add a registration-based system for extending the JS runtime with custom
native (Rust-implemented) modules that run inside the Hyperlight guest VM.

Key changes:

- hyperlight-js-runtime/Cargo.toml: Add [lib] target so the runtime can
  be used as a library dependency by extender crates.

- modules/mod.rs: Add global CUSTOM_MODULES registry with
  register_native_module() and builtin_module_names(). NativeModuleLoader
  checks custom registry first, falls back to built-in modules. Panics
  if custom module name conflicts with a built-in.

- native_modules! macro: Generates init_native_modules() (#[no_mangle])
  that registers custom modules. Called automatically via spin::Once on
  first NativeModuleLoader access — no explicit init needed.

- guest/mod.rs: Move hyperlight guest entry point (hyperlight_main,
  guest_dispatch_function, Host impl, stubs) from main/hyperlight.rs
  into the lib behind cfg(hyperlight). Extender binaries get all guest
  infrastructure for free by depending on the lib.

- hyperlight-js/build.rs: Add HYPERLIGHT_JS_RUNTIME_PATH env var override
  to embed custom runtime binaries instead of the default.

- host.rs: Restore original Host trait only (no FsHost extraction).

Testing:
- 13 unit/integration tests in hyperlight-js-runtime (loader, registry,
  macro, override prevention, E2E with native CLI fixture binary)
- 3 Hyperlight VM integration tests in hyperlight-js (#[ignore], run via
  just test-native-modules)
- Extended runtime fixture crate with shared native_math module
- just test-native-modules recipe for full hyperlight pipeline

Docs:
- docs/extending-runtime.md with quick start, host-side usage, native
  testing, API reference, and architecture diagram

Author: simongdavies

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = ["src/hyperlight-js", "src/js-host-api", "src/hyperlight-js-runtime"]
+exclude = ["src/hyperlight-js-runtime/tests/fixtures/extended_runtime"]
 
 [workspace.package]
 version = "0.1.1"

Justfile

@@ -136,6 +136,8 @@ clean:
     cargo clean
     cd src/hyperlight-js-runtime && cargo clean
     cd src/js-host-api && cargo clean
+    -rm -rf src/hyperlight-js-runtime/tests/fixtures/extended_runtime/target
+    -rm -rf src/hyperlight-js-runtime/tests/fixtures/native_math/target
     -rm -rf src/js-host-api/node_modules
     -rm -f src/js-host-api/*.node
     -rm -f src/js-host-api/index.js
@@ -151,15 +153,52 @@ test target=default-target features="": (build target)
     cd src/hyperlight-js && cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} handle_termination --profile={{ if target == "debug" {"dev"} else { target } }} -- --ignored --nocapture
     cd src/hyperlight-js && cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} test_metrics --profile={{ if target == "debug" {"dev"} else { target } }} -- --ignored --nocapture
     cargo test --manifest-path=./src/hyperlight-js-runtime/Cargo.toml --test=native_cli --profile={{ if target == "debug" {"dev"} else { target } }}
+    just test-native-modules {{ target }}
 
-# Test with monitor features enabled (wall-clock, CPU time, and guest-call-stats)
-# Note: We exclude test_metrics as it requires process isolation and is already run by `test` recipe
+# Test with monitor features enabled (wall-clock and CPU time monitors)
+# Note: We exclude test_metrics (requires process isolation, already run by `test`)
+# and native_modules (requires custom guest runtime, run by `test-native-modules`)
 test-monitors target=default-target:
-    cd src/hyperlight-js && cargo test --features monitor-wall-clock,monitor-cpu-time,guest-call-stats --profile={{ if target == "debug" {"dev"} else { target } }} -- --include-ignored --skip test_metrics
+    cd src/hyperlight-js && cargo test --features monitor-wall-clock,monitor-cpu-time --profile={{ if target == "debug" {"dev"} else { target } }} -- --include-ignored --skip test_metrics --skip custom_native_module --skip builtin_modules_work_with_custom --skip console_log_works_with_custom
 
 test-js-host-api target=default-target features="": (build-js-host-api target features)
     cd src/js-host-api && npm test
 
+# Test custom native modules:
+# 1. Runs the runtime crate's native_modules unit/pipeline tests (native binary)
+# 2. Builds the extended_runtime fixture for the hyperlight target
+# 3. Rebuilds hyperlight-js with the custom guest embedded via HYPERLIGHT_JS_RUNTIME_PATH
+# 4. Runs the ignored VM integration tests
+# 5. Rebuilds hyperlight-js with the default guest (unsets HYPERLIGHT_JS_RUNTIME_PATH)
+#
+# The build.rs in hyperlight-js has `cargo:rerun-if-env-changed=HYPERLIGHT_JS_RUNTIME_PATH`
+# so setting/unsetting the env var triggers a rebuild automatically.
+
+# Base path to the extended runtime fixture target directory
+extended_runtime_target := replace(justfile_dir(), "\\", "/") + "/src/hyperlight-js-runtime/tests/fixtures/extended_runtime/target/x86_64-hyperlight-none"
+
+test-native-modules target=default-target: (ensure-tools) (_test-native-modules-unit target) (_test-native-modules-build-guest target) (_test-native-modules-vm target) (_test-native-modules-restore target)
+
+[private]
+_test-native-modules-unit target=default-target:
+    cargo test --manifest-path=./src/hyperlight-js-runtime/Cargo.toml --test=native_modules --profile={{ if target == "debug" {"dev"} else { target } }}
+
+[private]
+_test-native-modules-build-guest target=default-target:
+    cargo hyperlight build \
+        --manifest-path src/hyperlight-js-runtime/tests/fixtures/extended_runtime/Cargo.toml \
+        --profile={{ if target == "debug" {"dev"} else { target } }} \
+        --target-dir src/hyperlight-js-runtime/tests/fixtures/extended_runtime/target
+
+[private]
+_test-native-modules-vm target=default-target:
+    {{ set-env-command }}HYPERLIGHT_JS_RUNTIME_PATH="{{extended_runtime_target}}/{{ if target == "debug" {"debug"} else { target } }}/extended-runtime" {{ if os() == "windows" { ";" } else { "&&" } }} cargo test -p hyperlight-js --test native_modules --profile={{ if target == "debug" {"dev"} else { target } }} -- --ignored --nocapture
+
+[private]
+_test-native-modules-restore target=default-target:
+    @echo "Rebuilding hyperlight-js with default guest runtime..."
+    cd src/hyperlight-js && cargo build --profile={{ if target == "debug" {"dev"} else { target } }}
+
 # Run js-host-api examples (simple.js, calculator.js, unload.js, interrupt.js, cpu-timeout.js, host-functions.js)
 run-js-host-api-examples target=default-target features="": (build-js-host-api target features)
     @echo "Running js-host-api examples..."

docs/extending-runtime.md

@@ -0,0 +1,209 @@
+# Extending the Runtime with Custom Native Modules
+
+This document describes how to extend `hyperlight-js-runtime` with custom
+native (Rust-implemented) modules that run alongside the built-in modules
+inside the Hyperlight guest VM.
+
+## Why Native Modules? 🤔
+
+Some operations are too slow in pure JavaScript. For example, DEFLATE
+compression can be 50–100× slower than native Rust, which may trigger CPU
+timeouts on large inputs. Native modules let you add high-performance Rust
+code that JavaScript handlers can `import` — without forking the runtime.
+
+## How It Works
+
+1. **`hyperlight-js-runtime` as a library** — the runtime crate exposes a
+   `[lib]` target so your crate can depend on it.
+2. **`native_modules!` macro** — registers custom modules into a global
+   registry. The runtime's `NativeModuleLoader` checks custom modules
+   first, then falls back to built-ins (io, crypto, console, require).
+3. **`HYPERLIGHT_JS_RUNTIME_PATH`** — a build-time env var that tells
+   `hyperlight-js` to embed your custom runtime binary instead of the
+   default one.
+
+## Quick Start
+
+### 1. Create your custom runtime crate
+
+```bash
+cargo init --bin my-custom-runtime
+```
+
+```toml
+[dependencies]
+hyperlight-js-runtime = { git = "https://github.com/hyperlight-dev/hyperlight-js" }
+rquickjs = { version = "0.11", default-features = false, features = ["bindgen", "futures", "macro", "loader"] }
+
+# Only needed for native CLI testing, not the hyperlight guest
+[target.'cfg(not(hyperlight))'.dependencies]
+anyhow = "1.0"
+
+[lints.rust]
+unexpected_cfgs = { level = "allow", check-cfg = ['cfg(hyperlight)'] }
+```
+
+> **Note:** The `rquickjs` version and features must match what
+> `hyperlight-js-runtime` uses. Check its `Cargo.toml` for the exact spec.
+
+### 2. Define your module and register it
+
+```rust
+#![cfg_attr(hyperlight, no_std)]
+#![cfg_attr(hyperlight, no_main)]
+
+#[rquickjs::module(rename_vars = "camelCase")]
+mod math {
+    #[rquickjs::function]
+    pub fn add(a: f64, b: f64) -> f64 { a + b }
+
+    #[rquickjs::function]
+    pub fn multiply(a: f64, b: f64) -> f64 { a * b }
+}
+
+hyperlight_js_runtime::native_modules! {
+    "math" => js_math,
+}
+```
+
+That's all the Rust you write for the Hyperlight guest. The macro generates
+an `init_native_modules()` function that the `NativeModuleLoader` calls
+automatically on first use. Built-in modules are inherited. The lib provides
+all hyperlight guest infrastructure (entry point, host function dispatch,
+libc stubs) — no copying files or build scripts needed.
+
+### 3. Build and embed in hyperlight-js
+
+Build your custom runtime for the Hyperlight target and embed it:
+
+```bash
+# Build for the hyperlight target
+cargo hyperlight build --manifest-path my-custom-runtime/Cargo.toml
+
+# Build hyperlight-js with your custom runtime embedded
+HYPERLIGHT_JS_RUNTIME_PATH=/path/to/my-custom-runtime \
+    cargo build -p hyperlight-js
+```
+
+### 4. Use from the host
+
+The host-side code is **identical** to any other `hyperlight-js` usage.
+Custom native modules are transparent — they're baked into the guest
+binary. Your handlers just `import` from them:
+
+```rust
+use hyperlight_js::{SandboxBuilder, Script};
+
+fn main() -> anyhow::Result<()> {
+    let proto = SandboxBuilder::new().build()?;
+    let mut sandbox = proto.load_runtime()?;
+
+    let handler = Script::from_content(r#"
+        import { add, multiply } from "math";
+        export function handler(event) {
+            return {
+                sum: add(event.a, event.b),
+                product: multiply(event.a, event.b),
+            };
+        }
+    "#);
+    sandbox.add_handler("compute", handler)?;
+
+    let mut loaded = sandbox.get_loaded_sandbox()?;
+    let result = loaded.handle_event("compute", r#"{"a":6,"b":7}"#.to_string(), None)?;
+
+    println!("{result}");
+    // {"sum":13,"product":42}
+
+    Ok(())
+}
+```
+
+### 5. Test natively (optional)
+
+For local development you can run your custom runtime as a native CLI
+without building for Hyperlight. Add a `main()` to your `main.rs`.
+
+Since your custom modules are registered via the macro (and built-ins are
+handled by the runtime), you don't need filesystem module resolution (But you can have it if you want it).
+A no-op `Host` is all that's needed — it only gets called for `.js` file
+imports, which native modules don't use:
+
+```rust
+struct NoOpHost;
+impl hyperlight_js_runtime::host::Host for NoOpHost {
+    fn resolve_module(&self, _base: String, name: String) -> anyhow::Result<String> {
+        anyhow::bail!("Module '{name}' not found")
+    }
+    fn load_module(&self, name: String) -> anyhow::Result<String> {
+        anyhow::bail!("Module '{name}' not found")
+    }
+}
+
+fn main() -> anyhow::Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+    let script = std::fs::read_to_string(&args[1])?;
+
+    let mut runtime = hyperlight_js_runtime::JsRuntime::new(NoOpHost)?;
+    runtime.register_handler("handler", script, ".")?;
+    let result = runtime.run_handler("handler".into(), args[2].clone(), false)?;
+    println!("{result}");
+    Ok(())
+}
+```
+
+```bash
+# handler.js
+cat > handler.js << 'EOF'
+import { add, multiply } from "math";
+export function handler(event) {
+    return { sum: add(event.a, event.b), product: multiply(event.a, event.b) };
+}
+EOF
+
+cargo run -- handler.js '{"a":6,"b":7}'
+# {"sum":13,"product":42}
+```
+
+## Complete Example
+
+See the [extended_runtime fixture](../src/hyperlight-js-runtime/tests/fixtures/extended_runtime/)
+for a working example with end-to-end tests.
+
+Run `just test-native-modules` to build the fixture for the Hyperlight
+target and run the full integration tests.
+
+## API Reference
+
+### `native_modules!`
+
+```rust
+hyperlight_js_runtime::native_modules! {
+    "module_name" => ModuleDefType,
+    "another"     => AnotherModuleDefType,
+}
+```
+
+Generates an `init_native_modules()` function that registers the listed
+modules into the global native module registry. Called automatically by the
+`NativeModuleLoader` on first use — you never need to call it yourself.
+Built-in modules are inherited automatically.
+
+**Restrictions:**
+- Custom module names **cannot** shadow built-in modules (`io`, `crypto`,
+  `console`, `require`). Attempting to register a built-in name panics.
+
+### `register_native_module`
+
+```rust
+hyperlight_js_runtime::modules::register_native_module(name, declaration_fn)
+```
+
+Register a single custom native module by name. Typically called via the
+`native_modules!` macro rather than directly.
+
+### `JsRuntime::new`
+
+```rust
+hyperlight_js_runtime::JsRuntime::new(host)
+```

src/hyperlight-js-runtime/Cargo.toml

@@ -7,11 +7,16 @@ license.workspace = true
 repository.workspace = true
 readme.workspace = true
 description = """
-hyperlight-js-runtime is a rust binary crate that provides the JavaScript runtime binary for hyperlight-js.
+hyperlight-js-runtime provides the JavaScript runtime for hyperlight-js, both as a library and binary.
 """
 
+[lib]
+name = "hyperlight_js_runtime"
+path = "src/lib.rs"
+
 [[bin]]
 name = "hyperlight-js-runtime"
+path = "src/main.rs"
 harness = false
 test = false
 

…rlight-js-runtime/src/main/hyperlight.rs …c/hyperlight-js-runtime/src/guest/mod.rssrc/hyperlight-js-runtime/src/main/hyperlight.rs renamed to src/hyperlight-js-runtime/src/guest/mod.rs src/hyperlight-js-runtime/src/main/hyperlight.rs renamed to src/hyperlight-js-runtime/src/guest/mod.rs

@@ -13,11 +13,23 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-extern crate alloc;
+
+//! Hyperlight guest entry point and infrastructure.
+//!
+//! This module provides the guest-side plumbing needed to run the JS runtime
+//! inside a Hyperlight VM. It includes:
+//! - The `Host` implementation that calls out to hyperlight host functions
+//! - The `hyperlight_main` entry point
+//! - Guest function registrations (register_handler, RegisterHostModules)
+//! - The `guest_dispatch_function` fallback for handler calls
+//! - Libc stub implementations required by QuickJS
+//!
+//! This is all `cfg(hyperlight)` — compiled out entirely for native builds.
 
 use alloc::format;
 use alloc::string::String;
 use alloc::vec::Vec;
+
 use anyhow::{anyhow, Context as _};
 use hashbrown::HashMap;
 use hyperlight_common::flatbuffer_wrappers::function_call::FunctionCall;
@@ -33,7 +45,7 @@ mod stubs;
 
 struct Host;
 
-pub trait CatchGuestErrorExt {
+trait CatchGuestErrorExt {
     type Ok;
     fn catch(self) -> anyhow::Result<Self::Ok>;
 }
@@ -45,7 +57,7 @@ impl<T> CatchGuestErrorExt for hyperlight_guest::error::Result<T> {
     }
 }
 
-impl hyperlight_js_runtime::host::Host for Host {
+impl crate::host::Host for Host {
     fn resolve_module(&self, base: String, name: String) -> anyhow::Result<String> {
         #[host_function("ResolveModule")]
         fn resolve_module(base: String, name: String) -> Result<String>;
@@ -65,17 +77,16 @@ impl hyperlight_js_runtime::host::Host for Host {
     }
 }
 
-static RUNTIME: spin::Lazy<Mutex<hyperlight_js_runtime::JsRuntime>> = spin::Lazy::new(|| {
-    Mutex::new(hyperlight_js_runtime::JsRuntime::new(Host).unwrap_or_else(|e| {
+static RUNTIME: spin::Lazy<Mutex<crate::JsRuntime>> = spin::Lazy::new(|| {
+    Mutex::new(crate::JsRuntime::new(Host).unwrap_or_else(|e| {
         panic!("Failed to initialize JS runtime: {e:#?}");
     }))
 });
 
 #[unsafe(no_mangle)]
 #[instrument(skip_all, level = "info")]
 pub extern "C" fn hyperlight_main() {
-    // dereference RUNTIME to force its initialization
-    // of the Lazy static
+    // Initialise the runtime (custom modules are registered lazily on first use)
     let _ = &*RUNTIME;
 }