feat(packages): package projection in sidecar + agentos-toolchain, registry {name,dir} migration, single base-filesystem

- Move package projection into the secure-exec sidecar (ConfigureVm.packages + LinkPackage wire request); clients forward, no client-side staging
- agentos-toolchain (pack) now lives in secure-exec; flat --out, --bundle removed, native-addon error names --prune-native
- Remove agentos-package.json manifest (package.json "bin" is the source); npm-shippable output (no bin/ symlinks)
- Migrate all registry software+agent packages to the {name,dir} descriptor with a clean dist/package build dir
- Rename @secure-exec/registry-types -> @agentos-software/manifest
- Remove file-system registry packages (s3/google-drive mounts are core)
- Single base-filesystem.json (direct include_str!, no build.rs); python3 shebang -> Pyodide file dispatch

Author: NathanFlurry

CLAUDE.md

@@ -60,7 +60,7 @@ Every bound that protects a shared resource — memory/heap, CPU/wall-clock, fd/
 ## Build And Assets
 
 - The VM base filesystem artifact is derived from Alpine Linux, but runtime source should stay generic.
-- Rebuild the base filesystem with `pnpm --dir packages/build-tools snapshot:alpine-defaults`, then `pnpm --dir packages/build-tools build:base-filesystem`.
+- Rebuild the base filesystem (requires Docker) with `pnpm --dir packages/build-tools build:base-filesystem`. The one script snapshots Alpine, applies the secure-exec transforms, and writes the single canonical `packages/core/fixtures/base-filesystem.json`, mirroring the same bytes into the crate-vendored `crates/sidecar/assets/` and `crates/vfs/assets/` copies (those exist only as the `cargo publish` fallback; never hand-edit them).
 - The V8 bridge bundle is generated from `packages/build-tools/scripts/build-v8-bridge.mjs`; keep its generated assets aligned with bridge-contract changes.
 - `registry/native` owns the Rust-to-WASM command build; package-local `registry/software/*/wasm/` output is release material.
 

crates/execution/src/javascript.rs

@@ -1808,12 +1808,26 @@ impl JavascriptExecutionEngine {
 
         // Build user code: prefer inline code, fall back to entrypoint-based
         let translator = GuestPathTranslator::from_request(&request);
-        let host_entrypoint = translator.resolve_host_entrypoint(&request.cwd, &request.argv[0]);
-        let guest_entrypoint = if request.argv[0] == "-e" || request.argv[0] == "--eval" {
+        let mut host_entrypoint =
+            translator.resolve_host_entrypoint(&request.cwd, &request.argv[0]);
+        let mut guest_entrypoint = if request.argv[0] == "-e" || request.argv[0] == "--eval" {
             request.argv[0].clone()
         } else {
             translator.host_to_guest_string(&host_entrypoint)
         };
+        // Run an `/opt/agentos` projected command from its REALPATH: the
+        // `/opt/agentos/bin/<cmd>` entrypoint is a symlink into the package, so
+        // following it (like Node's default `preserveSymlinks=false`) makes the
+        // runtime read the real file — its `import`s resolve against the package's
+        // own `node_modules` and its module mode is read from the real
+        // `package.json`. Scoped to `/opt/agentos` so the legacy
+        // `/root/node_modules` (pnpm) flow is untouched.
+        if guest_entrypoint.starts_with("/opt/agentos/") {
+            if let Ok(canonical) = std::fs::canonicalize(&host_entrypoint) {
+                guest_entrypoint = translator.host_to_guest_string(&canonical);
+                host_entrypoint = canonical;
+            }
+        }
         let process_argv = if matches!(guest_entrypoint.as_str(), "-e" | "--eval") {
             std::iter::once(String::from("node"))
                 .chain(request.argv.iter().skip(1).cloned())
@@ -1834,12 +1848,21 @@ impl JavascriptExecutionEngine {
                 .is_some_and(inline_code_uses_module_mode);
         if !matches!(guest_entrypoint.as_str(), "-e" | "--eval") && !use_module_mode {
             if let Some(inline_code) = inline_code.as_ref() {
-                if let Some(parent) = host_entrypoint.parent() {
-                    fs::create_dir_all(parent)
+                // Only materialize the import cache for a SYNTHESIZED entrypoint
+                // (no file on disk). Never overwrite an existing entrypoint: it may
+                // be a read-only mounted package command (e.g. `/opt/agentos/bin/*`),
+                // and writing the shebang-stripped `inline_code` over it corrupts
+                // the file so a second exec sees no `#!` and fails with ENOEXEC.
+                // `require()` strips the shebang in-process, so the on-disk file
+                // keeps its shebang and stays re-executable.
+                if !host_entrypoint.exists() {
+                    if let Some(parent) = host_entrypoint.parent() {
+                        fs::create_dir_all(parent)
+                            .map_err(JavascriptExecutionError::PrepareImportCache)?;
+                    }
+                    fs::write(&host_entrypoint, inline_code)
                         .map_err(JavascriptExecutionError::PrepareImportCache)?;
                 }
-                fs::write(&host_entrypoint, inline_code)
-                    .map_err(JavascriptExecutionError::PrepareImportCache)?;
             }
         }
         let user_code = if matches!(guest_entrypoint.as_str(), "-e" | "--eval") {
@@ -2210,9 +2233,16 @@ fn build_v8_user_code(entrypoint: &str, env: &BTreeMap<String, String>) -> Strin
 }
 
 fn host_entrypoint_uses_module_mode(entrypoint: &Path) -> bool {
-    match entrypoint.extension().and_then(|ext| ext.to_str()) {
+    // Follow symlinks before deciding: an `/opt/agentos/bin/<cmd>` command is an
+    // extensionless symlink into the package (e.g. → `.../dist/adapter.js`), so
+    // the real file's extension AND its nearest `package.json` "type" must be read
+    // from the resolved target, not the symlink. Without this, an ESM agent
+    // adapter loads as CommonJS and crashes with "Cannot use import statement
+    // outside a module".
+    let resolved = std::fs::canonicalize(entrypoint).unwrap_or_else(|_| entrypoint.to_path_buf());
+    match resolved.extension().and_then(|ext| ext.to_str()) {
         Some("mjs" | "mts") => true,
-        Some("js") => nearest_package_json_type(entrypoint).as_deref() == Some("module"),
+        Some("js") => nearest_package_json_type(&resolved).as_deref() == Some("module"),
         _ => false,
     }
 }
@@ -3452,16 +3482,20 @@ impl<'a, R: ModuleFsReader> ModuleResolver<'a, R> {
         }
 
         let source = self.reader.read_to_string(path)?;
-        Some(
-            if matches!(
-                Path::new(path).extension().and_then(|ext| ext.to_str()),
-                Some("js" | "mjs" | "cjs")
-            ) {
-                strip_javascript_hashbang(&source)
-            } else {
-                source
-            },
-        )
+        // Strip a leading `#!` shebang for JS modules AND for any extensionless
+        // executable entrypoint that carries one (e.g. a `/opt/agentos/bin/*`
+        // package command named without a `.js` suffix). Node strips shebangs
+        // regardless of extension; `strip_javascript_hashbang` is a no-op when the
+        // source has none, so this never affects shebang-free files.
+        let has_js_extension = matches!(
+            Path::new(path).extension().and_then(|ext| ext.to_str()),
+            Some("js" | "mjs" | "cjs")
+        );
+        Some(if has_js_extension || source.starts_with("#!") {
+            strip_javascript_hashbang(&source)
+        } else {
+            source
+        })
     }
 
     pub fn module_format(&mut self, path: &str) -> Option<LocalResolvedModuleFormat> {
@@ -6030,6 +6064,7 @@ fn builtin_named_exports(module_name: &str) -> &'static [&'static str] {
             "chmodSync",
             "closeSync",
             "constants",
+            "copyFileSync",
             "createReadStream",
             "createWriteStream",
             "existsSync",

crates/execution/src/lib.rs

@@ -34,7 +34,8 @@ pub use secure_exec_bridge::GuestRuntime;
 pub use secure_exec_v8_runtime::execution::GuestModuleReader;
 pub use signal::{NodeSignalDispositionAction, NodeSignalHandlerRegistration};
 pub use wasm::{
-    CreateWasmContextRequest, NativeBinaryFormat, StartWasmExecutionRequest, WasmContext,
+    detect_native_binary_format, CreateWasmContextRequest, NativeBinaryFormat,
+    StartWasmExecutionRequest, WasmContext,
     WasmExecution, WasmExecutionEngine, WasmExecutionError, WasmExecutionEvent,
     WasmExecutionLimits, WasmExecutionResult, WasmPermissionTier,
 };

crates/execution/src/wasm.rs

@@ -55,7 +55,7 @@ const DEFAULT_WASM_GUEST_HOME: &str = "/root";
 const DEFAULT_WASM_GUEST_USER: &str = "root";
 const DEFAULT_WASM_GUEST_SHELL: &str = "/bin/sh";
 const DEFAULT_WASM_GUEST_PATH: &str =
-    "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+    "/usr/local/sbin:/usr/local/bin:/opt/agentos/bin:/usr/sbin:/usr/bin:/sbin:/bin";
 // Warmup is a best-effort compile-cache optimization; fall back to a cold start
 // instead of burning minutes on a stalled prewarm session.
 const DEFAULT_WASM_PREWARM_TIMEOUT_MS: u64 = 30_000;
@@ -5134,7 +5134,7 @@ fn verify_wasm_module_header(
     })
 }
 
-fn detect_native_binary_format(header: &[u8]) -> Option<NativeBinaryFormat> {
+pub fn detect_native_binary_format(header: &[u8]) -> Option<NativeBinaryFormat> {
     if header.len() >= 4 && &header[..4] == b"\x7fELF" {
         return Some(NativeBinaryFormat::Elf);
     }

crates/kernel/src/command_registry.rs

@@ -83,13 +83,6 @@ impl CommandRegistry {
             .collect()
     }
 
-    pub fn populate_bin<F>(&self, vfs: &mut F) -> VfsResult<()>
-    where
-        F: VirtualFileSystem,
-    {
-        self.populate_commands(vfs, self.commands.keys())
-    }
-
     pub fn populate_driver_bin<F>(&self, vfs: &mut F, driver: &CommandDriver) -> VfsResult<()>
     where
         F: VirtualFileSystem,

crates/kernel/src/kernel.rs

@@ -3105,19 +3105,8 @@ impl<F: VirtualFileSystem + 'static> KernelVm<F> {
             return Err(KernelError::command_not_found(command));
         };
 
-        if let Some(registered_command) = self.resolve_registered_command_path(&path) {
-            let driver = self
-                .commands
-                .resolve(&registered_command)
-                .cloned()
-                .ok_or_else(|| KernelError::command_not_found(&registered_command))?;
-            return Ok(ResolvedSpawnCommand {
-                command: registered_command,
-                args: args.to_vec(),
-                driver,
-            });
-        }
-
+        // A resolved file is executed by its header (Linux binfmt), not mapped
+        // back to a registered command name via hardcoded /bin prefixes.
         let shebang = self
             .parse_shebang_command(&path)?
             .ok_or_else(|| KernelError::new("ENOEXEC", format!("exec format error: {path}")))?;
@@ -3129,7 +3118,24 @@ impl<F: VirtualFileSystem + 'static> KernelVm<F> {
         command: &str,
         cwd: &str,
     ) -> KernelResult<Option<String>> {
+        // A command with no slash is resolved by a real `$PATH` walk (Linux
+        // `execvp`): the first PATH entry holding an executable file wins; a
+        // present-but-non-executable match is skipped (so the search may end in
+        // ENOENT, not EACCES). An empty PATH element means the cwd. This is what
+        // makes `/opt/agentos/bin` packages resolvable by bare name.
         if !command.contains('/') {
+            let path_env = self.env.get("PATH").cloned().unwrap_or_default();
+            for entry in path_env.split(':') {
+                let dir = if entry.is_empty() { cwd } else { entry };
+                let candidate = normalize_path(&format!("{dir}/{command}"));
+                let Ok(stat) = self.filesystem.stat(&candidate) else {
+                    continue;
+                };
+                if stat.is_directory || stat.mode & EXECUTABLE_PERMISSION_BITS == 0 {
+                    continue;
+                }
+                return Ok(Some(candidate));
+            }
             return Ok(None);
         }
 
@@ -3154,29 +3160,6 @@ impl<F: VirtualFileSystem + 'static> KernelVm<F> {
         Ok(Some(path))
     }
 
-    fn resolve_registered_command_path(&self, path: &str) -> Option<String> {
-        let normalized = normalize_path(path);
-        for prefix in ["/bin/", "/usr/bin/", "/usr/local/bin/"] {
-            let Some(name) = normalized.strip_prefix(prefix) else {
-                continue;
-            };
-            if !name.is_empty() && !name.contains('/') && self.commands.resolve(name).is_some() {
-                return Some(name.to_owned());
-            }
-        }
-
-        if let Some(name) = normalized
-            .strip_prefix("/__secure_exec/commands/")
-            .and_then(|suffix| suffix.rsplit('/').next())
-        {
-            if !name.is_empty() && !name.contains('/') && self.commands.resolve(name).is_some() {
-                return Some(name.to_owned());
-            }
-        }
-
-        None
-    }
-
     fn parse_shebang_command(&mut self, path: &str) -> KernelResult<Option<ShebangCommand>> {
         let header = self.filesystem.pread(path, 0, SHEBANG_LINE_MAX_BYTES + 1)?;
         if !header.starts_with(b"#!") {
@@ -3224,10 +3207,17 @@ impl<F: VirtualFileSystem + 'static> KernelVm<F> {
                 ));
             }
             interpreter_args.remove(0)
-        } else if let Some(command) = self.resolve_registered_command_path(&interpreter) {
-            command
         } else if self.commands.resolve(&shebang.interpreter).is_some() {
             shebang.interpreter
+        } else if let Some(name) = interpreter
+            .rsplit('/')
+            .next()
+            .filter(|name| !name.is_empty() && self.commands.resolve(name).is_some())
+        {
+            // Resolve the interpreter by its basename (e.g. `/bin/sh` -> `sh`),
+            // which replaces the legacy 3-prefix `resolve_registered_command_path`
+            // with the Linux-like "the command is the file's basename" rule.
+            name.to_owned()
         } else {
             return Err(KernelError::command_not_found(&shebang.interpreter));
         };

crates/kernel/tests/command_registry.rs

@@ -77,14 +77,17 @@ fn records_warning_when_overriding_existing_command() {
 }
 
 #[test]
-fn populate_bin_creates_stub_entries() {
+fn populate_driver_bin_creates_stub_entries() {
     let mut vfs = MemoryFileSystem::new();
     let mut registry = CommandRegistry::new();
+    let driver = CommandDriver::new("wasmvm", ["grep", "cat"]);
     registry
-        .register(CommandDriver::new("wasmvm", ["grep", "cat"]))
+        .register(driver.clone())
         .expect("register commands");
 
-    registry.populate_bin(&mut vfs).expect("populate /bin");
+    registry
+        .populate_driver_bin(&mut vfs, &driver)
+        .expect("populate /bin");
 
     assert!(vfs.exists("/bin/grep"));
     assert!(vfs.exists("/bin/cat"));
@@ -157,7 +160,11 @@ fn kernel_driver_registration_rejects_command_path_names_without_writing_stubs()
 }
 
 #[test]
-fn mounted_agentos_command_paths_resolve_to_registered_drivers() {
+fn path_resolved_files_dispatch_by_header_not_registered_name() {
+    // The legacy 3-prefix `resolve_registered_command_path` is gone: a file
+    // resolved by path is executed by its header (binfmt), not mapped back to a
+    // registered command name. A `#!/bin/sh` stub therefore resolves to the `sh`
+    // interpreter (by basename) rather than to its own filename.
     let mut config = KernelVmConfig::new("vm-mounted-command-path");
     config.permissions = Permissions::allow_all();
     let mut kernel = KernelVm::new(MemoryFileSystem::new(), config);
@@ -191,6 +198,7 @@ fn mounted_agentos_command_paths_resolve_to_registered_drivers() {
         .get(&process.pid())
         .cloned()
         .expect("process info");
-    assert_eq!(info.command, "xu");
+    // Dispatched by the `#!/bin/sh` header → the `sh` interpreter, on the wasmvm driver.
+    assert_eq!(info.command, "sh");
     assert_eq!(info.driver, "wasmvm");
 }