Fix epoll_ctl dropping registrations in multi-threaded guests

In a multi-threaded guest host_fd_ref_open() hands back a dup of the
target fd that host_fd_ref_close() closes when the syscall returns.
sys_epoll_ctl() used that transient dup as the kqueue knote ident, and
the kernel drops a knote the moment its fd is closed -- so every epoll
registration made while multi-threaded was torn down the instant
epoll_ctl() returned, and epoll_pwait() never reported readiness again.
Single-threaded guests borrow the raw fd (no dup, no close) and never
hit it. Node's libuv DelayedTaskScheduler (eventfd + epoll backing
uv_async_send) relied on this path and hung forever at process exit:
the main thread blocked in WorkerThreadsTaskRunner::Shutdown ->
uv_thread_join on a scheduler thread that could no longer be woken.

Key the knote on the persistent host fd from the fd table. Take it from
the same atomic fd_snapshot() that validates the fd, so the ident comes
from the entry that was validated rather than a second fd_to_host()
lookup that could race a concurrent close/reopen. Result mapping already
uses udata (the guest fd), so the ident only needs to stay open and
refer to the same open file description.

Also implement the FIONBIO / FIOCLEX / FIONCLEX ioctls, which were
falling through to ENOTTY. libuv's uv_pipe_open() sets non-blocking via
FIONBIO, so Node's console.log() to a pipe threw "open ENOTTY". FIONBIO
maps to F_SETFL O_NONBLOCK (status flag, shared across the dup);
FIOCLEX/FIONCLEX mirror F_SETFD by toggling the fd_table cloexec bit
under fd_lock (re-checking the slot is still open) rather than the host
fd's FD_CLOEXEC, which is per-descriptor and lost on the dup.

Add tests/test-epoll-mt.c: a CLONE_THREAD sibling keeps the guest
multi-threaded across epoll_ctl, then asserts a registered eventfd and
pipe still deliver an EPOLLIN edge. It fails without the poll.c fix.

With these, node:alpine (node v26.3.0) runs JavaScript, timers, the
libuv threadpool, and promises, and exits cleanly.

Author: Max042004

src/syscall/abi.h

@@ -352,6 +352,9 @@ typedef struct {
 #define LINUX_TIOCSCTTY 0x540E  /* -> macOS TIOCSCTTY (same semantics) */
 #define LINUX_TIOCGWINSZ 0x5413 /* -> macOS TIOCGWINSZ (same struct) */
 #define LINUX_FIONREAD 0x541B   /* -> macOS FIONREAD (same semantics) */
+#define LINUX_FIONBIO 0x5421    /* set/clear O_NONBLOCK (arg: int *) */
+#define LINUX_FIONCLEX 0x5450   /* clear close-on-exec on fd */
+#define LINUX_FIOCLEX 0x5451    /* set close-on-exec on fd */
 #define LINUX_TIOCNOTTY 0x5422  /* -> macOS TIOCNOTTY (same semantics) */
 #define LINUX_TIOCGSID 0x5429   /* -> macOS TIOCGSID (same semantics) */
 /* termios2 variant (adds c_ispeed/c_ospeed) */

src/syscall/io.c

@@ -1688,6 +1688,51 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg)
         return 0;
     }
 
+    case LINUX_FIONBIO: {
+        /* Set/clear O_NONBLOCK on the fd. Linux FIONBIO takes an int* arg:
+         * nonzero enables non-blocking, zero disables it. libuv's
+         * uv__nonblock_ioctl() (its default on Linux) issues this on pipe and
+         * socket fds at setup; without it the guest's uv_pipe_open() fails with
+         * ENOTTY and Node's stdio stream construction throws.
+         */
+        int32_t on = 0;
+        if (guest_read_small(g, arg, &on, sizeof(on)) < 0) {
+            host_fd_ref_close(&host_ref);
+            return -LINUX_EFAULT;
+        }
+        int flags = fcntl(host_fd, F_GETFL);
+        if (flags < 0) {
+            host_fd_ref_close(&host_ref);
+            return linux_errno();
+        }
+        flags = on ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK);
+        int r = fcntl(host_fd, F_SETFL, flags);
+        host_fd_ref_close(&host_ref);
+        return r < 0 ? linux_errno() : 0;
+    }
+
+    case LINUX_FIOCLEX:
+    case LINUX_FIONCLEX:
+        /* Set (FIOCLEX) or clear (FIONCLEX) the guest close-on-exec flag; the
+         * ioctl form of fcntl(F_SETFD). libuv's uv__cloexec() uses these by
+         * default on Linux. Guest cloexec lives in fd_table linux_flags (not
+         * the host fd's FD_CLOEXEC, which is per-descriptor and would be lost
+         * on the dup that host_fd_ref hands multi-threaded callers), so mirror
+         * the F_SETFD path in sys_fcntl. The arg is ignored. Mutate under
+         * fd_lock and re-check the slot is still open so a concurrent
+         * close/reuse cannot flip CLOEXEC on a different fd that took the slot.
+         */
+        pthread_mutex_lock(&fd_lock);
+        if (fd_table[fd].type != FD_CLOSED) {
+            if (request == LINUX_FIOCLEX)
+                fd_table[fd].linux_flags |= LINUX_O_CLOEXEC;
+            else
+                fd_table[fd].linux_flags &= ~LINUX_O_CLOEXEC;
+        }
+        pthread_mutex_unlock(&fd_lock);
+        host_fd_ref_close(&host_ref);
+        return 0;
+
     default:
         host_fd_ref_close(&host_ref);
         return -LINUX_ENOTTY;

src/syscall/poll.c

@@ -707,18 +707,30 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
         return -LINUX_EINVAL;
     }
 
-    host_fd_ref_t target_ref;
-    if (host_fd_ref_open(fd, &target_ref) < 0) {
+    /* Validate the target fd and read its persistent host fd in a single
+     * fd_lock snapshot, so the kqueue knote ident is taken from the same entry
+     * that was validated. A kqueue knote is keyed by the fd number and the
+     * kernel drops it the moment that fd is closed, so the ident must be the
+     * persistent host fd from the fd table -- not the dup that
+     * host_fd_ref_open() hands multi-threaded callers, which
+     * host_fd_ref_close() closes when the syscall returns (silently tearing the
+     * registration down). Snapshotting (rather than host_fd_ref_open() + a
+     * separate fd_to_host()) also avoids a TOCTOU window where a concurrent
+     * close/reopen could key the knote on a different file than the one
+     * validated. Result mapping uses udata (the guest fd), so the ident only
+     * needs to stay open and refer to the same open file description. */
+    fd_entry_t target_snap;
+    if (!fd_snapshot(fd, &target_snap)) {
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EBADF;
     }
+    int target_host_fd = target_snap.host_fd;
 
     epoll_reg_t *reg = &inst->regs[fd];
 
     if (op == LINUX_EPOLL_CTL_DEL) {
         /* Linux returns ENOENT when removing an unregistered fd */
         if (!reg->active) {
-            host_fd_ref_close(&target_ref);
             host_fd_ref_close(&epoll_ref);
             return -LINUX_ENOENT;
         }
@@ -730,12 +742,12 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
         int nchanges = 0;
         {
             if (reg->events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-                EV_SET(&changes[nchanges], target_ref.fd, EVFILT_READ,
+                EV_SET(&changes[nchanges], target_host_fd, EVFILT_READ,
                        EV_DELETE, 0, 0, NULL);
                 nchanges++;
             }
             if (reg->events & LINUX_EPOLLOUT) {
-                EV_SET(&changes[nchanges], target_ref.fd, EVFILT_WRITE,
+                EV_SET(&changes[nchanges], target_host_fd, EVFILT_WRITE,
                        EV_DELETE, 0, 0, NULL);
                 nchanges++;
             }
@@ -745,7 +757,6 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
             /* Clear stale state for potential re-add */
             reg->oneshot_armed = false;
         }
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return 0;
     }
@@ -755,20 +766,17 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
      * (EPOLLONESHOT fired, waiting for re-arm) are still valid for MOD.
      */
     if (op == LINUX_EPOLL_CTL_ADD && reg->active) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EEXIST;
     }
     if (op == LINUX_EPOLL_CTL_MOD && !reg->active && !reg->oneshot_armed) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_ENOENT;
     }
 
     /* ADD or MOD: read the epoll_event from guest */
     linux_epoll_event_t ev;
     if (guest_read_small(g, event_gva, &ev, sizeof(ev)) < 0) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EFAULT;
     }
@@ -786,11 +794,11 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
     if (op == LINUX_EPOLL_CTL_MOD && reg->active) {
         struct kevent del;
         if (reg->events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-            EV_SET(&del, target_ref.fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+            EV_SET(&del, target_host_fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
             kevent(epoll_ref.fd, &del, 1, NULL, 0, NULL);
         }
         if (reg->events & LINUX_EPOLLOUT) {
-            EV_SET(&del, target_ref.fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
+            EV_SET(&del, target_host_fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
             kevent(epoll_ref.fd, &del, 1, NULL, 0, NULL);
         }
     }
@@ -820,19 +828,18 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
     void *udata = (void *) (uintptr_t) fd;
 
     if (ev.events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-        EV_SET(&changes[nchanges], target_ref.fd, EVFILT_READ, kflags, 0, 0,
+        EV_SET(&changes[nchanges], target_host_fd, EVFILT_READ, kflags, 0, 0,
                udata);
         nchanges++;
     }
     if (ev.events & LINUX_EPOLLOUT) {
-        EV_SET(&changes[nchanges], target_ref.fd, EVFILT_WRITE, kflags, 0, 0,
+        EV_SET(&changes[nchanges], target_host_fd, EVFILT_WRITE, kflags, 0, 0,
                udata);
         nchanges++;
     }
 
     if (nchanges > 0) {
         if (kevent(epoll_ref.fd, changes, nchanges, NULL, 0, NULL) < 0) {
-            host_fd_ref_close(&target_ref);
             host_fd_ref_close(&epoll_ref);
             return linux_errno();
         }
@@ -846,7 +853,6 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
     reg->active = true;
     reg->oneshot_armed = false;
 
-    host_fd_ref_close(&target_ref);
     host_fd_ref_close(&epoll_ref);
     return 0;
 }

tests/test-epoll-mt.c

@@ -0,0 +1,147 @@
+/* Multi-threaded epoll registration regression test
+ *
+ * Copyright 2026 elfuse contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Regression for the epoll_ctl host-fd-reference bug: in a multi-threaded
+ * guest, host_fd_ref_open() hands back a *dup* of the target fd that is
+ * closed when the syscall returns. sys_epoll_ctl() used that transient dup
+ * as the kqueue knote ident, so the kernel dropped the registration the
+ * moment epoll_ctl() returned -- and epoll_pwait() never reported readiness
+ * again. Single-threaded guests borrow the raw fd (no dup, no close) and so
+ * never hit it; this only reproduces with at least one CLONE_THREAD sibling
+ * active. Node's libuv DelayedTaskScheduler relied on exactly this path
+ * (eventfd + epoll for uv_async_send) and hung forever at process exit.
+ *
+ * The test keeps a sibling thread alive across the epoll_ctl() call, then
+ * checks that both a pipe and an eventfd registered while multi-threaded
+ * still deliver an EPOLLIN edge.
+ *
+ * Syscalls exercised: clone(220), epoll_create1(20), epoll_ctl(21),
+ *                     epoll_pwait(22), eventfd2(19), pipe2(59), write(64),
+ *                     read(63), close(57), futex(98), exit(93)
+ */
+
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+
+#include "test-harness.h"
+#include "raw-syscall.h"
+
+int passes = 0, fails = 0;
+
+static volatile int child_should_exit = 0;
+static char sibling_stack[16384] __attribute__((aligned(16)));
+
+/* Sibling thread: stays alive (raw nanosleep loop) so the guest is
+ * multi-threaded for the duration of the parent's epoll_ctl() calls, then
+ * exits via the raw exit syscall. Uses only raw syscalls because a
+ * clone(CLONE_THREAD) child has no libc TLS set up.
+ */
+static int sibling_fn(void *arg)
+{
+    (void) arg;
+    struct {
+        long tv_sec, tv_nsec;
+    } ts = {0, 5000000}; /* 5ms */
+    while (!child_should_exit)
+        raw_syscall2(__NR_nanosleep, (long) &ts, 0);
+    raw_syscall1(__NR_exit, 0);
+    return 0;
+}
+
+/* Register host_fd for EPOLLIN on epfd, make it readable via make_ready(),
+ * and assert epoll_pwait() observes the edge within the timeout. Returns 1
+ * on success. */
+static int expect_ready_edge(int epfd, int fd, void (*make_ready)(int), int arg)
+{
+    struct epoll_event ev = {.events = EPOLLIN, .data.fd = fd};
+    if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev) < 0)
+        return 0;
+
+    make_ready(arg);
+
+    struct epoll_event out[4];
+    /* 2s budget: the bug manifests as an indefinite miss, so any generous
+     * finite timeout distinguishes pass from fail without flaking. */
+    int n = epoll_wait(epfd, out, 4, 2000);
+    return n == 1 && out[0].data.fd == fd;
+}
+
+static void poke_eventfd(int fd)
+{
+    uint64_t one = 1;
+    (void) !write(fd, &one, sizeof(one));
+}
+
+static int g_pipe_wr;
+static void poke_pipe(int unused)
+{
+    (void) unused;
+    (void) !write(g_pipe_wr, "x", 1);
+}
+
+int main(void)
+{
+    printf("test-epoll-mt: epoll registration under CLONE_THREAD\n");
+
+    /* Spawn a CLONE_THREAD sibling so host_fd_ref_open() takes the dup path.
+     * Flags: CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
+     *        CLONE_CHILD_CLEARTID.
+     */
+    long flags = 0x00000100 | 0x00000200 | 0x00000400 | 0x00000800 |
+                 0x00010000 | 0x00200000;
+    volatile uint32_t child_tid = 1;
+    long ret = raw_syscall5(__NR_clone, flags,
+                            (long) (sibling_stack + sizeof(sibling_stack)), 0,
+                            0, (long) &child_tid);
+    if (ret == 0) {
+        sibling_fn(NULL);
+        return 0; /* unreachable: sibling_fn exits the thread */
+    }
+
+    TEST("clone sibling for multi-threaded context");
+    EXPECT_TRUE(ret > 0, "clone failed");
+
+    /* eventfd registered + signalled while multi-threaded (the Node path). */
+    TEST("MT epoll: eventfd EPOLLIN edge delivered");
+    {
+        int epfd = epoll_create1(EPOLL_CLOEXEC);
+        int efd = eventfd(0, EFD_NONBLOCK);
+        EXPECT_TRUE(epfd >= 0 && efd >= 0, "epoll/eventfd create failed");
+        EXPECT_TRUE(expect_ready_edge(epfd, efd, poke_eventfd, efd),
+                    "eventfd registration lost across epoll_ctl");
+        close(efd);
+        close(epfd);
+    }
+
+    /* Same with a pipe read end, to show the fix is fd-type independent. */
+    TEST("MT epoll: pipe EPOLLIN edge delivered");
+    {
+        int epfd = epoll_create1(EPOLL_CLOEXEC);
+        int pipefd[2];
+        EXPECT_TRUE(epfd >= 0 && pipe(pipefd) == 0, "epoll/pipe create failed");
+        g_pipe_wr = pipefd[1];
+        EXPECT_TRUE(expect_ready_edge(epfd, pipefd[0], poke_pipe, 0),
+                    "pipe registration lost across epoll_ctl");
+        close(pipefd[0]);
+        close(pipefd[1]);
+        close(epfd);
+    }
+
+    /* Release the sibling and join via the CLONE_CHILD_CLEARTID futex. */
+    child_should_exit = 1;
+    for (int i = 0; i < 200 && child_tid != 0; i++) {
+        struct {
+            long tv_sec, tv_nsec;
+        } ts = {0, 10000000}; /* 10ms */
+        raw_syscall6(__NR_futex, (long) &child_tid, 0 /* FUTEX_WAIT */,
+                     child_tid, (long) &ts, 0, 0);
+    }
+
+    SUMMARY("test-epoll-mt");
+    return fails > 0 ? 1 : 0;
+}