Fix epoll_ctl dropping registrations in multi-threaded guests

In a multi-threaded guest host_fd_ref_open() hands back a dup of the
target fd that host_fd_ref_close() closes when the syscall returns.
sys_epoll_ctl() used that transient dup as the kqueue knote ident, and
the kernel drops a knote the moment its fd is closed -- so every epoll
registration made while multi-threaded was torn down the instant
epoll_ctl() returned, and epoll_pwait() never reported readiness again.
Single-threaded guests borrow the raw fd (no dup, no close) and never
hit it. Node's libuv DelayedTaskScheduler (eventfd + epoll backing
uv_async_send) relied on this path and hung forever at process exit:
the main thread blocked in WorkerThreadsTaskRunner::Shutdown ->
uv_thread_join on a scheduler thread that could no longer be woken.

Key the knote on the persistent host fd from the fd table. Take it from
the same atomic fd_snapshot() that validates the fd, so the ident comes
from the entry that was validated rather than a second fd_to_host()
lookup that could race a concurrent close/reopen. Result mapping already
uses udata (the guest fd), so the ident only needs to stay open and
refer to the same open file description.

Guard the close+reopen ABA with a per-slot generation counter. fd_table
entries now carry a monotonic generation bumped on every allocation;
epoll registrations stamp it at ADD/MOD. If the guest closes a watched
fd and reopens it (reusing the guest fd number), the kernel has already
dropped the original knote, yet reg->active still looks live -- a later
DEL/MOD would EV_DELETE the wrong knote on the reused host fd. A
mismatched generation now marks the registration gone, so DEL/MOD report
ENOENT (matching Linux's auto-removal on close) and ADD starts fresh.

Also implement the FIONBIO / FIOCLEX / FIONCLEX ioctls, which were
falling through to ENOTTY. libuv's uv_pipe_open() sets non-blocking via
FIONBIO, so Node's console.log() to a pipe threw "open ENOTTY". FIONBIO
maps to F_SETFL O_NONBLOCK (status flag, shared across the dup).
FIOCLEX/FIONCLEX mirror F_SETFD by toggling the fd_table cloexec bit
rather than the host fd's FD_CLOEXEC, which is per-descriptor and lost on
the dup. They need no host fd, so they dispatch before
host_fd_ref_open_regular_io() -- which rejects O_PATH (FD_PATH) with
EBADF, while Linux allows these ioctls (like F_SETFD) on O_PATH fds --
and validate the slot and flip the flag in a single fd_lock section, so
there is no validate-then-mutate window for a concurrent close/reuse to
flip cloexec on a different file.

Add tests/test-epoll-mt.c: a CLONE_THREAD sibling keeps the guest
multi-threaded across epoll_ctl, then asserts a registered eventfd and
pipe still deliver an EPOLLIN edge. It fails without the poll.c fix.
Add tests/test-ioctl-cloexec.c covering FIOCLEX/FIONCLEX round-trip on
both a regular and an O_PATH fd. Both are listed in tests/manifest.txt
so the driver runs them under make check.

With these, node:alpine (node v26.3.0) runs JavaScript, timers, the
libuv threadpool, and promises, and exits cleanly.

Author: Max042004

src/syscall/abi.h

@@ -352,6 +352,9 @@ typedef struct {
 #define LINUX_TIOCSCTTY 0x540E  /* -> macOS TIOCSCTTY (same semantics) */
 #define LINUX_TIOCGWINSZ 0x5413 /* -> macOS TIOCGWINSZ (same struct) */
 #define LINUX_FIONREAD 0x541B   /* -> macOS FIONREAD (same semantics) */
+#define LINUX_FIONBIO 0x5421    /* set/clear O_NONBLOCK (arg: int *) */
+#define LINUX_FIONCLEX 0x5450   /* clear close-on-exec on fd */
+#define LINUX_FIOCLEX 0x5451    /* set close-on-exec on fd */
 #define LINUX_TIOCNOTTY 0x5422  /* -> macOS TIOCNOTTY (same semantics) */
 #define LINUX_TIOCGSID 0x5429   /* -> macOS TIOCGSID (same semantics) */
 /* termios2 variant (adds c_ispeed/c_ospeed) */
@@ -700,6 +703,10 @@ typedef struct {
     int type;        /* FD_CLOSED, FD_STDIO, FD_REGULAR, FD_DIR */
     int host_fd;     /* Underlying macOS file descriptor */
     int linux_flags; /* Linux open flags (for CLOEXEC tracking) */
+    uint64_t generation; /* Monotonic stamp bumped on every (re)allocation of
+                          * this slot. Lets long-lived references (e.g. epoll
+                          * registrations) detect a close+reopen ABA where the
+                          * slot now holds a different open file. */
     void *dir;       /* DIR* for FD_DIR entries (NULL otherwise) */
     char proc_path[FD_VIRTUAL_PATH_MAX]; /* Virtual /proc dir root for *at */
     int seals; /* F_SEAL_* bits (non-zero only for memfd_create fds) */

src/syscall/fdtable.c

@@ -59,6 +59,14 @@ int fd_get_rlimit_nofile(void)
 #define FD_BITMAP_WORDS (FD_TABLE_SIZE / 64)
 static uint64_t fd_free_bitmap[FD_BITMAP_WORDS];
 
+/* Monotonic generation source. Bumped (under fd_lock, in fd_init_entry) on
+ * every slot allocation so each open file gets a value distinct from whatever
+ * previously occupied the slot. 64 bits never wraps in practice, so a stamped
+ * generation uniquely identifies one allocation of one fd. Starts at 0; the
+ * first allocation gets 1, leaving 0 to mean "never stamped".
+ */
+static uint64_t fd_generation_next;
+
 static inline void fd_bitmap_set_free(int fd)
 {
     fd_free_bitmap[fd / 64] |= BIT64(fd % 64);
@@ -77,6 +85,7 @@ static inline void fd_init_entry(int fd,
     fd_bitmap_set_used(fd);
     fd_table[fd].type = type;
     fd_table[fd].host_fd = host_fd;
+    fd_table[fd].generation = ++fd_generation_next;
     fd_table[fd].linux_flags = 0;
     fd_table[fd].dir = NULL;
     fd_table[fd].proc_path[0] = '\0';

src/syscall/io.c

@@ -1476,6 +1476,33 @@ int64_t sys_pwritev2(guest_t *g,
 
 int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg)
 {
+    /* FIOCLEX/FIONCLEX are the ioctl form of fcntl(F_SETFD): they set/clear the
+     * guest close-on-exec flag, which lives in fd_table linux_flags (not the
+     * host fd's FD_CLOEXEC, which is per-descriptor and would be lost on the dup
+     * that host_fd_ref hands multi-threaded callers, so mirror the F_SETFD path
+     * in sys_fcntl). They need no host fd, so dispatch them before
+     * host_fd_ref_open_regular_io(): that helper rejects O_PATH (FD_PATH) fds
+     * with EBADF, but Linux allows these ioctls -- like fcntl(F_SETFD) -- on
+     * O_PATH descriptors. Validate the slot and mutate the flag in a single
+     * fd_lock section so there is no validate-then-mutate window in which a
+     * concurrent close/reuse could flip CLOEXEC on a different file that took
+     * the slot. The arg is ignored. */
+    if (request == LINUX_FIOCLEX || request == LINUX_FIONCLEX) {
+        if (!RANGE_CHECK(fd, 0, FD_TABLE_SIZE))
+            return -LINUX_EBADF;
+        pthread_mutex_lock(&fd_lock);
+        if (fd_table[fd].type == FD_CLOSED) {
+            pthread_mutex_unlock(&fd_lock);
+            return -LINUX_EBADF;
+        }
+        if (request == LINUX_FIOCLEX)
+            fd_table[fd].linux_flags |= LINUX_O_CLOEXEC;
+        else
+            fd_table[fd].linux_flags &= ~LINUX_O_CLOEXEC;
+        pthread_mutex_unlock(&fd_lock);
+        return 0;
+    }
+
     host_fd_ref_t host_ref;
     int64_t err = host_fd_ref_open_regular_io(fd, &host_ref);
     if (err < 0)
@@ -1688,6 +1715,29 @@ int64_t sys_ioctl(guest_t *g, int fd, uint64_t request, uint64_t arg)
         return 0;
     }
 
+    case LINUX_FIONBIO: {
+        /* Set/clear O_NONBLOCK on the fd. Linux FIONBIO takes an int* arg:
+         * nonzero enables non-blocking, zero disables it. libuv's
+         * uv__nonblock_ioctl() (its default on Linux) issues this on pipe and
+         * socket fds at setup; without it the guest's uv_pipe_open() fails with
+         * ENOTTY and Node's stdio stream construction throws.
+         */
+        int32_t on = 0;
+        if (guest_read_small(g, arg, &on, sizeof(on)) < 0) {
+            host_fd_ref_close(&host_ref);
+            return -LINUX_EFAULT;
+        }
+        int flags = fcntl(host_fd, F_GETFL);
+        if (flags < 0) {
+            host_fd_ref_close(&host_ref);
+            return linux_errno();
+        }
+        flags = on ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK);
+        int r = fcntl(host_fd, F_SETFL, flags);
+        host_fd_ref_close(&host_ref);
+        return r < 0 ? linux_errno() : 0;
+    }
+
     default:
         host_fd_ref_close(&host_ref);
         return -LINUX_ENOTTY;

src/syscall/poll.c

@@ -620,6 +620,11 @@ typedef struct {
 typedef struct {
     uint32_t events;    /* Registered EPOLL* events mask */
     uint64_t data;      /* User data to return in epoll_wait */
+    uint64_t generation; /* fd_entry_t.generation captured at ADD/MOD. Detects a
+                          * close+reopen ABA: if the guest fd's current
+                          * generation no longer matches, the registered open
+                          * file is gone and this stale entry must not drive
+                          * kevent against the reused host fd. */
     bool active;        /* Registered in this instance */
     bool oneshot_armed; /* EPOLLONESHOT and event already fired,
                          * waiting for EPOLL_CTL_MOD re-arm.
@@ -707,18 +712,43 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
         return -LINUX_EINVAL;
     }
 
-    host_fd_ref_t target_ref;
-    if (host_fd_ref_open(fd, &target_ref) < 0) {
+    /* Validate the target fd and read its persistent host fd in a single
+     * fd_lock snapshot, so the kqueue knote ident is taken from the same entry
+     * that was validated. A kqueue knote is keyed by the fd number and the
+     * kernel drops it the moment that fd is closed, so the ident must be the
+     * persistent host fd from the fd table -- not the dup that
+     * host_fd_ref_open() hands multi-threaded callers, which
+     * host_fd_ref_close() closes when the syscall returns (silently tearing the
+     * registration down). Snapshotting (rather than host_fd_ref_open() + a
+     * separate fd_to_host()) keeps the validate and the ident read atomic under
+     * one fd_lock. The snapshot's generation then guards the cross-call ABA
+     * below. Result mapping uses udata (the guest fd), so the ident only needs
+     * to stay open and refer to the same open file description. */
+    fd_entry_t target_snap;
+    if (!fd_snapshot(fd, &target_snap)) {
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EBADF;
     }
+    int target_host_fd = target_snap.host_fd;
 
     epoll_reg_t *reg = &inst->regs[fd];
 
+    /* Cross-call ABA guard. If the guest closed this fd and reopened it (or the
+     * slot was reused) since the registration was stamped, the kernel already
+     * dropped the original knote when the old host fd closed, yet the guest fd
+     * number -- and thus reg->active -- still looks live. Acting on it would
+     * EV_DELETE/EV_MOD the wrong knote on the reused host fd. A mismatched
+     * generation means the registration is gone: drop it so DEL/MOD report
+     * ENOENT (matching Linux's auto-removal on close) and ADD starts fresh. */
+    if ((reg->active || reg->oneshot_armed) &&
+        reg->generation != target_snap.generation) {
+        reg->active = false;
+        reg->oneshot_armed = false;
+    }
+
     if (op == LINUX_EPOLL_CTL_DEL) {
         /* Linux returns ENOENT when removing an unregistered fd */
         if (!reg->active) {
-            host_fd_ref_close(&target_ref);
             host_fd_ref_close(&epoll_ref);
             return -LINUX_ENOENT;
         }
@@ -730,12 +760,12 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
         int nchanges = 0;
         {
             if (reg->events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-                EV_SET(&changes[nchanges], target_ref.fd, EVFILT_READ,
+                EV_SET(&changes[nchanges], target_host_fd, EVFILT_READ,
                        EV_DELETE, 0, 0, NULL);
                 nchanges++;
             }
             if (reg->events & LINUX_EPOLLOUT) {
-                EV_SET(&changes[nchanges], target_ref.fd, EVFILT_WRITE,
+                EV_SET(&changes[nchanges], target_host_fd, EVFILT_WRITE,
                        EV_DELETE, 0, 0, NULL);
                 nchanges++;
             }
@@ -745,7 +775,6 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
             /* Clear stale state for potential re-add */
             reg->oneshot_armed = false;
         }
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return 0;
     }
@@ -755,20 +784,17 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
      * (EPOLLONESHOT fired, waiting for re-arm) are still valid for MOD.
      */
     if (op == LINUX_EPOLL_CTL_ADD && reg->active) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EEXIST;
     }
     if (op == LINUX_EPOLL_CTL_MOD && !reg->active && !reg->oneshot_armed) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_ENOENT;
     }
 
     /* ADD or MOD: read the epoll_event from guest */
     linux_epoll_event_t ev;
     if (guest_read_small(g, event_gva, &ev, sizeof(ev)) < 0) {
-        host_fd_ref_close(&target_ref);
         host_fd_ref_close(&epoll_ref);
         return -LINUX_EFAULT;
     }
@@ -786,11 +812,11 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
     if (op == LINUX_EPOLL_CTL_MOD && reg->active) {
         struct kevent del;
         if (reg->events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-            EV_SET(&del, target_ref.fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+            EV_SET(&del, target_host_fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
             kevent(epoll_ref.fd, &del, 1, NULL, 0, NULL);
         }
         if (reg->events & LINUX_EPOLLOUT) {
-            EV_SET(&del, target_ref.fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
+            EV_SET(&del, target_host_fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
             kevent(epoll_ref.fd, &del, 1, NULL, 0, NULL);
         }
     }
@@ -820,33 +846,34 @@ int64_t sys_epoll_ctl(guest_t *g, int epfd, int op, int fd, uint64_t event_gva)
     void *udata = (void *) (uintptr_t) fd;
 
     if (ev.events & (LINUX_EPOLLIN | LINUX_EPOLLRDHUP)) {
-        EV_SET(&changes[nchanges], target_ref.fd, EVFILT_READ, kflags, 0, 0,
+        EV_SET(&changes[nchanges], target_host_fd, EVFILT_READ, kflags, 0, 0,
                udata);
         nchanges++;
     }
     if (ev.events & LINUX_EPOLLOUT) {
-        EV_SET(&changes[nchanges], target_ref.fd, EVFILT_WRITE, kflags, 0, 0,
+        EV_SET(&changes[nchanges], target_host_fd, EVFILT_WRITE, kflags, 0, 0,
                udata);
         nchanges++;
     }
 
     if (nchanges > 0) {
         if (kevent(epoll_ref.fd, changes, nchanges, NULL, 0, NULL) < 0) {
-            host_fd_ref_close(&target_ref);
             host_fd_ref_close(&epoll_ref);
             return linux_errno();
         }
     }
 
     /* Store registration data in per-instance table.
-     * Clear oneshot_armed when MOD successfully re-arms.
+     * Clear oneshot_armed when MOD successfully re-arms. Stamp the snapshot's
+     * generation so a later close+reopen of this guest fd is detected as a
+     * stale registration by the ABA guard above.
      */
     reg->events = ev.events;
     reg->data = ev.data;
+    reg->generation = target_snap.generation;
     reg->active = true;
     reg->oneshot_armed = false;
 
-    host_fd_ref_close(&target_ref);
     host_fd_ref_close(&epoll_ref);
     return 0;
 }

tests/manifest.txt

@@ -62,8 +62,11 @@ test-signalfd
 test-signalfd-hardening
 test-epoll
 test-epoll-edge
+test-epoll-mt
+test-epoll-aba
 test-timerfd
 test-large-io-boundary
+test-ioctl-cloexec
 
 [section] /proc and /dev emulation tests
 test-proc