hook write (#440)

loongs-zhang · web-flow · commit 697ed2d9adb0 · 2026-04-06T09:14:31.000+08:00
diff --git a/core/src/monitor.rs b/core/src/monitor.rs
@@ -3,7 +3,7 @@ use crate::common::constants::{CoroutineState, MONITOR_BEAN};
 use crate::common::{get_timeout_time, now, CondvarBlocker};
 use crate::coroutine::listener::Listener;
 use crate::coroutine::local::CoroutineLocal;
-use crate::scheduler::SchedulableSuspender;
+use crate::scheduler::{SchedulableCoroutine, SchedulableSuspender};
 use crate::{catch, error, impl_current_for, impl_display_by_debug, info};
 #[cfg(unix)]
 use nix::sys::pthread::{pthread_kill, pthread_self, Pthread};
@@ -78,6 +78,24 @@ impl Monitor {
                 set.remove(Signal::SIGURG);
                 set.thread_set_mask()
                     .expect("Failed to remove SIGURG signal mask!");
+                //不抢占处于Syscall状态的协程。
+                //MonitorListener的设计理念是不对Syscall状态的协程发送信号。
+                //但由于NOTIFY_NODE移除和monitor线程遍历之间存在竞态条件，
+                //SIGURG可能在协程刚进入Syscall状态时到达。
+                //如果此时抢占，协程会被放入syscall_map但无人唤醒（因为没有io_uring/epoll注册），
+                //导致死锁。
+                // Skip preemption for coroutines in Syscall state.
+                // MonitorListener's design is to NOT send signals to Syscall-state
+                // coroutines. However, a race between NOTIFY_NODE removal and the
+                // monitor's queue iteration can cause SIGURG to arrive just after
+                // the coroutine entered Syscall state. If preempted here, the
+                // coroutine lands in the syscall map with no io_uring/epoll/timer
+                // registration to wake it, causing a deadlock.
+                if let Some(co) = SchedulableCoroutine::current() {
+                    if matches!(co.state(), CoroutineState::Syscall((), _, _)) {
+                        return;
+                    }
+                }
                 if let Some(suspender) = SchedulableSuspender::current() {
                     suspender.suspend();
                 }
@@ -89,7 +107,7 @@ impl Monitor {
                 // install panic hook
                 std::panic::set_hook(Box::new(|panic_hook_info| {
                     let syscall = crate::common::constants::SyscallName::panicking;
-                    if let Some(co) = crate::scheduler::SchedulableCoroutine::current() {
+                    if let Some(co) = SchedulableCoroutine::current() {
                         let new_state = crate::common::constants::SyscallState::Executing;
                         if co.syscall((), syscall, new_state).is_err() {
                             error!(
@@ -109,7 +127,7 @@ impl Monitor {
                         "stack backtrace:\n{}",
                         std::backtrace::Backtrace::force_capture()
                     );
-                    if let Some(co) = crate::scheduler::SchedulableCoroutine::current() {
+                    if let Some(co) = SchedulableCoroutine::current() {
                         if co.running().is_err() {
                             error!("{} change to running state failed !", co.name());
                         }
@@ -523,6 +541,24 @@ extern "C" fn do_preempt() {
             // coroutine never yielded (no hooked syscalls) — it is truly CPU-bound.
             // Force immediate suspension.
             flag.set(false);
+            //不抢占处于Syscall状态的协程。
+            //MonitorListener的设计理念是不对Syscall状态的协程发送信号。
+            //但由于NOTIFY_NODE移除和monitor线程遍历之间存在竞态条件，
+            //SIGURG可能在协程刚进入Syscall状态时到达。
+            //如果此时抢占，协程会被放入syscall_map但无人唤醒（因为没有io_uring/epoll注册），
+            //导致死锁。
+            // Skip preemption for coroutines in Syscall state.
+            // MonitorListener's design is to NOT send signals to Syscall-state
+            // coroutines. However, a race between NOTIFY_NODE removal and the
+            // monitor's queue iteration can cause SIGURG to arrive just after
+            // the coroutine entered Syscall state. If preempted here, the
+            // coroutine lands in the syscall map with no io_uring/epoll/timer
+            // registration to wake it, causing a deadlock.
+            if let Some(co) = SchedulableCoroutine::current() {
+                if matches!(co.state(), CoroutineState::Syscall((), _, _)) {
+                    return;
+                }
+            }
             if let Some(suspender) = SchedulableSuspender::current() {
                 suspender.suspend();
             }
diff --git a/core/src/syscall/unix/write.rs b/core/src/syscall/unix/write.rs
@@ -15,9 +15,52 @@ impl_syscall!(WriteSyscallFacade, IoUringWriteSyscall, NioWriteSyscall, RawWrite
     write(fd: c_int, buf: *const c_void, len: size_t) -> ssize_t
 );
 
-impl_facade!(WriteSyscallFacade, WriteSyscall,
-    write(fd: c_int, buf: *const c_void, len: size_t) -> ssize_t
-);
+//write的facade需要特殊处理：stdout/stderr的write由日志框架(tracing)触发，
+//必须跳过所有中间层(facade/io_uring/NIO)直接调用原始系统调用，否则：
+//1. facade内部的info!()会再次触发write导致stdout RefCell重复借用（无限递归）
+//2. io_uring层会提交写操作并阻塞在condvar等待完成，导致死锁
+// The write facade needs special handling: writes to stdout/stderr are
+// triggered by the logging framework (tracing). They must bypass ALL layers
+// (facade, io_uring, NIO) and call the raw syscall directly. Otherwise:
+// 1. The facade's info!() re-triggers write → stdout RefCell double-borrow
+// 2. The io_uring layer submits the write and blocks on condvar → deadlock
+#[repr(C)]
+#[derive(Debug, Default)]
+struct WriteSyscallFacade<I: WriteSyscall> {
+    inner: I,
+}
+
+impl<I: WriteSyscall> WriteSyscall for WriteSyscallFacade<I> {
+    extern "C" fn write(
+        &self,
+        fn_ptr: Option<&extern "C" fn(c_int, *const c_void, size_t) -> ssize_t>,
+        fd: c_int,
+        buf: *const c_void,
+        len: size_t,
+    ) -> ssize_t {
+        if fd == libc::STDOUT_FILENO || fd == libc::STDERR_FILENO {
+            return RawWriteSyscall::default().write(fn_ptr, fd, buf, len);
+        }
+        let syscall = crate::common::constants::SyscallName::write;
+        crate::info!("enter syscall {}", syscall);
+        if let Some(co) = crate::scheduler::SchedulableCoroutine::current() {
+            let new_state = crate::common::constants::SyscallState::Executing;
+            if co.syscall((), syscall, new_state).is_err() {
+                crate::error!("{} change to syscall {} {} failed !",
+                    co.name(), syscall, new_state
+                );
+            }
+        }
+        let r = self.inner.write(fn_ptr, fd, buf, len);
+        if let Some(co) = crate::scheduler::SchedulableCoroutine::current() {
+            if co.running().is_err() {
+                crate::error!("{} change to running state failed !", co.name());
+            }
+        }
+        crate::info!("exit syscall {} {:?} {}", syscall, r, std::io::Error::last_os_error());
+        r
+    }
+}
 
 impl_io_uring_write!(IoUringWriteSyscall, WriteSyscall,
     write(fd: c_int, buf: *const c_void, len: size_t) -> ssize_t
diff --git a/hook/src/syscall/unix.rs b/hook/src/syscall/unix.rs
@@ -62,6 +62,7 @@ impl_hook!(PREADV, preadv(fd: c_int, iov: *const iovec, iovcnt: c_int, offset: o
 impl_hook!(RECVMSG, recvmsg(fd: c_int, msg: *mut msghdr, flags: c_int) -> ssize_t);
 impl_hook!(SEND, send(fd: c_int, buf: *const c_void, len: size_t, flags: c_int) -> ssize_t);
 impl_hook!(SENDTO, sendto(fd: c_int, buf: *const c_void, len: size_t, flags: c_int, addr: *const sockaddr, addrlen: socklen_t) -> ssize_t);
+impl_hook!(WRITE, write(fd: c_int, buf: *const c_void, count: size_t) -> ssize_t);
 impl_hook!(PWRITE, pwrite(fd: c_int, buf: *const c_void, count: size_t, offset: off_t) -> ssize_t);
 impl_hook!(WRITEV, writev(fd: c_int, iov: *const iovec, iovcnt: c_int) -> ssize_t);
 impl_hook!(PWRITEV, pwritev(fd: c_int, iov: *const iovec, iovcnt: c_int, offset: off_t) -> ssize_t);
@@ -82,7 +83,6 @@ impl_hook!(RENAMEAT2, renameat2(olddirfd: c_int, oldpath: *const c_char, newdirf
 // NOTE: unhook poll due to mio's poller
 // impl_hook!(POLL, poll(fds: *mut pollfd, nfds: nfds_t, timeout: c_int) -> c_int);
 
-// NOTE: unhook write/pthread_mutex_lock/pthread_mutex_unlock due to stack overflow or bug
-// impl_hook!(WRITE, write(fd: c_int, buf: *const c_void, count: size_t) -> ssize_t);
+// NOTE: unhook pthread_mutex_lock/pthread_mutex_unlock due to stack overflow or bug
 // impl_hook!(PTHREAD_MUTEX_LOCK, pthread_mutex_lock(lock: *mut pthread_mutex_t) -> c_int);
 // impl_hook!(PTHREAD_MUTEX_UNLOCK, pthread_mutex_unlock(lock: *mut pthread_mutex_t) -> c_int);
