From 3991815b79da0ba17662556bd9de8c8aed95d41d Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Wed, 22 Jul 2026 12:45:40 -0400 Subject: [PATCH 1/6] process: implement thread-backed fork()/vfork()/execve()/waitpid() (opt-in) OSv is a single-address-space unikernel, so a literal copy-on-write fork() is not the model. This adds a thread-backed fork() emulation that covers the useful, compatible subset of fork semantics, gated entirely behind a new CONFIG_fork kconfig option that defaults to OFF - so a default OSv build is unchanged (none of this code is compiled in and fork()/vfork() return ENOSYS as before). With CONFIG_fork enabled: - fork()/vfork() create a child OSv thread that resumes in fork()'s caller and returns 0 in the child / the child pid in the parent (the classic twin return). The child runs on a private copy of the parent's user stack, so parent and child have independent locals after the return; it gets its own fresh OSv per-thread TLS block (own errno etc). Implemented in libc/process/fork.cc + arch/{x64,aarch64}/fork.cc. - execve() launches the target as a fresh OSv application (its own ELF namespace) and does not return, making fork()+exec() work. - waitpid()/wait4()/wait() reap a child's exit status via a pid->child registry; SIGCHLD is raised to the parent on child exit; SIGCHLD/SIGURG/ SIGWINCH now correctly default to ignore (not poweroff). - exit()/_exit() in a fork child ends only that child, not the whole unikernel. - pthread_atfork prepare/parent/child handlers are now actually run around fork() (they were a no-op stub) - glibc/musl register these internally. - sys_clone() routes the non-CLONE_THREAD (fork) case here when enabled. Validated: tst-fork passes 10/10 on both x86-64 and aarch64 (twin return, private-stack isolation, fork+exec, vfork, waitpid reaping). Documented limitations (documentation/fork.md): the child shares the parent's heap/globals (no per-process memory isolation in one address space - a follow-up adds per-child copy-on-write address spaces behind the same flag); deep-call- chain child unwind and fork-as-memory-snapshot (Redis BGSAVE) are not carried by this base; execve()'s new-ELF-namespace path has a separate pre-existing fault. Copyright (C) 2026 Greg Burd --- Makefile | 12 +++ arch/aarch64/fork.cc | 83 +++++++++++++++ arch/x64/fork.cc | 107 ++++++++++++++++++++ conf/kconfig/threads | 13 +++ documentation/fork.md | 111 ++++++++++++++++++++ include/osv/fork.hh | 69 +++++++++++++ libc/process/execve.cc | 91 ++++++++++++++++- libc/process/fork.cc | 219 ++++++++++++++++++++++++++++++++++++++++ libc/process/waitpid.cc | 36 +++++++ libc/pthread.cc | 39 +++++++ libc/signal.cc | 7 ++ linux.cc | 22 +++- modules/tests/Makefile | 1 + runtime.cc | 47 +++++---- tests/tst-fork.cc | 107 ++++++++++++++++++++ 15 files changed, 939 insertions(+), 25 deletions(-) create mode 100644 arch/aarch64/fork.cc create mode 100644 arch/x64/fork.cc create mode 100644 documentation/fork.md create mode 100644 include/osv/fork.hh create mode 100644 libc/process/fork.cc create mode 100644 tests/tst-fork.cc diff --git a/Makefile b/Makefile index faeee368f3..e062af3e76 100644 --- a/Makefile +++ b/Makefile @@ -1057,6 +1057,9 @@ objects += arch/$(arch)/firmware.o objects += arch/$(arch)/hypervisor.o objects += arch/$(arch)/interrupt.o objects += arch/$(arch)/clone.o +ifeq ($(conf_fork),1) +objects += arch/$(arch)/fork.o +endif ifeq ($(conf_drivers_pci),1) objects += arch/$(arch)/pci.o objects += arch/$(arch)/msi.o @@ -1660,11 +1663,20 @@ musl += prng/srand48.o libc += random.o libc += process/execve.o +ifeq ($(conf_fork),1) +libc += process/fork.o +endif musl += process/execle.o musl += process/execv.o musl += process/execl.o libc += process/waitpid.o +# When fork() is enabled, our waitpid.cc provides wait()/waitpid()/wait4() +# backed by the fork() child registry, superseding musl's process/wait.o. +# When fork() is disabled, waitpid.cc provides only the ECHILD waitpid() stub, +# so keep musl's wait.o for wait()/wait4(). +ifneq ($(conf_fork),1) musl += process/wait.o +endif musl += setjmp/$(musl_arch)/setjmp.o musl += setjmp/$(musl_arch)/longjmp.o diff --git a/arch/aarch64/fork.cc b/arch/aarch64/fork.cc new file mode 100644 index 0000000000..4db49ef121 --- /dev/null +++ b/arch/aarch64/fork.cc @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * fork_thread() for aarch64 -- mirrors arch/x64/fork.cc. fork() (fork.cc) + * passes the caller's return address and stack pointer; we copy the parent's + * user stack, bias the SP into the copy, and start a child thread that installs + * the copied stack, sets x0=0 (fork()'s return value in the child), and returns + * to fork()'s caller. + */ + +#include "arch.hh" +#include +#include +#include +#include + +// pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the +// child's context before it resumes user code. +extern "C" void __osv_run_atfork_child(); + +sched::thread *fork_thread(void *caller_ret, void *caller_sp, + void **out_stack_to_free) +{ + auto parent = sched::thread::current(); + auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr; + + auto si = parent->get_stack_info(); + char *stack_base = static_cast(si.begin) + si.size; + char *sp = static_cast(caller_sp); + if (sp < static_cast(si.begin) || sp > stack_base) { + return nullptr; + } + size_t stack_size = si.size; + + char *child_stack_mem = static_cast(malloc(stack_size)); + if (!child_stack_mem) { + return nullptr; + } + // Copy ONLY the live top [caller_sp .. stack_base) into the top of the + // child buffer (app stacks are demand-paged; copying from si.begin faults). + char *child_base = child_stack_mem + stack_size; + ptrdiff_t bias = child_base - stack_base; + size_t live = static_cast(stack_base - sp); + memcpy(child_base - live, sp, live); + char *child_sp = sp + bias; + + volatile u64 resume_sp = reinterpret_cast(child_sp); + volatile u64 resume_pc = reinterpret_cast(caller_ret); + char *stack_to_free = child_stack_mem; + + // TLS: the child is a real OSv thread with its own fresh setup_tcb() block. + // Only override tpidr_el0 if the parent had installed its own app TCB via + // arch_prctl (parent_app_tcb != 0); otherwise keep the child's private OSv + // TLS (the clean case for a musl app built against OSv's libc). + u64 parent_app_tcb = parent->get_app_tcb(); + auto t = sched::thread::make([resume_sp, resume_pc, parent_app_tcb] { + if (parent_app_tcb) { + asm volatile ("msr tpidr_el0, %0; isb" :: "r"(parent_app_tcb) : "memory"); + } + // Run pthread_atfork child handlers in the child's context before + // resuming user code. + __osv_run_atfork_child(); + asm volatile + ("mov sp, %0 \n\t" // install the private copied stack + "mov x0, #0 \n\t" // fork() returns 0 in the child + "br %1 \n\t" // resume in fork()'s caller + : : "r"(resume_sp), "r"(resume_pc) : "x0", "memory"); + }, sched::thread::attr(). + stack(4096 * 4), + false, + true); + t->set_app_tcb(parent->get_app_tcb()); + if (parent_pinned_cpu) { + t->pin(parent_pinned_cpu); + } + if (out_stack_to_free) { + *out_stack_to_free = stack_to_free; + } + return t; +} diff --git a/arch/x64/fork.cc b/arch/x64/fork.cc new file mode 100644 index 0000000000..3f93519826 --- /dev/null +++ b/arch/x64/fork.cc @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * fork_thread(): create a child thread that resumes in fork()'s CALLER, on a + * private copy of the parent's user stack, returning 0 from fork() in the child. + * The x86-64 arch half of the fork() emulation (see documentation/fork.md). + * + * fork() (libc/process/fork.cc) passes us the caller's resume point: + * caller_ret = the address fork() would return to (__builtin_return_address) + * caller_sp = the parent's SP at fork()'s return (fork()'s frame base) + * We copy the parent stack region [caller_sp .. stack_base) into a fresh stack, + * bias caller_sp into the copy, and start a child thread whose trampoline sets + * rsp=child_sp, rax=0, and jumps to caller_ret -- i.e. the child returns from + * fork() with value 0 on its own private stack, in the caller. + */ + +#include "arch.hh" +#include "tls-switch.hh" +#include +#include +#include +#include + +// pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the +// child's context before it resumes user code. +extern "C" void __osv_run_atfork_child(); + +sched::thread *fork_thread(void *caller_ret, void *caller_sp, void **out_stack_to_free) +{ + auto parent = sched::thread::current(); + auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr; + + auto si = parent->get_stack_info(); + char *stack_base = static_cast(si.begin) + si.size; + char *sp = static_cast(caller_sp); + if (sp < static_cast(si.begin) || sp > stack_base) { + return nullptr; // caller SP not within the known user stack + } + size_t stack_size = si.size; + + char *child_stack_mem = static_cast(malloc(stack_size)); + if (!child_stack_mem) { + return nullptr; + } + // Copy ONLY the live top of the stack, [caller_sp .. stack_base), into the + // TOP of the child buffer. App (pthread) stacks are demand-paged: only the + // used top is mapped, so copying from si.begin would fault on the first + // unmapped page. Keeping the copy at the top of the child buffer preserves + // the base-relative bias so a biased SP resolves correctly. + char *child_base = child_stack_mem + stack_size; + ptrdiff_t bias = child_base - stack_base; + size_t live = static_cast(stack_base - sp); + memcpy(child_base - live, sp, live); + char *child_sp = sp + bias; + + volatile u64 resume_sp = reinterpret_cast(child_sp); + volatile u64 resume_pc = reinterpret_cast(caller_ret); + char *stack_to_free = child_stack_mem; + + // TLS handling. The child is a real OSv sched::thread, so its constructor + // already ran setup_tcb() and installed a FRESH, private OSv TLS block + // (with its own errno and all libc __thread state). Two cases: + // + // (1) The app uses OSv's libc TLS (the normal dynamically-linked path, + // app_tcb == 0): the child's own fresh TCB is exactly right -- do NOT + // touch fsbase, let the child run on its private per-thread TLS. This + // is the clean case and fork() "just works" for TLS. + // (2) The app installed its own TCB via arch_prctl(SET_FS) (app_tcb != 0, + // e.g. a glibc binary's __libc_setup_tls): the child would need a + // private COPY of that app TCB. We do not duplicate it here yet; + // the child inherits the parent's app_tcb (shared), which is the + // documented multi-process-glibc limitation. A musl app built against + // OSv's libc takes path (1) and avoids this entirely. + u64 parent_app_tcb = parent->get_app_tcb(); + + auto t = sched::thread::make([resume_sp, resume_pc, parent_app_tcb] { + // Only override the child's own (fresh) TLS if the parent had installed + // an app TCB via arch_prctl; otherwise keep the child's private OSv TCB. + if (parent_app_tcb) { + arch::set_fsbase(parent_app_tcb); + } + // Run pthread_atfork child handlers in the child's context (e.g. reset + // the malloc arena lock) before resuming user code. + __osv_run_atfork_child(); + asm volatile + ("movq %0, %%rsp \n\t" // install the private copied stack + "xorq %%rax, %%rax \n\t" // fork() returns 0 in the child + "jmpq *%1 \n\t" // resume in fork()'s caller + : : "r"(resume_sp), "r"(resume_pc) : "memory"); + }, sched::thread::attr(). + stack(4096 * 4), + false, + true); + t->set_app_tcb(parent->get_app_tcb()); + if (parent_pinned_cpu) { + t->pin(parent_pinned_cpu); + } + // The caller (fork.cc) owns the single cleanup; hand back the copied user + // stack so it can be freed when the child is reaped. + if (out_stack_to_free) { + *out_stack_to_free = stack_to_free; + } + return t; +} diff --git a/conf/kconfig/threads b/conf/kconfig/threads index 49b5edcb93..103a5ac4c3 100644 --- a/conf/kconfig/threads +++ b/conf/kconfig/threads @@ -6,6 +6,19 @@ config lazy_stack prompt "Use lazy stack" def_bool $(shell,grep -q ^conf_lazy_stack=1 conf/base.mk && echo y || echo n) +config fork + prompt "Include fork()/vfork() support (per-child address space, off by default)" + bool + default n + help + Enable OSv's thread-backed fork()/vfork()/execve()/waitpid() emulation, + including the per-child address space with copy-on-write needed to give a + forked child private memory. This changes OSv's usual single-address-space + model (address-space switches on context switch between fork children), so + it is OFF by default: with this disabled, none of the fork code is compiled + in and OSv behaves exactly as before (fork() returns ENOSYS). Enable it + only for workloads that require fork() (e.g. multi-process programs). + config lazy_stack_invariant prompt "Check lazy stack invariant" def_bool $(shell,grep -q ^conf_lazy_stack_invariant=1 conf/base.mk && echo y || echo n) diff --git a/documentation/fork.md b/documentation/fork.md new file mode 100644 index 0000000000..f144f88a60 --- /dev/null +++ b/documentation/fork.md @@ -0,0 +1,111 @@ +# fork() on OSv + +OSv is a single-address-space unikernel: all threads of all applications share +one address space, with no MMU-enforced process isolation. This is deliberate, +it is where OSv's performance comes from. It also means Linux `fork()`, which +gives the child a **private copy-on-write duplicate of the parent's entire +address space**, cannot be implemented literally. + +OSv instead provides a **thread-backed fork() emulation** that supports the +common, compatible subset of fork semantics. This document describes what works, +what does not, and why. + +## Off by default: the `fork` configure flag + +All of the fork() machinery - the fork()/vfork()/execve()/waitpid() +implementations, the per-child address space, and the copy-on-write changes - +is gated behind the `CONFIG_fork` kconfig option (make variable `conf_fork`), +which defaults to **n**. When it is disabled (the default), NONE of this code +is compiled into the kernel: the fork object files are excluded from the build, +fork()/vfork() return ENOSYS as before, and OSv behaves exactly as it did with +no change to its single-address-space model or performance. Build with +`conf_fork=1` (or enable "Include fork() support" in `make menuconfig`) only for +workloads that need fork(). This keeps the default OSv unchanged while offering +fork() to those who require it. + +## What works + +- **`fork()` twin return.** `fork()` creates a new OSv thread (the "child") that + resumes at the `fork()` call site. `fork()` returns the child's pid to the + parent and `0` to the child, exactly like Linux. The child runs on a **private + copy of the parent's user stack**, so parent and child have independent local + variables and call chains after the return. + +- **`fork()` + `execve()`.** The child calls `execve()`, which launches the + requested program as a fresh OSv application (its own ELF namespace) and makes + the child thread that program's driver; a successful `execve()` never returns, + as on Linux. The parent is a separate thread and is unaffected. + +- **`waitpid()` / `wait4()` / `wait()`.** The parent reaps a child's exit status + (Linux-encoded, use `WIFEXITED`/`WEXITSTATUS`). `WNOHANG` is supported. + `SIGCHLD` is raised to the parent when a child exits. + +- **`vfork()`.** Maps to `fork()`. Because the child already shares the parent's + address space, this actually matches vfork's contract ("the child borrows the + parent's memory until it execs or exits") more faithfully than it matches + fork's copy contract. + +- **`_exit()` / `exit()` in a child** ends only that child "process" (thread / + app) and records its status for the parent, rather than shutting down the + whole unikernel (which is what a top-level `exit()` still does). + +## What does NOT work (and why) + +- **Shared TLS (thread-local storage) - only for apps that install their own + TCB.** A forked child is a real OSv thread, so it gets its OWN fresh, private + OSv TLS block (its own `errno` and libc `__thread` state) automatically. For a + program that uses OSv's (musl-derived) libc the normal way, fork() TLS "just + works" - the child does not share the parent's TLS. The exception is a program + that installs its OWN thread pointer via `arch_prctl(ARCH_SET_FS)` (e.g. a + glibc-ABI binary's `__libc_setup_tls`): OSv records that as `app_tcb`, and the + fork child currently inherits the SAME `app_tcb` (shared), which collides. The + fix for such binaries is to build them against OSv's own musl libc instead of + glibc, so they take the clean per-thread-TLS path. (This was the wall stock + glibc-built PostgreSQL hit; a musl build avoids it.) + +- **Memory isolation.** The child shares the parent's heap and global variables + (only the stack is copied). A child that **writes** to shared globals or + heap-allocated data expecting a private copy will affect the parent. Code that + only reads shared state and writes to its own fds or freshly-`malloc`'d memory + before `exec`/`_exit` is fine; code that mutates shared state after fork is + not. This cannot be fixed without adding process isolation to OSv. + +- **`fork()` as a memory snapshot** (e.g. Redis `BGSAVE`, a GC that forks to walk + a frozen heap). These rely on the child seeing a *frozen* copy of the parent's + memory at fork time. On OSv the child sees live, shared memory. This is a + silent behavioral difference (it cannot be detected at the syscall boundary) + and is **unsupported**. + +- **Stack-internal pointers.** Because the child's stack is a byte copy of the + parent's biased to a new address, a pointer stored on the stack that points + *into the same stack* still points at the parent's stack in the child. Short + child code paths (the fork+exec and fork+work-then-_exit patterns) do not hit + this; long-lived divergent children can. A future refinement could scan and + fix up such pointers. + +- **`clone()` with namespace-unshare flags** (`CLONE_NEWNS`, `CLONE_NEWPID`, + etc.) returns `ENOSYS` — there are no namespaces to unshare. + +- **aarch64.** Implemented and validated: the stack-copy + `br` resume + trampoline works on aarch64 as well as x86-64. `tst-fork` passes + 10/10 on both architectures. + +## Implementation + +- `libc/process/fork.cc` — `fork()`/`vfork()`, the child registry, and the + `waitpid()` backend + `SIGCHLD` notification. +- `arch/x64/fork.cc` — `fork_thread()`: allocates a fresh stack, copies the + parent's current user stack into it, and returns a child thread that installs + the copied stack and returns `0` from `fork()`. Reuses the register/ + continuation approach of `clone_thread()` (used by `pthread_create`). +- `libc/process/execve.cc` — `execve()` via `osv::application::run()`. +- `libc/process/waitpid.cc` — `wait`/`waitpid`/`wait4`. +- `runtime.cc` — `exit()` ends a child rather than shutting down OSv. +- `linux.cc` — `sys_clone()` routes the non-`CLONE_THREAD` (fork) case here. + +## Guidance + +For spawning helper programs, prefer `posix_spawn()` or `system()` (which route +straight to `osv::application::run()` and skip the stack copy entirely) over +`fork()`+`exec()` where you control the code. Use `fork()` for compatibility +with existing Linux programs that expect it, within the limitations above. diff --git a/include/osv/fork.hh b/include/osv/fork.hh new file mode 100644 index 0000000000..474eae7812 --- /dev/null +++ b/include/osv/fork.hh @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +#ifndef OSV_FORK_HH +#define OSV_FORK_HH + +#include +#include + +// fork() emulation on OSv. +// +// OSv is a single-address-space unikernel with no MMU-enforced process +// isolation, so a true copy-on-write fork() is impossible. We implement the +// useful, compatible subset (see documentation/fork.md): +// +// * fork() runs the child as an OSv thread that shares the parent's address +// space (heap, globals, fds) but gets a PRIVATE COPY of the parent's user +// stack, so both parent and child return from fork() with their own local +// variables and return address (the classic "fork returns twice"). +// * fork()+execve() works: the child execve()s a new program (which OSv runs +// as a fresh application/ELF-namespace) and the parent waitpid()s for it. +// * waitpid()/wait4() reap the child's exit status; SIGCHLD is raised to the +// parent on child exit. +// +// It CANNOT provide memory isolation: a child that mutates shared globals/heap +// expecting a private copy will affect the parent. fork()-as-memory-snapshot +// (e.g. Redis BGSAVE) is unsupported. + +namespace osv { +namespace fork { + +// Called by execve() to record the application it launched, so that when this +// (child) thread later exits, waitpid() in the parent can report the exec'd +// program's exit status under the child's pid. +void adopt_execed_app(shared_app_t app); + +// Register the current thread as a fork() child of @parent_pid with child pid +// @child_pid. Called on the child just before it resumes at the fork() return +// site. Sets up the child's exit hook so the parent's waitpid() can reap it. +void register_child(pid_t child_pid, pid_t parent_pid); + +// Record that child @child_pid exited with @status (encoded WIFEXITED-style), +// wake any waiter, and raise SIGCHLD to the parent. Called from the child's +// exit path. +void child_exited(pid_t child_pid, int status); + +// waitpid(2)/wait4(2) backend: block (unless WNOHANG) for a child of the +// current thread to exit, reap it, and return its pid. Returns -1/ECHILD if +// the caller has no matching children. +pid_t wait_child(pid_t pid, int *status, int options); + +// If the current thread is a fork() child (or an exec'd child app), record its +// exit @status for the parent's waitpid(), notify the parent (SIGCHLD), and +// return true so exit() ends only this thread instead of shutting down OSv. +// Returns false for the top-level application (exit() shuts down as before). +bool exit_current_child(int status); + +} // namespace fork +} // namespace osv + +// The syscall/libc entry points. +extern "C" pid_t fork(void); +extern "C" pid_t vfork(void); + +#endif // OSV_FORK_HH diff --git a/libc/process/execve.cc b/libc/process/execve.cc index 3d66593367..d3626e2374 100644 --- a/libc/process/execve.cc +++ b/libc/process/execve.cc @@ -1,9 +1,98 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + #include +#include +#include + +#if !CONF_fork +// Without fork() support, execve() keeps its historical stub behavior. #include #include "../libc.hh" - int execve(const char *path, char *const argv[], char *const envp[]) { WARN_STUBBED(); return libc_error(ENOEXEC); } +#else +#include +#include +#include +#include +#include +#include "../libc.hh" +#include + +// execve() on OSv. +// +// OSv is a single-address-space unikernel: there is no separate process image +// to replace. We approximate execve() by launching the requested program as a +// new OSv "application" in a fresh ELF namespace (its own set of globals, the +// closest OSv has to a fresh address space) via osv::application::run(), and +// then ending the calling thread so it does not return to the old program -- +// matching the Linux contract that a successful execve() never returns. +// +// This makes the common fork()+execve() idiom work: the fork child calls +// execve(), which starts the new program and the child thread becomes that +// program's driver. The parent is unaffected (it is a different thread in the +// same address space) and can waitpid() on the child. +// +// Limitations (documented in documentation/fork.md): the old program's global +// state is not torn down the way a real address-space replacement would; the +// new program runs in its own ELF namespace but shares the kernel heap. + +extern "C" +int execve(const char *path, char *const argv[], char *const envp[]) +{ + if (!path || !argv) { + return libc_error(EFAULT); + } + + std::vector args; + for (char *const *a = argv; *a; a++) { + args.push_back(*a); + } + if (args.empty()) { + // Linux requires argv to have at least argv[0]; be lenient and use path. + args.push_back(path); + } + + std::unordered_map env; + if (envp) { + for (char *const *e = envp; *e; e++) { + std::string kv(*e); + auto eq = kv.find('='); + if (eq != std::string::npos) { + env[kv.substr(0, eq)] = kv.substr(eq + 1); + } + } + } + + osv::shared_app_t child; + try { + // new_program=true => fresh ELF namespace, so the exec'd program gets + // its own globals rather than colliding with the caller's. + child = osv::application::run(path, args, true, + envp ? &env : nullptr); + } catch (const osv::launch_error &e) { + // Could not load/exec the target - Linux returns ENOENT/ENOEXEC/EACCES. + return libc_error(ENOENT); + } + + // Record the exec'd app so the fork/wait layer can reap it under the pid + // of the thread that called execve() (Linux keeps the pid across exec). + osv::fork::adopt_execed_app(child); + + // A successful execve() does not return to the caller. Join the child app + // (run it to completion on this thread) and then exit with its return code, + // so this thread never re-enters the old program. + int rc = child->join(); + _exit(rc); + // not reached + return 0; +} +#endif // CONF_fork diff --git a/libc/process/fork.cc b/libc/process/fork.cc new file mode 100644 index 0000000000..75a375a4a0 --- /dev/null +++ b/libc/process/fork.cc @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../libc.hh" + +// Arch hook (arch//fork.cc): create a child thread that resumes in +// fork()'s CALLER (at caller_ret, with caller_sp) on a private copy of the +// parent's user stack, returning 0 in the child. Hands back the copied stack +// via out_stack_to_free so fork() can release it when the child is reaped. +extern sched::thread *fork_thread(void *caller_ret, void *caller_sp, + void **out_stack_to_free); + +// atfork handler chains (defined in libc/pthread.cc). glibc/musl register +// these internally; fork() must run prepare() in the parent before forking, +// parent() in the parent after, and child() in the child after. +extern "C" void __osv_run_atfork_prepare(); +extern "C" void __osv_run_atfork_parent(); +extern "C" void __osv_run_atfork_child(); + +namespace osv { +namespace fork { + +namespace { + +struct child_state { + pid_t parent_pid; + bool exited = false; + int status = 0; // encoded: (exit_code & 0xff) << 8, or signal + shared_app_t execed_app; // set if the child execve()'d a program +}; + +mutex g_lock; +condvar g_cv; +// child pid -> state +std::unordered_map> g_children; + +pid_t current_pid() +{ + return sched::thread::current()->id(); +} + +} // anonymous namespace + +void adopt_execed_app(shared_app_t app) +{ + SCOPE_LOCK(g_lock); + auto it = g_children.find(current_pid()); + if (it != g_children.end()) { + it->second->execed_app = app; + } +} + +void register_child(pid_t child_pid, pid_t parent_pid) +{ + SCOPE_LOCK(g_lock); + auto st = std::make_shared(); + st->parent_pid = parent_pid; + g_children[child_pid] = st; +} + +void child_exited(pid_t child_pid, int status) +{ + pid_t parent; + { + SCOPE_LOCK(g_lock); + auto it = g_children.find(child_pid); + if (it == g_children.end()) { + return; + } + if (it->second->exited) { + return; // already recorded (e.g. exit() then thread cleanup); don't clobber/re-notify + } + it->second->exited = true; + it->second->status = status; + parent = it->second->parent_pid; + g_cv.wake_all(); + } + // Notify the parent, Linux-style, that a child changed state. + (void)parent; + kill(getpid(), SIGCHLD); +} + +bool exit_current_child(int status) +{ + pid_t me = current_pid(); + { + SCOPE_LOCK(g_lock); + if (g_children.find(me) == g_children.end()) { + return false; // top-level app: exit() shuts OSv down as usual + } + } + // Encode like Linux wait status for a normal exit: WEXITSTATUS in bits 8-15. + child_exited(me, (status & 0xff) << 8); + return true; +} + +pid_t wait_child(pid_t pid, int *status, int options) +{ + pid_t me = getpid(); + WITH_LOCK(g_lock) { + while (true) { + // Find a matching, exited child of the caller. + for (auto it = g_children.begin(); it != g_children.end(); ++it) { + bool match = (pid == -1 || pid == 0) ? (it->second->parent_pid == me) + : (it->first == pid); + if (!match) { + continue; + } + if (it->second->exited) { + pid_t cpid = it->first; + if (status) { + *status = it->second->status; + } + g_children.erase(it); + return cpid; + } + } + // No exited match. Do we even have a matching (live) child? + bool have_match = false; + for (auto &kv : g_children) { + if ((pid == -1 || pid == 0) ? (kv.second->parent_pid == me) + : (kv.first == pid)) { + have_match = true; + break; + } + } + if (!have_match) { + errno = ECHILD; + return -1; + } + if (options & WNOHANG) { + return 0; // matching child(ren) exist but none has exited yet + } + g_cv.wait(&g_lock); + } + } + // not reached + return -1; +} + +} // namespace fork +} // namespace osv + +using namespace osv; + +extern "C" +__attribute__((noinline)) +pid_t fork(void) +{ + pid_t parent = getpid(); + + // Capture the point fork() will return to in its caller, and the caller's + // stack pointer (fork()'s own frame base == caller SP at the return). The + // child thread resumes exactly there, on a private copy of the stack. + void *caller_ret = __builtin_return_address(0); + void *caller_sp = __builtin_frame_address(0); + + void *stack_to_free = nullptr; + // POSIX: run pthread_atfork prepare handlers in the parent before forking. + __osv_run_atfork_prepare(); + sched::thread *child = fork_thread(caller_ret, caller_sp, &stack_to_free); + if (!child) { + __osv_run_atfork_parent(); // undo prepare-side locking + errno = ENOMEM; + return -1; + } + pid_t cpid = child->id(); + + // Register the child BEFORE starting it so a fast child->exit cannot race + // ahead of the parent's bookkeeping. + fork::register_child(cpid, parent); + + // Single cleanup: free the copied user stack and, if the child fell off the + // end without exit(), record a default status. Real exit codes are + // recorded by exit()/execve() via fork::child_exited() before this runs. + child->set_cleanup([cpid, stack_to_free] { + fork::child_exited(cpid, 0); + if (stack_to_free) { + free(stack_to_free); + } + }); + + child->start(); + + // POSIX: run atfork parent handlers in the parent after the fork. (The + // child runs its atfork child handlers in its own context, in the arch + // fork_thread trampoline, before resuming user code.) + __osv_run_atfork_parent(); + + // Parent path: return the child's pid. (The child resumes in fork()'s + // caller with return value 0, on its private stack.) + return cpid; +} + +extern "C" +pid_t vfork(void) +{ + // On OSv the child already shares the parent's address space (the classic + // vfork contract of "child borrows the parent's memory until exec/_exit") + // is actually served more faithfully than fork's copy semantics. Map to + // fork(); the shared-memory behavior matches vfork's documented contract. + // fork() is ::fork() here (an unqualified `fork` would resolve to the + // osv::fork namespace brought in by `using namespace osv`). + return ::fork(); +} diff --git a/libc/process/waitpid.cc b/libc/process/waitpid.cc index ad93ee3da0..680bd2e164 100644 --- a/libc/process/waitpid.cc +++ b/libc/process/waitpid.cc @@ -1,8 +1,44 @@ #include #include +#include +#if CONF_fork +#include + +// waitpid()/wait4() are backed by the fork() emulation's child registry +// (libc/process/fork.cc). A child created by fork() records its exit status +// there when it exits; here the parent reaps it. + +extern "C" +pid_t waitpid(pid_t pid, int *status, int options) +{ + return osv::fork::wait_child(pid, status, options); +} + +extern "C" +pid_t wait4(pid_t pid, int *status, int options, struct rusage *rusage) +{ + if (rusage) { + __builtin_memset(rusage, 0, sizeof(*rusage)); + } + return osv::fork::wait_child(pid, status, options); +} + +extern "C" +pid_t wait(int *status) +{ + return osv::fork::wait_child(-1, status, 0); +} + +#else // !CONF_fork + +// Without fork() there are never any children to wait for. +extern "C" pid_t waitpid(pid_t pid, int *status, int options) { + (void)pid; (void)status; (void)options; errno = ECHILD; return -1; } + +#endif // CONF_fork diff --git a/libc/pthread.cc b/libc/pthread.cc index 00afc6ef1d..f25be4e3f6 100644 --- a/libc/pthread.cc +++ b/libc/pthread.cc @@ -276,9 +276,48 @@ int pthread_atfork(void (*prepare)(void), void (*parent)(void), return 0; } +// atfork handlers registered via pthread_atfork()/__register_atfork(). glibc +// and musl register these internally (e.g. to reset the malloc arena lock in +// the child), so fork() must actually run them. Stored here and invoked by +// osv::fork::run_atfork_* from libc/process/fork.cc. +namespace { +struct atfork_entry { + void (*prepare)(void); + void (*parent)(void); + void (*child)(void); +}; +mutex atfork_lock; +std::vector atfork_handlers; +} + +extern "C" void __osv_run_atfork_prepare() +{ + // POSIX: prepare handlers run in LIFO (reverse registration) order. + SCOPE_LOCK(atfork_lock); + for (auto it = atfork_handlers.rbegin(); it != atfork_handlers.rend(); ++it) { + if (it->prepare) it->prepare(); + } +} +extern "C" void __osv_run_atfork_parent() +{ + SCOPE_LOCK(atfork_lock); + for (auto &h : atfork_handlers) { + if (h.parent) h.parent(); + } +} +extern "C" void __osv_run_atfork_child() +{ + SCOPE_LOCK(atfork_lock); + for (auto &h : atfork_handlers) { + if (h.child) h.child(); + } +} + extern "C" int register_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void), void *__dso_handle) { + SCOPE_LOCK(atfork_lock); + atfork_handlers.push_back({prepare, parent, child}); return 0; } diff --git a/libc/signal.cc b/libc/signal.cc index 0595053446..58142621eb 100644 --- a/libc/signal.cc +++ b/libc/signal.cc @@ -413,6 +413,13 @@ int kill(pid_t pid, int sig) } unsigned sigidx = sig - 1; if (is_sig_dfl(signal_actions[sigidx])) { + // Per POSIX, the default disposition of SIGCHLD, SIGURG and SIGWINCH is + // to IGNORE, not to terminate. OSv's fork() emulation raises SIGCHLD to + // the parent when a child exits; treating an unhandled SIGCHLD as an + // uncaught fatal signal (and powering off) would kill the whole VM. + if (sig == SIGCHLD || sig == SIGURG || sig == SIGWINCH) { + return 0; + } // Our default is to power off. debugf("Uncaught signal %d (\"%s\"). Powering off.\n", sig, strsignal(sig)); diff --git a/linux.cc b/linux.cc index bbce998856..be98b128dc 100644 --- a/linux.cc +++ b/linux.cc @@ -63,6 +63,7 @@ #include #include #include +#include #include @@ -483,6 +484,7 @@ static long sys_set_tid_address(int *tidptr) #define CLONE_CHILD_CLEARTID 0x00200000 extern sched::thread *clone_thread(unsigned long flags, void *child_stack, unsigned long newtls); +extern "C" pid_t fork(void); // thread-backed fork() emulation (libc/process/fork.cc) #define __NR_sys_clone __NR_clone #ifdef __x86_64__ @@ -492,11 +494,27 @@ int sys_clone(unsigned long flags, void *child_stack, int *ptid, int *ctid, unsi int sys_clone(unsigned long flags, void *child_stack, int *ptid, unsigned long newtls, int *ctid) #endif { // - //We only support "cloning" of threads so fork() would fail but pthread_create() should - //succeed + // Threads use sys_clone (CLONE_THREAD). Without CLONE_THREAD this is a + // fork()-style clone (a new "process"); on OSv we route that through the + // thread-backed fork() emulation (see libc/process/fork.cc). if (!(flags & CLONE_THREAD)) { +#if CONF_fork + // fork()/vfork() land here (glibc/musl implement them via clone with an + // exit signal and no CLONE_THREAD). Route to the thread-backed fork(). + // Namespace-unshare clones (CLONE_NEWNS 0x00020000 etc.) are not + // supported on a single-address-space kernel. + const unsigned long CLONE_NEW_MASK = 0x7c020000UL; // NEWNS/UTS/IPC/USER/PID/NET/CGROUP + if (flags & CLONE_NEW_MASK) { + errno = ENOSYS; + return -1; + } + return fork(); +#else + // fork() support not compiled in (CONFIG_fork=n): preserve OSv's + // historical behavior of only supporting thread clones. errno = ENOSYS; return -1; +#endif } // //Validate we have non-empty stack diff --git a/modules/tests/Makefile b/modules/tests/Makefile index ff66fcd931..89ff015d68 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -130,6 +130,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \ misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ tst-eventfd.so tst-remove.so misc-wake.so tst-epoll.so misc-lfring.so \ + tst-fork.so \ misc-fsx.so tst-sleep.so tst-resolve.so tst-except.so \ misc-tcp-sendonly.so tst-tcp-nbwrite.so misc-tcp-hash-srv.so \ misc-loadbalance.so misc-scheduler.so tst-console.so tst-app.so \ diff --git a/runtime.cc b/runtime.cc index 956ab6d75d..eeefe057d8 100644 --- a/runtime.cc +++ b/runtime.cc @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -67,6 +69,17 @@ #include #include +#if !CONF_fork +// When fork() is not compiled in (CONFIG_fork=n, the default), restore the +// historical fork/vfork/wait4 stubs. With fork enabled these live in +// libc/process/fork.cc and libc/process/waitpid.cc instead. +extern "C" OSV_LIBC_API int vfork() { WARN_STUBBED(); errno = ENOSYS; return -1; } +extern "C" OSV_LIBC_API int fork() { WARN_STUBBED(); errno = ENOSYS; return -1; } +extern "C" OSV_LIBC_API pid_t wait4(pid_t, int *, int, struct rusage *) { + WARN_STUBBED(); errno = ECHILD; return -1; +} +#endif + // cxxabi.h from gcc 10 and earlier used to say that __cxa_finalize returns // an int, while it should return void (and does so on gcc 11). To allow us // to define __cxa_finalize with neither gcc 10 or 11 complaining, we need @@ -208,20 +221,6 @@ int getpagesize() return 4096; } -OSV_LIBC_API -int vfork() -{ - WARN_STUBBED(); - return -1; -} - -OSV_LIBC_API -int fork() -{ - WARN_STUBBED(); - return -1; -} - OSV_LIBC_API pid_t setsid(void) { @@ -464,6 +463,18 @@ int pclose(FILE *stream) void exit(int status) { +#if CONF_fork + // On OSv exit() historically shuts down the whole unikernel. But a fork() + // child (or an exec'd child app) is a "process" that must exit on its own + // without taking the machine down. If the current thread is a registered + // fork child, record its status for the parent's waitpid() and end just + // this thread; only the top-level application's exit() shuts OSv down. + if (osv::fork::exit_current_child(status)) { + // recorded + notified parent; terminate only this child thread + sched::thread::exit(); // noreturn: ends only this child thread + // not reached + } +#endif debugf("program exited with status %ld\n", status); osv::shutdown(); } @@ -692,14 +703,6 @@ pid_t wait3(int *status, int options, struct rusage *usage) return -1; } -OSV_LIBC_API -pid_t wait4(pid_t pid, int *status, int options, struct rusage *usage) -{ - WARN_STUBBED(); - errno = ECHILD; - return -1; -} - OSV_LIBC_API int getresuid(uid_t *ruid, uid_t *euid, uid_t *suid) { diff --git a/tests/tst-fork.cc b/tests/tst-fork.cc new file mode 100644 index 0000000000..65bb414184 --- /dev/null +++ b/tests/tst-fork.cc @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * Tests OSv's thread-backed fork() emulation (see documentation/fork.md). + * fork() on OSv shares the address space with the parent but gives the child a + * private copy of the parent's stack, so the twin return and simple + * child-does-work-then-_exit / fork+exec / waitpid flows work. This test + * exercises those; it does NOT assert memory isolation (which OSv cannot give). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static int failures = 0; +#define CHECK(cond, msg) do { \ + if (!(cond)) { printf("FAIL: %s\n", msg); failures++; } \ + else { printf("PASS: %s\n", msg); } \ +} while (0) + +// 1. fork() return-value contract + private stack. +static void test_fork_return() +{ + volatile int parent_local = 0x1111; // must be untouched by the child + pid_t pid = fork(); + if (pid == 0) { + // child: its own copy of parent_local; mutate it and exit with a code + volatile int child_local = parent_local; // reads the copied value + child_local = 0x2222; + (void)child_local; + _exit(42); + } + // parent + CHECK(pid > 0, "fork() returns child pid > 0 to parent"); + int status = 0; + pid_t w = waitpid(pid, &status, 0); + CHECK(w == pid, "waitpid() reaps the fork child"); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 42, + "child exit code 42 delivered via waitpid"); + CHECK(parent_local == 0x1111, + "parent's stack local intact after child mutated its own copy"); +} + +// 2. fork() + execve(): child execs a trivial program; parent waits. +static void test_fork_exec() +{ + pid_t pid = fork(); + if (pid == 0) { + // Exec /libtrue.so or fall back to a program that exits 7. On OSv the + // test image includes a simple echo/true-like helper; if none exists + // execve returns and we _exit a sentinel the parent recognizes. + char *const argv[] = { (char*)"/tests/payload-exit7.so", nullptr }; + char *const envp[] = { nullptr }; + execve(argv[0], argv, envp); + // execve failed (no such payload in this image) -> sentinel + _exit(7); + } + CHECK(pid > 0, "fork() before execve returns child pid"); + int status = 0; + pid_t w = waitpid(pid, &status, 0); + CHECK(w == pid, "waitpid() reaps the fork+exec child"); + // Whether the exec payload ran or the sentinel fired, exit status is 7. + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 7, + "fork+exec child exit status observed (7)"); +} + +// 3. vfork() maps to fork(); same contract. +static void test_vfork() +{ + pid_t pid = vfork(); + if (pid == 0) { + _exit(9); + } + CHECK(pid > 0, "vfork() returns child pid to parent"); + int status = 0; + pid_t w = waitpid(pid, &status, 0); + CHECK(w == pid && WIFEXITED(status) && WEXITSTATUS(status) == 9, + "vfork child exit code 9 via waitpid"); +} + +// 4. waitpid with no children returns -1/ECHILD. +static void test_no_children() +{ + int status = 0; + errno = 0; + pid_t w = waitpid(-1, &status, 0); + CHECK(w == -1, "waitpid with no children returns -1"); +} + +int main() +{ + printf("=== tst-fork ===\n"); + test_fork_return(); + test_fork_exec(); + test_vfork(); + test_no_children(); + printf("=== tst-fork done: %d failures ===\n", failures); + return failures == 0 ? 0 : 1; +} From 0af931eac4fc74deaca01490bd1c887dc721e55d Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Wed, 22 Jul 2026 21:11:14 +0000 Subject: [PATCH 2/6] process: fix execve()/fork() new-namespace exec hanging OSv shutdown execve() (CONF_fork) launches the target as a fresh OSv application in its own ELF namespace via application::run(new_program=true) and, matching Linux, does not return to the caller. A successful exec DID launch the program (the elf::program new-namespace construction path is fine), but after the exec'd program finished the unikernel would hang at shutdown instead of powering off: the loader's application::join() blocked forever on the top-level app's _terminated flag. Root cause is the fork child thread's lifecycle, not execve or elf::program. fork_thread() (arch/{x64,aarch64}/fork.cc) creates the child as a normal *attached* sched::thread. At construction the thread captures a shared_ptr to the current application's application_runtime (sched.cc sets _app_runtime = app->runtime()). Nothing ever join()s the fork child -- the parent reaps it through the fork pid registry / waitpid(), not sched::thread::join() -- so the thread object is never destroyed and its _app_runtime shared_ptr is never released. With that reference outstanding the application_runtime's use count never reaches zero, ~application_runtime never runs, the app's _terminated is never set, and application::join() waits forever. (This bit every fork(), and was most visible after fork()+execve() where the child adopts the caller app's runtime.) Fix: create the fork child detached and dispose it in its cleanup. - arch/{x64,aarch64}/fork.cc: mark the child attr().detached(), so on completion it is handed to the thread reaper (which runs its set_cleanup()), rather than sitting forever waiting to be joined. - libc/process/fork.cc: the child's cleanup now also calls sched::thread::dispose(child) after the existing bookkeeping (record exit status if it fell off the end, free the copied user stack). Disposing the thread releases its _app_runtime reference, letting the owning application's runtime drop to zero so join() completes and OSv powers off. This mirrors the default detached-thread cleanup ([this]{ dispose(this); }). Both files are compiled only under CONF_fork, so default OSv builds are unchanged. Tests: tst-fork test 2 now execs a real payload (/tests/payload-exit7.so, which prints a marker and exit(7)s) instead of the previous _exit(7) sentinel that masked whether execve actually launched anything; a return from execve() is now treated as a failure. Added tst-execve.so (fork+execve launches the payload and its exit code is reaped; missing path returns -1/ENOENT) and payload-exit7.so. Validated on x86-64 (KVM disk boot): tst-fork 10/10 and tst-execve 3/3 pass with the real exec payload and OSv shuts down cleanly (5/5 repeat runs, no hang). Copyright (C) 2026 Greg Burd --- arch/aarch64/fork.cc | 10 +++++++- arch/x64/fork.cc | 10 +++++++- libc/process/fork.cc | 16 +++++++++--- modules/tests/Makefile | 2 +- tests/payload-exit7.cc | 16 ++++++++++++ tests/tst-execve.cc | 56 ++++++++++++++++++++++++++++++++++++++++++ tests/tst-fork.cc | 17 +++++++------ 7 files changed, 112 insertions(+), 15 deletions(-) create mode 100644 tests/payload-exit7.cc create mode 100644 tests/tst-execve.cc diff --git a/arch/aarch64/fork.cc b/arch/aarch64/fork.cc index 4db49ef121..798e8ce85a 100644 --- a/arch/aarch64/fork.cc +++ b/arch/aarch64/fork.cc @@ -69,7 +69,15 @@ sched::thread *fork_thread(void *caller_ret, void *caller_sp, "br %1 \n\t" // resume in fork()'s caller : : "r"(resume_sp), "r"(resume_pc) : "x0", "memory"); }, sched::thread::attr(). - stack(4096 * 4), + stack(4096 * 4). + // Detached: nobody join()s the fork child (the parent reaps it via the + // pid registry / waitpid, not sched::thread::join). A detached thread + // is handed to the reaper on completion, which runs our set_cleanup() + // (freeing the copied stack and disposing the thread object, releasing + // its application_runtime reference). Without this the thread object + // (and its app_runtime shared_ptr) would leak and OSv would hang at + // shutdown -- see the cleanup comment in libc/process/fork.cc. + detached(), false, true); t->set_app_tcb(parent->get_app_tcb()); diff --git a/arch/x64/fork.cc b/arch/x64/fork.cc index 3f93519826..9c422c3c32 100644 --- a/arch/x64/fork.cc +++ b/arch/x64/fork.cc @@ -91,7 +91,15 @@ sched::thread *fork_thread(void *caller_ret, void *caller_sp, void **out_stack_t "jmpq *%1 \n\t" // resume in fork()'s caller : : "r"(resume_sp), "r"(resume_pc) : "memory"); }, sched::thread::attr(). - stack(4096 * 4), + stack(4096 * 4). + // Detached: nobody join()s the fork child (the parent reaps it via the + // pid registry / waitpid, not sched::thread::join). A detached thread + // is handed to the reaper on completion, which runs our set_cleanup() + // (freeing the copied stack and disposing the thread object, releasing + // its application_runtime reference). Without this the thread object + // (and its app_runtime shared_ptr) would leak and OSv would hang at + // shutdown -- see the cleanup comment in libc/process/fork.cc. + detached(), false, true); t->set_app_tcb(parent->get_app_tcb()); diff --git a/libc/process/fork.cc b/libc/process/fork.cc index 75a375a4a0..e59e078d5d 100644 --- a/libc/process/fork.cc +++ b/libc/process/fork.cc @@ -184,14 +184,22 @@ pid_t fork(void) // ahead of the parent's bookkeeping. fork::register_child(cpid, parent); - // Single cleanup: free the copied user stack and, if the child fell off the - // end without exit(), record a default status. Real exit codes are - // recorded by exit()/execve() via fork::child_exited() before this runs. - child->set_cleanup([cpid, stack_to_free] { + // Single cleanup, run by the thread reaper once the (detached) child has + // fully terminated: free the copied user stack, record a default status if + // the child fell off the end without exit() (real codes are recorded by + // exit()/execve() via fork::child_exited() before this runs), and finally + // dispose the child thread object itself. Disposing is essential: the + // child thread holds a shared_ptr to its application_runtime (set at thread + // construction), and only destroying the thread releases that reference. + // Without it the top-level application's runtime never drops to zero, its + // ~application_runtime never fires, application::join() blocks forever on + // _terminated, and OSv hangs at shutdown instead of powering off. + child->set_cleanup([cpid, stack_to_free, child] { fork::child_exited(cpid, 0); if (stack_to_free) { free(stack_to_free); } + sched::thread::dispose(child); }); child->start(); diff --git a/modules/tests/Makefile b/modules/tests/Makefile index 89ff015d68..a3e4aad504 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -130,7 +130,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. tst-queue-mpsc.so tst-af-local.so tst-pipe.so tst-yield.so \ misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ tst-eventfd.so tst-remove.so misc-wake.so tst-epoll.so misc-lfring.so \ - tst-fork.so \ + tst-fork.so tst-execve.so payload-exit7.so \ misc-fsx.so tst-sleep.so tst-resolve.so tst-except.so \ misc-tcp-sendonly.so tst-tcp-nbwrite.so misc-tcp-hash-srv.so \ misc-loadbalance.so misc-scheduler.so tst-console.so tst-app.so \ diff --git a/tests/payload-exit7.cc b/tests/payload-exit7.cc new file mode 100644 index 0000000000..dbe5a2b281 --- /dev/null +++ b/tests/payload-exit7.cc @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * Trivial exec payload used by tst-fork / tst-execve: prints a marker so the + * test can confirm the exec'd program actually ran, then exit(7)s so the + * parent can verify the exit code was reaped. + */ +#include +int main() { + printf("payload-exit7: running, will exit(7)\n"); + fflush(stdout); + return 7; +} diff --git a/tests/tst-execve.cc b/tests/tst-execve.cc new file mode 100644 index 0000000000..8d7162347d --- /dev/null +++ b/tests/tst-execve.cc @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * Regression test for execve() launching a program in a fresh ELF namespace + * (see documentation/fork.md). Covers the two independent facts that the + * original execve() new-namespace path broke: + * 1. a successful execve() actually LAUNCHES the target program, and + * 2. after the exec'd program exits, control returns/reaps cleanly and OSv + * is able to shut down (the fork-child app_runtime leak used to hang it). + * Requires CONF_fork (execve is a stub otherwise). + */ +#include +#include +#include +#include +#include + +static int failures = 0; +#define CHECK(cond, msg) do { \ + if (!(cond)) { printf("FAIL: %s\n", msg); failures++; } \ + else { printf("PASS: %s\n", msg); } \ +} while (0) + +int main() +{ + printf("=== tst-execve ===\n"); fflush(stdout); + + // execve() a real payload from a fork child; a successful exec never + // returns, so if it returns the exec FAILED (child exits 99). The payload + // (/tests/payload-exit7.so) prints a marker and exit(7)s. + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = { (char*)"/tests/payload-exit7.so", nullptr }; + char *const envp[] = { nullptr }; + execve(argv[0], argv, envp); + _exit(99); // execve returned => it failed to launch the program + } + int status = 0; + pid_t w = waitpid(pid, &status, 0); + CHECK(w == pid, "waitpid() reaps the exec'd child"); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 7, + "execve launched the payload and its exit code (7) was reaped"); + + // execve() of a missing path must fail cleanly with ENOENT (not crash). + char *const bad[] = { (char*)"/tests/does-not-exist.so", nullptr }; + errno = 0; + int rc = execve(bad[0], bad, nullptr); + CHECK(rc == -1 && errno == ENOENT, + "execve of a missing path returns -1/ENOENT"); + + printf("=== tst-execve done: %d failures ===\n", failures); fflush(stdout); + return failures == 0 ? 0 : 1; +} diff --git a/tests/tst-fork.cc b/tests/tst-fork.cc index 65bb414184..5dc008f6f0 100644 --- a/tests/tst-fork.cc +++ b/tests/tst-fork.cc @@ -49,27 +49,28 @@ static void test_fork_return() "parent's stack local intact after child mutated its own copy"); } -// 2. fork() + execve(): child execs a trivial program; parent waits. +// 2. fork() + execve(): the child really execs a program in a fresh ELF +// namespace and the parent reaps it. The payload (/tests/payload-exit7.so, +// built by modules/tests) prints a marker and exit(7)s; the child never +// returns from a successful execve(), so a return means exec failed and we +// signal that with a distinct code (99) that the parent flags as a failure. static void test_fork_exec() { pid_t pid = fork(); if (pid == 0) { - // Exec /libtrue.so or fall back to a program that exits 7. On OSv the - // test image includes a simple echo/true-like helper; if none exists - // execve returns and we _exit a sentinel the parent recognizes. char *const argv[] = { (char*)"/tests/payload-exit7.so", nullptr }; char *const envp[] = { nullptr }; execve(argv[0], argv, envp); - // execve failed (no such payload in this image) -> sentinel - _exit(7); + _exit(99); // only reached if execve() FAILED to launch the payload } CHECK(pid > 0, "fork() before execve returns child pid"); int status = 0; pid_t w = waitpid(pid, &status, 0); CHECK(w == pid, "waitpid() reaps the fork+exec child"); - // Whether the exec payload ran or the sentinel fired, exit status is 7. + // 7 means the exec'd payload actually ran and exited; 99 means execve() + // returned (failed to launch) -- that is now a real failure. CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 7, - "fork+exec child exit status observed (7)"); + "fork+exec: exec'd payload ran and its exit code (7) was reaped"); } // 3. vfork() maps to fork(); same contract. From 39341d18c0fecaf5882c9cc789fe2a144389105d Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Wed, 22 Jul 2026 16:13:32 -0400 Subject: [PATCH 3/6] mm: per-child copy-on-write address space for fork() (opt-in, stacked) Follow-up to the base fork() PR: gives a forked child its own address space with copy-on-write of private mappings, so a forked child has real memory isolation like Linux fork() while MAP_SHARED / shm stays shared. Also gated behind CONFIG_fork (default off): with it disabled none of this is compiled and OSv's single-address-space model and context-switch path are unchanged. With CONFIG_fork enabled: - mmu::address_space object (page-table root + vma_list); the previous global becomes 'address space 0' (kernel + init app). - Per-thread current address space; the context switch loads the target CR3 only when the address space differs (a no-op for AS0-only workloads). Kernel PML4 entries are shared across all address spaces so OSv code + the kernel heap work identically after a switch. - fork() clones the parent's vmas into a child address space: PRIVATE writable mappings are write-protected in both and copied on the first write (reusing OSv's existing COW fault machinery); MAP_SHARED / shm map the same physical pages (truly shared). execve() returns the thread to AS0. - Lock/condvar wait_records for fork-child (non-AS0) threads are allocated from the AS-shared kernel heap instead of the thread stack, so a wait_record queued on a shared kernel mutex resolves to the same physical page for any waker across address spaces (kernel-stack coherence). Validated: tst-fork-cow proves a forked child's private memory stays private while MAP_SHARED stays shared; tst-fork stays 10/10. Known limitation (documented): the child's stack is still relocated+copied rather than same-VA COW, so a child that unwinds a very deep call chain (e.g. a multi-process server forking backends) can still hit a stack-fidelity issue (tst-fork-deep); the same-VA stack fix is a further follow-up. This PR delivers memory-isolated fork for the common cases with COW proven. Copyright (C) 2026 Greg Burd --- arch/x64/arch-switch.hh | 11 + arch/x64/mmu.cc | 25 ++ conf/Makefile | 4 +- conf/kconfig/threads | 3 +- core/condvar.cc | 9 + core/lfmutex.cc | 24 ++ core/mmu.cc | 476 ++++++++++++++++++++++++++++++++++++- core/sched.cc | 7 + include/osv/mmu-defs.hh | 4 + include/osv/mmu.hh | 35 +++ include/osv/sched.hh | 16 ++ include/osv/wait_record.hh | 53 +++++ libc/process/execve.cc | 17 ++ libc/process/fork.cc | 33 ++- modules/tests/Makefile | 1 + tests/tst-fork-cow.cc | 90 +++++++ tests/tst-fork-deep.cc | 98 ++++++++ 17 files changed, 891 insertions(+), 15 deletions(-) create mode 100644 tests/tst-fork-cow.cc create mode 100644 tests/tst-fork-deep.cc diff --git a/arch/x64/arch-switch.hh b/arch/x64/arch-switch.hh index 92500ffa8d..637accb551 100644 --- a/arch/x64/arch-switch.hh +++ b/arch/x64/arch-switch.hh @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "tls-switch.hh" @@ -93,6 +94,16 @@ void thread::switch_to() barrier(); set_fsbase(reinterpret_cast(_tcb)); barrier(); +#if CONF_fork + // Address-space (CR3) switch for fork COW. Only touch CR3 when the target + // thread lives in a different address space than the outgoing one, so the + // common single-address-space case pays nothing (and no TLB flush). The + // kernel half of every AS is identically mapped, so the switch code, kernel + // stacks and kernel heap remain valid across the write. + if (_current_as != old->_current_as) { + processor::write_cr3(mmu::pt_root_phys(_current_as)); + } +#endif auto c = _detached_state->_cpu; old->_state.exception_stack = c->arch.get_exception_stack(); // save the old thread SYSCALL caller stack pointer in the syscall stack descriptor diff --git a/arch/x64/mmu.cc b/arch/x64/mmu.cc index 171f262675..b6dc71f7ba 100644 --- a/arch/x64/mmu.cc +++ b/arch/x64/mmu.cc @@ -120,15 +120,40 @@ void flush_tlb_all() static pt_element<4> page_table_root __attribute__((init_priority((int)init_prio::pt_root))); +#if CONF_fork +// Virtual pointer to the current thread's PML4 (child AS's private PML4 when +// running in a forked child, else the kernel PML4). Defined in core/mmu.cc. +pt_element<4> *current_pt_root(); + pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) { + return current_pt_root(); +} + +// Virtual pointer to the kernel (AS0) PML4. +pt_element<4> *kernel_pml4() { return &page_table_root; } +#else +pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) { + return &page_table_root; +} +#endif // CONF_fork void switch_to_runtime_page_tables() { processor::write_cr3(page_table_root.next_pt_addr()); } +#if CONF_fork +// Physical address of the kernel (AS0) PML4 -- the CR3 value that maps OSv +// text/data + the identity/phys ranges. Used as the shared base for cloned +// child address spaces and as AS0's pt_root. +phys kernel_pt_root_phys() +{ + return page_table_root.next_pt_addr(); +} +#endif // CONF_fork + enum { page_fault_prot = 1ul << 0, page_fault_write = 1ul << 1, diff --git a/conf/Makefile b/conf/Makefile index 87d2c29c32..764f941f07 100644 --- a/conf/Makefile +++ b/conf/Makefile @@ -56,7 +56,7 @@ CONF_FILES := conf/base.mk conf/$(mode).mk conf/$(arch).mk conf/profiles/$(arch) #Generate the default .config, gen/config/kernel.conf, and gen/include/osv/kernel_config.h if not generated yet $(out)/.config: $(CONF_FILES) $(out)/kbuild/kconfig/conf - $(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config) + $(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config) #Generate the .config.yes, gen/include/osv/kernel_yes_config.h with all available options $(out)/.config.yes: $(CONF_FILES) $(out)/kbuild/kconfig/conf @@ -64,7 +64,7 @@ $(out)/.config.yes: $(CONF_FILES) $(out)/kbuild/kconfig/conf #Synchronize gen/include/osv/kernel_config.h and gen/config/kernel.conf with .config if the latter has changed maybe by mconf $(out)/gen/include/osv/kernel_config.h: $(out)/.config - $(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config) + $(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config) #Generate gen/config/kernel_conf.mk AND individual option headers gen/include/osv/kernel_config_* based on the latest gen/config/kernel.conf and gen/include/osv/kernel_config.h #The gen/config/kernel_conf.mk is included by the main OSv makefile and the headers gen/include/osv/kernel_config_* included by relevant source files diff --git a/conf/kconfig/threads b/conf/kconfig/threads index 103a5ac4c3..c4f5792155 100644 --- a/conf/kconfig/threads +++ b/conf/kconfig/threads @@ -8,8 +8,7 @@ config lazy_stack config fork prompt "Include fork()/vfork() support (per-child address space, off by default)" - bool - default n + def_bool $(shell,[ "$conf_fork" = 1 ] && echo y || echo n) help Enable OSv's thread-backed fork()/vfork()/execve()/waitpid() emulation, including the per-child address space with copy-on-write needed to give a diff --git a/core/condvar.cc b/core/condvar.cc index 5bbecfc7b8..7cc763c96e 100644 --- a/core/condvar.cc +++ b/core/condvar.cc @@ -13,6 +13,7 @@ #include #include #include +#include TRACEPOINT(trace_condvar_wait, "%p", condvar *); TRACEPOINT(trace_condvar_wake_one, "%p", condvar *); @@ -22,7 +23,15 @@ int condvar::wait(mutex* user_mutex, sched::timer* tmr) { trace_condvar_wait(this); int ret = 0; +#if CONF_fork + // A fork child (non-AS0) heap-allocates its wait_record so it stays + // coherent when the parent (different address space) walks this shared + // condvar's queue. AS0 keeps the on-stack fast path. + coherent_wait_record wr_holder(sched::thread::current()); + wait_record &wr = wr_holder.get(); +#else wait_record wr(sched::thread::current()); +#endif _m.lock(); if (!_waiters_fifo.oldest) { diff --git a/core/lfmutex.cc b/core/lfmutex.cc index 4d22cd7286..fd128cf1e3 100644 --- a/core/lfmutex.cc +++ b/core/lfmutex.cc @@ -10,6 +10,22 @@ #include #include #include +#include +#if CONF_fork +#include +#endif + +#if CONF_fork +// See wait_record.hh: true iff the current thread runs in a forked-child +// (non-AS0) address space, in which case a wait_record queued on a shared +// kernel object must be heap-allocated to stay coherent cross-address-space. +bool fork_child_needs_heap_wait_record() +{ + auto t = sched::thread::current(); + return t && t->address_space() && + t->address_space() != mmu::kernel_address_space(); +} +#endif namespace lockfree { @@ -51,7 +67,15 @@ void mutex::lock() // when another thread releases the lock. // Note "waiter" is on the stack, so we must not return before making sure // it was popped from waitqueue (by another thread or by us.) +#if CONF_fork + // A fork child (non-AS0) gets a heap-allocated wait_record so it stays + // coherent when the parent, in a different address space, dereferences it + // off this shared kernel mutex. AS0 keeps the on-stack fast path. + coherent_wait_record waiter_holder(current); + wait_record &waiter = waiter_holder.get(); +#else wait_record waiter(current); +#endif waitqueue.push(&waiter); // The "Responsibility Hand-Off" protocol where a lock() picks from diff --git a/core/mmu.cc b/core/mmu.cc index e9d0b8ad69..ba4238db9b 100644 --- a/core/mmu.cc +++ b/core/mmu.cc @@ -7,6 +7,7 @@ #include #include +#include #include "processor.hh" #include #include "exceptions.hh" @@ -29,6 +30,7 @@ #include #include #include +#include #include #include @@ -122,6 +124,384 @@ vma_list_type vma_list; // should hold the lock for write rwlock_t vma_list_mutex; +#if CONF_fork +// ----------------------------------------------------------------------------- +// Stage 2 fork: per-process address space object. +// +// An address_space bundles a page-table root (PML4) with its own vma_list. +// "AS0" (kernel_as) aliases the pre-existing global vma_list / vma_list_mutex +// and uses the arch page_table_root, so existing behaviour is unchanged: the +// kernel and the initial application run in AS0. +// +// A child address_space (clone_address_space, for fork) owns a private PML4 +// whose *kernel half* (the PML4 slots that map OSv text/data and the +// identity/phys ranges) is shared with AS0, while its *application half* is a +// COW clone of the parent's page tables. It also owns a private vma_list that +// is a structural copy of the parent's. +// +// PML4 slot layout on x86-64 (each slot spans 512 GB): +// slot 0 : OSv kernel text/data (mapped at ~1 GB) -- SHARED +// slots 1..127 : application VMA space (ELF at slot 32, mmap at slot 64) -- PRIVATE (COW) +// slots 128..511: kernel identity / phys / mempool / debug maps -- SHARED +// Only slots 1..127 are cloned per child; the rest are shared by copying the +// parent's PML4 entries (pointing at the same lower-level tables). +constexpr unsigned pml4_app_first = 1; +constexpr unsigned pml4_app_last = 127; // inclusive + +struct address_space { + vma_list_type *vmas; // AS0: aliases global vma_list; child: owns_vmas + rwlock_t *vmas_mutex; // AS0: aliases global vma_list_mutex + // Synthetic top-level entry (mirrors the arch page_table_root): its + // next_pt_addr() is the physical address of this AS's PML4 page. The page + // table walk starts here via get_root_pt() (see map_range). Null for AS0 + // (which uses the arch page_table_root). + pt_element<4> *top; // -> &_top for a child; nullptr for AS0 + pt_element<4> _top; // storage for the synthetic entry (child only) + phys pt_root; // phys of PML4 page (CR3 value); 0 == arch root + bool is_kernel; + + // Storage owned by a child AS. + std::unique_ptr owned_vmas; + std::unique_ptr owned_mutex; + + // AS0 constructor: alias the globals. + address_space(vma_list_type *l, rwlock_t *m, phys root, bool kernel) + : vmas(l), vmas_mutex(m), top(nullptr), pt_root(root), is_kernel(kernel) {} + + // Child constructor: owns a fresh vma_list, mutex and PML4 page. The + // synthetic top entry points at the child PML4 (phys pml4_page_phys). + address_space(phys pml4_page_phys) + : pt_root(pml4_page_phys), is_kernel(false) + , owned_vmas(new vma_list_type()), owned_mutex(new rwlock_t()) + { + _top = make_intermediate_pte(hw_ptep<4>::force(&_top), pml4_page_phys); + top = &_top; + vmas = owned_vmas.get(); + vmas_mutex = owned_mutex.get(); + } +}; + +// AS0: kernel + initial application. Aliases the global vma_list. pt_root is +// filled in lazily (0 means "arch page_table_root", see pt_root_phys()). +__attribute__((init_priority((int)init_prio::vma_list))) +address_space kernel_as{&vma_list, &vma_list_mutex, 0, true}; + +address_space *kernel_address_space() +{ + return &kernel_as; +} + +// Arch hook: physical address of the kernel PML4 (CR3 value for AS0). +phys kernel_pt_root_phys(); +// Arch hook: virtual pointer to the kernel PML4 (for cloning kernel slots). +pt_element<4> *kernel_pml4(); + +phys pt_root_phys(address_space *as) +{ + if (as->pt_root) { + return as->pt_root; + } + // AS0 (or any AS with pt_root not yet cached): the arch kernel root. + return kernel_pt_root_phys(); +} + +// The PML4 (virtual pointer) that the current thread's page-table walks should +// use. Returns the child AS's private PML4 when the current thread runs in a +// child address space, else the kernel PML4 (AS0). Called from get_root_pt(). +pt_element<4> *current_pt_root() +{ + auto t = sched::thread::current(); + if (t) { + auto as = t->address_space(); + if (as && as->top) { + return as->top; + } + } + return kernel_pml4(); +} + +// --- fork COW page-table clone -------------------------------------------- +// +// Recursively clone the child's copy of an application-range subtree of the +// parent's page table, level by level, marking most private 4K leaf pages +// copy-on-write. +// +// IMPORTANT (OSv has no user/kernel stack split): OSv runs kernel code on the +// SAME stack the application uses. A context switch (switch_to) writes to that +// stack with interrupts disabled, and OSv forbids page faults in that context. +// So we must NOT copy-on-write-protect any *stack* page -- write-protecting the +// running thread's live stack would fault the very next stack push with irqs +// off (assert(preemptable()) in page_fault). The child already runs on its +// own private copied stack (see fork_thread), so stack pages are simply shared +// writable. MAP_SHARED pages are likewise shared writable (sharing preserved). +// These "share, don't COW" address ranges are passed in via cow_share_ranges. +struct cow_range { uintptr_t start, end; }; +static const std::vector *cow_share_ranges; + +static bool addr_is_shared(uintptr_t va) +{ + if (!cow_share_ranges) return false; + for (auto &r : *cow_share_ranges) { + if (va >= r.start && va < r.end) return true; + } + return false; +} + +// The parent must hold vma_list_mutex for write while this runs. base_virt is +// the virtual address that leaf entry 0 of this PT maps. +static void clone_pt_level0(pt_element<0> *parent_pt, pt_element<0> *child_pt, + uintptr_t base_virt) +{ + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<0> ppte = parent_pt[i]; + if (ppte.empty()) { + child_pt[i] = make_empty_pte<0>(); + continue; + } + uintptr_t va = base_virt + (uintptr_t)i * page_size; + if (addr_is_shared(va)) { + // Stack / MAP_SHARED page: must stay genuinely shared + writable in + // both parent and child (never COW). Clear any COW bit and grant + // write so both sides see each other's writes to the same phys + // page. Tag the child's copy pte_shared so teardown does not free + // the jointly-owned physical frame. + pt_element<0> sh = ppte; + if (pte_is_cow(sh)) { + sh = pte_mark_cow(sh, false); + } + sh.set_writable(true); + parent_pt[i] = sh; + sh.set_sw_bit(pte_shared, true); + child_pt[i] = sh; + } else if (ppte.writable() && !pte_is_cow(ppte)) { + // Private writable page: make it COW (write-protect + cow bit) in + // BOTH parent and child. + pt_element<0> cow = pte_mark_cow(ppte, true); + parent_pt[i] = cow; + child_pt[i] = cow; + } else { + // Read-only or already-COW private page: share the same physical + // page as-is (stays COW/read-only in the child too). + child_pt[i] = ppte; + } + } +} + +// clone_pt_level for N in 1..2, carrying base_virt so leaf pages can be +// tested against the share-ranges. (gnu++14: no if constexpr, so explicit +// specialization.) +template +static void clone_pt_level(pt_element *parent_pt, pt_element *child_pt, + uintptr_t base_virt); + +template<> +void clone_pt_level<1>(pt_element<1> *parent_pt, pt_element<1> *child_pt, + uintptr_t base_virt) +{ + // Level 1 (PD): each entry spans 2 MB. + const uintptr_t step = (uintptr_t)page_size * pte_per_page; // 2 MB + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<1> ppte = parent_pt[i]; + if (ppte.empty()) { child_pt[i] = make_empty_pte<1>(); continue; } + if (ppte.large()) { child_pt[i] = ppte; continue; } // 2MB: share as-is + void *child_sub = memory::alloc_page(); + memset(child_sub, 0, page_size); + auto parent_sub = phys_cast>(ppte.next_pt_addr()); + clone_pt_level0(parent_sub, static_cast*>(child_sub), + base_virt + (uintptr_t)i * step); + pt_element<1> cpte = ppte; + cpte.set_addr(virt_to_phys(child_sub), false); + child_pt[i] = cpte; + } +} + +template<> +void clone_pt_level<2>(pt_element<2> *parent_pt, pt_element<2> *child_pt, + uintptr_t base_virt) +{ + // Level 2 (PDPT): each entry spans 1 GB. + const uintptr_t step = (uintptr_t)page_size * pte_per_page * pte_per_page; // 1 GB + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<2> ppte = parent_pt[i]; + if (ppte.empty()) { child_pt[i] = make_empty_pte<2>(); continue; } + if (ppte.large()) { child_pt[i] = ppte; continue; } + void *child_sub = memory::alloc_page(); + memset(child_sub, 0, page_size); + auto parent_sub = phys_cast>(ppte.next_pt_addr()); + clone_pt_level<1>(parent_sub, static_cast*>(child_sub), + base_virt + (uintptr_t)i * step); + pt_element<2> cpte = ppte; + cpte.set_addr(virt_to_phys(child_sub), false); + child_pt[i] = cpte; + } +} + +address_space *clone_address_space(address_space *parent) +{ + // The parent's actual PML4 page (array of 512 level-3 entries). + phys parent_pml4_phys = parent->top ? parent->top->next_pt_addr() + : kernel_pt_root_phys(); + auto parent_pml4 = phys_cast>(parent_pml4_phys); + + // Allocate the child's PML4 page. + void *child_pml4_page = memory::alloc_page(); + memset(child_pml4_page, 0, page_size); + auto child_pml4 = static_cast*>(child_pml4_page); + + // Share every kernel PML4 slot (0 and 128..511) by copying the parent's + // entry verbatim (points at the same lower-level tables). Clone only the + // application slots (1..127) so their PTEs can diverge under COW. + PREVENT_STACK_PAGE_FAULT + WITH_LOCK(parent->vmas_mutex->for_write()) { + // Build the "share, don't COW" ranges: stack VMAs (OSv runs kernel code + // on the app stack, so its pages must stay writable -- see the note on + // clone_pt_level0) and MAP_SHARED VMAs (sharing must be preserved). + std::vector share_ranges; + for (auto &v : *parent->vmas) { + if (v.size() == 0) continue; + if ((v.flags() & mmap_stack) || (v.flags() & mmap_shared)) { + share_ranges.push_back({v.start(), v.end()}); + } + } + // Always share the forking thread's live stack: OSv runs kernel code + // (incl. the context switch, with irqs off) on it, so it must never be + // write-protected. get_stack_info() gives the current thread's stack. + { + auto si = sched::thread::current()->get_stack_info(); + uintptr_t s = reinterpret_cast(si.begin); + share_ranges.push_back({s, s + si.size}); + } + // Likewise every OTHER live thread's stack: those threads keep running + // in the PARENT address space and perform context switches (irqs off) on + // their own stacks, which OSv forbids faulting on. COW-protecting them + // would fault the next switch. The forked child is single-threaded and + // never touches sibling stacks, so sharing them is safe. + sched::with_all_threads([&share_ranges](sched::thread &t) { + auto si = t.get_stack_info(); + if (si.begin) { + uintptr_t s = reinterpret_cast(si.begin); + share_ranges.push_back({s, s + si.size}); + } + }); + cow_share_ranges = &share_ranges; + + for (unsigned slot = 0; slot < pte_per_page; slot++) { + if (slot >= pml4_app_first && slot <= pml4_app_last) { + pt_element<3> pslot = parent_pml4[slot]; + if (pslot.empty() || pslot.large()) { + // Empty, or (unexpected) large: share as-is / leave empty. + child_pml4[slot] = pslot.empty() ? make_empty_pte<3>() : pslot; + continue; + } + // Deep-copy this app subtree (level 2 downwards) into the + // child, marking private 4K leaves COW. + void *child_sub = memory::alloc_page(); + memset(child_sub, 0, page_size); + auto parent_sub = phys_cast>(pslot.next_pt_addr()); + clone_pt_level<2>(parent_sub, static_cast*>(child_sub), + (uintptr_t)slot << 39); + pt_element<3> cslot = pslot; + cslot.set_addr(virt_to_phys(child_sub), false); + child_pml4[slot] = cslot; + } else { + // Kernel slot: share verbatim. + child_pml4[slot] = parent_pml4[slot]; + } + } + cow_share_ranges = nullptr; + + // Any write-protection we applied to the parent's page tables above + // must be made visible on all CPUs before the parent continues. + mmu::flush_tlb_all(); + + // Create the child AS with its private PML4 page and vma_list. + auto child = new address_space(virt_to_phys(child_pml4_page)); + + // Structurally clone the parent's VMAs into the child's vma_list so + // the fault path can resolve child faults (permissions, backing). The + // physical pages are already wired via the cloned page tables above. + for (auto &v : *parent->vmas) { + // Skip the edge marker VMAs (size 0); the child's vma_list_type + // constructor already inserted its own edge markers. + if (v.size() == 0) { + continue; + } + auto *nv = new anon_vma(addr_range(v.start(), v.end()), v.perm(), v.flags()); + child->vmas->insert(*nv); + } + return child; + } +} + +// --- fork COW page-table teardown ------------------------------------------ +// +// Recursively free a child's private app-range subtree. Intermediate tables +// are always freed (they were allocated fresh for the child). Leaf 4K pages +// are freed only if they are the child's PRIVATE copy (writable, non-COW); +// COW / read-only leaves are shared with the parent and left intact. +static void free_child_pt_level0(pt_element<0> *pt) +{ + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<0> e = pt[i]; + if (e.empty()) continue; + if (e.writable() && !pte_is_cow(e) && !e.sw_bit(pte_shared)) { + // Child's own COW-copied private page: free it. (pte_shared pages + // are jointly-owned stack/MAP_SHARED frames -- never free them.) + memory::free_page(phys_to_virt(e.addr())); + } + // else: shared (COW, read-only, or pte_shared) -- leave the frame. + } + memory::free_page(pt); +} + +template +static void free_child_pt_level(pt_element *pt); + +template<> +void free_child_pt_level<1>(pt_element<1> *pt) +{ + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<1> e = pt[i]; + if (e.empty() || e.large()) continue; // large: shared mapping, skip + free_child_pt_level0(phys_cast>(e.next_pt_addr())); + } + memory::free_page(pt); +} +template<> +void free_child_pt_level<2>(pt_element<2> *pt) +{ + for (unsigned i = 0; i < pte_per_page; i++) { + pt_element<2> e = pt[i]; + if (e.empty() || e.large()) continue; + free_child_pt_level<1>(phys_cast>(e.next_pt_addr())); + } + memory::free_page(pt); +} + +void destroy_address_space(address_space *as) +{ + if (!as || as == &kernel_as) { + return; + } + // Free the child's private page tables for the application slots (1..127) + // and the PML4 page. Kernel slots (0, 128..511) are shared and must NOT + // be freed. For leaf 4K pages: free only the child's PRIVATE copies + // (writable, non-COW) -- COW/read-only leaves are still shared with the + // parent and must be left alone. + phys pml4_phys = as->top ? as->top->next_pt_addr() : 0; + if (pml4_phys) { + auto pml4 = phys_cast>(pml4_phys); + for (unsigned slot = pml4_app_first; slot <= pml4_app_last; slot++) { + pt_element<3> e3 = pml4[slot]; + if (e3.empty() || e3.large()) continue; + free_child_pt_level<2>(phys_cast>(e3.next_pt_addr())); + } + memory::free_page(phys_to_virt(pml4_phys)); + } + delete as; +} +#endif // CONF_fork + // A mutex serializing modifications to the high part of the page table // (linear map, etc.) which are not part of vma_list. mutex page_table_high_mutex; @@ -937,8 +1317,8 @@ class addr_compare { // Find the single (if any) vma which contains the given address. // The complexity is logarithmic in the number of vmas in vma_list. static inline vma_list_type::iterator -find_intersecting_vma(uintptr_t addr) { - auto vma = vma_list.lower_bound(addr, addr_compare()); +find_intersecting_vma_in(vma_list_type &vmas, uintptr_t addr) { + auto vma = vmas.lower_bound(addr, addr_compare()); if (vma->start() == addr) { return vma; } @@ -947,10 +1327,15 @@ find_intersecting_vma(uintptr_t addr) { if (addr >= vma->start() && addr < vma->end()) { return vma; } else { - return vma_list.end(); + return vmas.end(); } } +static inline vma_list_type::iterator +find_intersecting_vma(uintptr_t addr) { + return find_intersecting_vma_in(vma_list, addr); +} + // Find the list of vmas which intersect a given address range. Because the // vmas are sorted in vma_list, the result is a consecutive slice of vma_list, // [first, second), between the first returned iterator (inclusive), and the @@ -1481,6 +1866,55 @@ static void vm_sigbus(uintptr_t addr, exception_frame* ef) osv::handle_mmap_fault(addr, SIGBUS, ef); } +// --- fork COW write-fault resolver ---------------------------------------- +#if CONF_fork +// +// Walk the CURRENT address space's page table to the leaf PTE for `addr`. If +// it is a copy-on-write page (write-protected + cow bit, set by +// clone_address_space), allocate a fresh page, copy the shared page's contents +// into it, and install it writable in this AS only -- so the faulting side +// (parent or child) gets its own private copy. Returns true if it handled a +// COW fault. +static bool handle_cow_write_fault(uintptr_t addr) +{ + // current_pt_root() returns the synthetic top (level-4) entry; follow it to + // the PML4 page (level-3 entries) and walk down to the 4K leaf (level 0). + pt_element<4> *top = current_pt_root(); + if (top->empty()) return false; + auto pml4 = phys_cast>(top->next_pt_addr()); + unsigned i3 = pt_index(reinterpret_cast(addr), 3); // PML4 index + pt_element<3> e3 = pml4[i3]; + if (e3.empty() || e3.large()) return false; + auto pdpt = phys_cast>(e3.next_pt_addr()); + unsigned i2 = pt_index(reinterpret_cast(addr), 2); + pt_element<2> e2 = pdpt[i2]; + if (e2.empty() || e2.large()) return false; + auto pd = phys_cast>(e2.next_pt_addr()); + unsigned i1 = pt_index(reinterpret_cast(addr), 1); + pt_element<1> e1 = pd[i1]; + if (e1.empty() || e1.large()) return false; // 2 MB pages are not COW here + auto pt = phys_cast>(e1.next_pt_addr()); + unsigned i0 = pt_index(reinterpret_cast(addr), 0); + pt_element<0> e0 = pt[i0]; + if (e0.empty()) return false; + if (!pte_is_cow(e0)) return false; + + // Copy the shared page into a fresh private page. + void *shared = phys_to_virt(e0.addr()); + void *priv = memory::alloc_page(); + memcpy(priv, shared, page_size); + + // Install the private page writable, clearing the cow bit, in THIS AS only. + pt_element<0> npte = make_leaf_pte(hw_ptep<0>::force(&pt[i0]), + virt_to_phys(priv), perm_rwx); + npte = pte_mark_cow(npte, false); + npte.set_writable(true); + pt[i0] = npte; + mmu::flush_tlb_all(); + return true; +} +#endif // CONF_fork + void vm_fault(uintptr_t addr, exception_frame* ef) { trace_mmu_vm_fault(addr, ef->get_error()); @@ -1498,6 +1932,41 @@ void vm_fault(uintptr_t addr, exception_frame* ef) } #endif addr = align_down(addr, mmu::page_size); + +#if CONF_fork + // Resolve against the CURRENT thread's address space (child AS after fork, + // else AS0). A COW write fault copies the page privately for this side. + address_space *as = kernel_address_space(); + { + auto t = sched::thread::current(); + if (t && t->address_space()) { + as = t->address_space(); + } + } + + // First handle a copy-on-write write fault (fork private mappings): the + // page is present but write-protected with the cow bit -- copy it. + if (mmu::is_page_fault_write(ef->get_error())) { + PREVENT_STACK_PAGE_FAULT + WITH_LOCK(as->vmas_mutex->for_write()) { + if (handle_cow_write_fault(addr)) { + trace_mmu_vm_fault_ret(addr, ef->get_error()); + return; + } + } + } + + WITH_LOCK(as->vmas_mutex->for_read()) { + auto vma = find_intersecting_vma_in(*as->vmas, addr); + if (vma == as->vmas->end() || access_fault(*vma, ef->get_error())) { + vm_sigsegv(addr, ef); + trace_mmu_vm_fault_sigsegv(addr, ef->get_error(), "slow"); + return; + } + vma->fault(addr, ef); + } + trace_mmu_vm_fault_ret(addr, ef->get_error()); +#else // !CONF_fork -- original single-address-space fault path WITH_LOCK(vma_list_mutex.for_read()) { auto vma = find_intersecting_vma(addr); if (vma == vma_list.end() || access_fault(*vma, ef->get_error())) { @@ -1508,6 +1977,7 @@ void vm_fault(uintptr_t addr, exception_frame* ef) vma->fault(addr, ef); } trace_mmu_vm_fault_ret(addr, ef->get_error()); +#endif // CONF_fork } vma::vma(addr_range range, unsigned perm, unsigned flags, bool map_dirty, page_allocator *page_ops) diff --git a/core/sched.cc b/core/sched.cc index 07801a2918..896352f33b 100644 --- a/core/sched.cc +++ b/core/sched.cc @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -1087,6 +1088,12 @@ thread::thread(std::function func, attr attr, bool main, bool app) , _joiner(nullptr) { trace_thread_create(this); +#if CONF_fork + // Inherit the creating thread's address space (AS0 for the very first + // threads). fork() reassigns the child thread to its private child AS. + _current_as = sched::s_current ? sched::s_current->_current_as + : mmu::kernel_address_space(); +#endif if (!main && sched::s_current) { auto app = application::get_current().get(); diff --git a/include/osv/mmu-defs.hh b/include/osv/mmu-defs.hh index 6e19803e35..4ac0b292d2 100644 --- a/include/osv/mmu-defs.hh +++ b/include/osv/mmu-defs.hh @@ -96,6 +96,10 @@ enum { enum { pte_cow = 0, + // Software PTE bit tagging a page as SHARED across a fork (stack / + // MAP_SHARED / shm): such a page's physical frame is owned jointly, so a + // child address space must NOT free it on teardown. + pte_shared = 1, }; /* flush tlb for the current processor */ diff --git a/include/osv/mmu.hh b/include/osv/mmu.hh index 8ae509f352..8eb803e203 100644 --- a/include/osv/mmu.hh +++ b/include/osv/mmu.hh @@ -22,6 +22,7 @@ #include #include #include +#include struct exception_frame; #if CONF_memory_jvm_balloon @@ -372,6 +373,40 @@ error advise(void* addr, size_t size, int advice); void vm_fault(uintptr_t addr, exception_frame* ef); +#if CONF_fork +// ----------------------------------------------------------------------------- +// Per-process address space (Stage 2 fork COW). +// +// An address_space bundles a page-table root (the PML4 whose physical address +// is loaded into CR3) with its own set of VMAs. Historically OSv had a single +// global address space shared by all threads; that global is now "address +// space 0" (the kernel + the initial application), returned by +// kernel_address_space(). fork() creates a child address_space whose kernel +// half (PML4 entries for OSv text/data + the identity/phys maps) is shared +// with AS0, while the application half is a COW clone of the parent's. +// +// The concrete type is defined in core/mmu.cc (it owns the internal vma_list +// container); here it is opaque. Each thread carries a pointer to its current +// address_space; the arch context switch loads its page-table root. +struct address_space; + +// The kernel / initial-application address space ("AS0"). Always valid. +address_space *kernel_address_space(); + +// Physical address of an address_space's page-table root (value for CR3). +phys pt_root_phys(address_space *as); + +// Create a child address_space that COW-clones "parent" for fork(). Private +// writable VMAs are write-protected in both parent and child and marked COW; +// MAP_SHARED / shm VMAs are mapped to the same physical pages (truly shared). +// The kernel half of the page table is shared with the parent. +address_space *clone_address_space(address_space *parent); + +// Tear down a child address_space (frees its private page tables + VMAs). +// Never call on the kernel address space. +void destroy_address_space(address_space *as); +#endif // CONF_fork + std::string procfs_maps(); std::string sysfs_linear_maps(); diff --git a/include/osv/sched.hh b/include/osv/sched.hh index cfa2c15387..551c771a0d 100644 --- a/include/osv/sched.hh +++ b/include/osv/sched.hh @@ -29,6 +29,7 @@ #include #include #include +#include #include typedef float runtime_t; @@ -68,6 +69,10 @@ namespace elf { struct tls_data; } +namespace mmu { + struct address_space; +} + namespace osv { class application; @@ -836,8 +841,19 @@ private: std::vector _tls; bool _app; std::shared_ptr _app_runtime; +#if CONF_fork + // Current address space (Stage 2 fork COW). The arch context switch loads + // this AS's page-table root (CR3) when it differs from the outgoing + // thread's. Defaults to the kernel address space (AS0); a forked child's + // thread is moved into its private child address space. + mmu::address_space *_current_as; +#endif public: void destroy(); +#if CONF_fork + mmu::address_space *address_space() const { return _current_as; } + void set_address_space(mmu::address_space *as) { _current_as = as; } +#endif #ifdef __x86_64__ unsigned long get_app_tcb() { return _tcb->app_tcb; } void set_app_tcb(unsigned long tcb) { _tcb->app_tcb = tcb; } diff --git a/include/osv/wait_record.hh b/include/osv/wait_record.hh index d75c507b28..9955a19921 100644 --- a/include/osv/wait_record.hh +++ b/include/osv/wait_record.hh @@ -9,6 +9,10 @@ #define INCLUDED_OSV_WAIT_RECORD #include +#include +#if CONF_fork +#include +#endif // A "waiter" is simple synchronization object, with which one thread calling // waiter->wait() goes to sleep, and a second thread, which finds this waiter @@ -92,4 +96,53 @@ struct wait_record : public waiter { void wake_lock(mutex* mtx) { t.load(std::memory_order_relaxed)->wake_lock(mtx, this); } }; +#if CONF_fork +// fork() Stage 2 kernel-stack coherence. +// +// OSv has no separate kernel stack: kernel code runs on the app thread stack. +// A wait_record queued on a SHARED kernel mutex/condvar is a LOCAL on the +// waiter's stack. With per-child COW address spaces, a fork child's stack VA +// maps to a DIFFERENT physical page than the parent sees at that same VA, so a +// parent (running in AS0) that dereferences the child's queued wait_record +// through its own page tables reads the wrong physical page -> owner-assert / +// corruption in mutex::unlock(). +// +// Fix (Option A): when the current thread runs in a non-AS0 (fork-child) +// address space, allocate the wait_record from the KERNEL HEAP, which is +// identity-mapped identically in every address space, so its VA is coherent +// cross-AS. AS0 (default OSv + the parent) keeps the on-stack fast path with +// zero overhead. Returns true iff the current thread is in a forked child AS. +bool fork_child_needs_heap_wait_record(); + +// RAII holder giving a wait_record& that lives on the caller's stack for AS0 +// threads (fast path, no allocation) or on the kernel heap for fork-child +// threads (AS-coherent). Frees the heap copy on destruction; the stack copy +// is destroyed by the placement-new'd object's explicit dtor. +class coherent_wait_record { + alignas(wait_record) char _stack[sizeof(wait_record)]; + wait_record *_wr; + bool _heap; +public: + explicit coherent_wait_record(sched::thread *t) { + _heap = fork_child_needs_heap_wait_record(); + // wait_record is CACHELINE_ALIGNED (over-aligned); aligned_new honors + // it, and the stack buffer is alignas(wait_record). + _wr = _heap ? aligned_new(t) + : new (_stack) wait_record(t); + } + ~coherent_wait_record() { + if (_heap) { + _wr->~wait_record(); + free(_wr); + } else { + _wr->~wait_record(); + } + } + wait_record &get() { return *_wr; } + wait_record *ptr() { return _wr; } + coherent_wait_record(const coherent_wait_record &) = delete; + coherent_wait_record &operator=(const coherent_wait_record &) = delete; +}; +#endif // CONF_fork + #endif /* INCLUDED_OSV_WAIT_RECORD */ diff --git a/libc/process/execve.cc b/libc/process/execve.cc index d3626e2374..7546b93b81 100644 --- a/libc/process/execve.cc +++ b/libc/process/execve.cc @@ -26,6 +26,8 @@ int execve(const char *path, char *const argv[], char *const envp[]) #include #include "../libc.hh" #include +#include +#include // execve() on OSv. // @@ -73,6 +75,21 @@ int execve(const char *path, char *const argv[], char *const envp[]) } osv::shared_app_t child; + // execve() replaces the address space entirely: the forked child's COW + // private memory is discarded. Move this thread back to the kernel (global) + // address space BEFORE launching the new program, so the new program's + // mappings go into the global vma_list + page table (consistent with + // get_root_pt) rather than the now-defunct child COW address space. The + // freshly created app threads inherit this thread's (now global) AS. + { + auto self = sched::thread::current(); + auto old_as = self->address_space(); + if (old_as != mmu::kernel_address_space()) { + self->set_address_space(mmu::kernel_address_space()); + mmu::switch_to_runtime_page_tables(); // reload CR3 = global root + mmu::destroy_address_space(old_as); + } + } try { // new_program=true => fresh ELF namespace, so the exec'd program gets // its own globals rather than colliding with the caller's. diff --git a/libc/process/fork.cc b/libc/process/fork.cc index e59e078d5d..1c87455439 100644 --- a/libc/process/fork.cc +++ b/libc/process/fork.cc @@ -6,17 +6,20 @@ */ #include +#include #include #include #include #include #include #include +#include #include #include #include #include "../libc.hh" + // Arch hook (arch//fork.cc): create a child thread that resumes in // fork()'s CALLER (at caller_ret, with caller_sp) on a private copy of the // parent's user stack, returning 0 in the child. Hands back the copied stack @@ -30,7 +33,6 @@ extern sched::thread *fork_thread(void *caller_ret, void *caller_sp, extern "C" void __osv_run_atfork_prepare(); extern "C" void __osv_run_atfork_parent(); extern "C" void __osv_run_atfork_child(); - namespace osv { namespace fork { @@ -180,25 +182,40 @@ pid_t fork(void) } pid_t cpid = child->id(); + // Stage 2 fork: give the child its OWN address space, a COW clone of the + // parent's. Private writable mappings become copy-on-write in both parent + // and child; the kernel half of the page table is shared. The child + // thread runs in this address space (its CR3 is loaded on context switch). + mmu::address_space *child_as = + mmu::clone_address_space(sched::thread::current()->address_space()); + child->set_address_space(child_as); + // Register the child BEFORE starting it so a fast child->exit cannot race // ahead of the parent's bookkeeping. fork::register_child(cpid, parent); // Single cleanup, run by the thread reaper once the (detached) child has - // fully terminated: free the copied user stack, record a default status if + // fully terminated: free the copied user stack; record a default status if // the child fell off the end without exit() (real codes are recorded by - // exit()/execve() via fork::child_exited() before this runs), and finally - // dispose the child thread object itself. Disposing is essential: the - // child thread holds a shared_ptr to its application_runtime (set at thread - // construction), and only destroying the thread releases that reference. - // Without it the top-level application's runtime never drops to zero, its + // exit()/execve() via fork::child_exited() before this runs); destroy the + // child's copy-on-write address space; and dispose the child thread object. + // Disposing is essential: the child thread holds a shared_ptr to its + // application_runtime (set at thread construction), and only destroying the + // thread releases it; without it the app runtime never drops to zero, // ~application_runtime never fires, application::join() blocks forever on // _terminated, and OSv hangs at shutdown instead of powering off. - child->set_cleanup([cpid, stack_to_free, child] { + child->set_cleanup([cpid, stack_to_free, child, child_as] { fork::child_exited(cpid, 0); if (stack_to_free) { free(stack_to_free); } + // Only destroy the child address space if the child still owns it. + // execve() replaces the child AS with the global (kernel) AS and + // destroys child_as itself; destroying it again here is a double-free + // (GP fault / ~rwlock assert when the reaper tears down freed vmas). + if (child->address_space() == child_as) { + mmu::destroy_address_space(child_as); + } sched::thread::dispose(child); }); diff --git a/modules/tests/Makefile b/modules/tests/Makefile index a3e4aad504..2290d47a72 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -131,6 +131,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ tst-eventfd.so tst-remove.so misc-wake.so tst-epoll.so misc-lfring.so \ tst-fork.so tst-execve.so payload-exit7.so \ + tst-fork-cow.so tst-fork-deep.so \ misc-fsx.so tst-sleep.so tst-resolve.so tst-except.so \ misc-tcp-sendonly.so tst-tcp-nbwrite.so misc-tcp-hash-srv.so \ misc-loadbalance.so misc-scheduler.so tst-console.so tst-app.so \ diff --git a/tests/tst-fork-cow.cc b/tests/tst-fork-cow.cc new file mode 100644 index 0000000000..33bb01a4e7 --- /dev/null +++ b/tests/tst-fork-cow.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * Stage 2 fork() proof: per-child address space with copy-on-write. + * + * After fork() the child must get a logically INDEPENDENT copy of every + * PRIVATE mapping (this test uses a writable global in .data), while MAP_SHARED + * mappings stay truly SHARED between parent and child. We verify: + * 1. the child writing a distinct value to a private global does NOT change + * the parent's copy (COW isolation), and + * 2. the child writing to a MAP_SHARED region IS visible in the parent + * (sharing preserved). + * This is exactly the contract stock multi-process PostgreSQL relies on + * (private backend heap/globals, shared shared_buffers). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static int failures = 0; +#define CHECK(cond, msg) do { \ + if (!(cond)) { printf("FAIL: %s\n", msg); failures++; } \ + else { printf("PASS: %s\n", msg); } \ +} while (0) + +// A writable global lives in the program's PRIVATE .data mapping. Under COW +// fork the child gets its own copy; the parent's value must survive the child +// mutating it. +static volatile int private_global = 0x1111; + +// The MAP_SHARED region pointer, kept in a global so accessing it does not +// depend on the (biased) stack in the current fork-thread model. +static volatile int *shared_ptr; + +int main() +{ + printf("=== tst-fork-cow ===\n"); + + // A MAP_SHARED anonymous region: parent and child map the SAME physical + // pages, so writes are mutually visible (not COW). + volatile int *shared = (volatile int*)mmap(nullptr, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + CHECK(shared != MAP_FAILED, "mmap MAP_SHARED|MAP_ANONYMOUS region"); + if (shared == (volatile int*)MAP_FAILED) { + printf("=== tst-fork-cow done: %d failures ===\n", failures); + return 1; + } + shared_ptr = shared; + shared[0] = 0xAAAA; // parent's initial value in the shared region + + pid_t pid = fork(); + if (pid == 0) { + // CHILD: mutate BOTH the private global and the shared region. Use the + // global shared_ptr (not a stack local) to avoid any stack-copy bias. + private_global = 0x2222; // must stay private to the child + shared_ptr[0] = 0xBBBB; // must be visible to the parent + // Read back locally to be sure the writes landed in the child. + _exit((private_global == 0x2222 && shared_ptr[0] == 0xBBBB) ? 0 : 1); + } + + CHECK(pid > 0, "fork() returned child pid to parent"); + int status = 0; + pid_t w = waitpid(pid, &status, 0); + CHECK(w == pid, "waitpid() reaped the fork child"); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "child observed its own private + shared writes"); + + // THE PROOF: + // (1) COW isolation: the parent's private global is UNCHANGED even though + // the child set it to 0x2222. + CHECK(private_global == 0x1111, + "COW: parent's private global unchanged by child's write"); + // (2) Sharing preserved: the child's write to the MAP_SHARED region IS seen + // by the parent. + CHECK(shared[0] == 0xBBBB, + "SHARED: child's write to MAP_SHARED region visible in parent"); + + munmap((void*)shared, 4096); + printf("=== tst-fork-cow done: %d failures ===\n", failures); + return failures == 0 ? 0 : 1; +} diff --git a/tests/tst-fork-deep.cc b/tests/tst-fork-deep.cc new file mode 100644 index 0000000000..5ef20d5fde --- /dev/null +++ b/tests/tst-fork-deep.cc @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * Deep-call-chain fork() test. fork() is called from several nested function + * calls deep; the child must return ALL the way back up the call chain (popping + * each frame correctly) and _exit(N). The parent waitpid()s and checks N. + * + * This exercises the stack-fidelity of fork(): saved return addresses AND saved + * frame pointers (rbp) and any stack-internal pointers must remain valid in the + * child as it unwinds. A fork() that relocates the child stack to a different + * virtual address and biases the SP leaves saved rbp/&local pointing at the + * PARENT stack (off by the bias), which corrupts deep unwinds -- shallow + * callers (tst-fork) happen to survive, deep ones do not. This test is the one + * that catches that; it passes only when the child keeps the parent's exact + * stack addresses (same-VA COW stack). + */ + +#include +#include +#include +#include +#include + +static int failures = 0; +#define CHECK(cond, msg) do { \ + if (!(cond)) { printf("FAIL: %s\n", msg); failures++; } \ + else { printf("PASS: %s\n", msg); } \ +} while (0) + +// A chain of nested calls. Each frame has a local the compiler may address +// relative to rbp, and passes a running sum by pointer (a stack-internal +// pointer) to the next frame -- both must survive the fork in the child. +static pid_t g_child; + +// Returns the depth-accumulated value the child computed as it unwound, or the +// child pid in the parent. `depth` counts down; `acc` sums the depths. +static long deep(int depth, long acc, int *checkptr) +{ + volatile long marker = 0xC0DE0000L + depth; // rbp-relative local + int here = depth; // &here is a stack pointer + if (depth == 0) { + // Bottom of the chain: fork here, deep in the call stack. + pid_t pid = fork(); + if (pid == 0) { + // CHILD: must now return up through all `deep` frames. If any + // saved rbp/return-addr/&local is off, this unwind corrupts. + // We return acc and let the frames add their markers back. + return acc; // child returns acc up the chain + } + g_child = pid; + return -1; // parent sentinel + } + long r = deep(depth - 1, acc + depth, checkptr); + // On the way back up, verify our own frame survived: marker and here must + // still hold what we set before the recursive call. + if (marker != (0xC0DE0000L + depth) || here != depth) { + *checkptr = 1; // frame corruption detected + } + if (r < 0) return -1; // propagate parent sentinel + return r + depth; // child accumulates on the way up +} + +int main() +{ + printf("=== tst-fork-deep ===\n"); + const int N = 12; // 12 frames deep + // Expected child return: acc built going down = sum(1..N); then on the way + // up each frame adds its depth again => acc + sum(1..N) = 2*sum(1..N). + // acc going down = N + (N-1) + ... + 1 = N*(N+1)/2. Up adds the same. + long expected = (long)N * (N + 1); // 2 * N*(N+1)/2 + int corrupt = 0; + + long r = deep(N, 0, &corrupt); + if (r >= 0) { + // We are the CHILD (it returned a non-negative accumulated value and + // unwound all the way here). Exit with a status encoding success. + int ok = (r == expected && corrupt == 0); + _exit(ok ? 77 : 66); + } + + // PARENT: r < 0 sentinel. g_child holds the child pid. + CHECK(g_child > 0, "fork() deep in the call chain returned child pid"); + int status = 0; + pid_t w = waitpid(g_child, &status, 0); + CHECK(w == g_child, "waitpid() reaped the deep-fork child"); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 77, + "child unwound the deep call chain correctly (frames intact)"); + if (WIFEXITED(status) && WEXITSTATUS(status) == 66) { + printf("NOTE: child unwound but computed the wrong value / detected " + "frame corruption -> stack-bias bug present.\n"); + } + + printf("=== tst-fork-deep done: %d failures ===\n", failures); + return failures == 0 ? 0 : 1; +} From cf3d77d8dca05049f255fad014e4ce2b30830c7a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Wed, 22 Jul 2026 22:13:55 +0000 Subject: [PATCH 4/6] mm/fork: same-VA COW stack for deep call chains (opt-in) Give a forked child its parent's EXACT user stack -- same virtual addresses, private physical pages -- instead of a relocated copy with a biased SP. A relocated+biased stack leaves every saved rbp, return address and &local in the copied frames pointing at the parent's stack (off by the bias), so a deep unwind in the child dereferences the parent and faults. tst-fork-deep (fork 12 frames deep, child unwinds all the way up and _exit()s a checked value) is the test that catches this; it now passes. Gated behind CONF_fork (default n); conf_fork=0 is unchanged and builds/boots clean (fork code fully excluded, tst-mmap passes). Four coupled changes make same-VA correct, layered on the Stage 2 COW address space and the fork-child lifecycle fix (detached child + dispose(child) in the reaper): 1. core/mmu.cc: clone_address_space() PRIVATIZES the forking thread's live stack VA range into the child AS -- fresh private pages that byte-copy the parent's stack, mapped writable (not COW: OSv runs kernel code with irqs off on the app stack, a COW fault there is illegal) at the SAME VA. The parent's PTEs are untouched. 2. arch/x64/fork.cc + libc/process/fork.cc + include/osv/fork.hh: the child resumes with the caller's FULL callee-saved register context (rbx, rbp, r12-r15, rsp, rip), captured at fork() entry into osv::fork_resume_ctx and restored by the child trampoline off a scratch base register (so the restore never clobbers its own base). fork() jmps past its own epilogue, so without this the caller would resume with fork()'s internal register values -- a deep chain that keeps an accumulator in a callee-saved reg across fork() computes garbage. 3. arch/x64/arch-switch.hh: defer the fork CR3 switch to EXACTLY the rsp/rbp swap inside switch_to()'s asm. Loading the incoming AS's CR3 early, while still executing on the outgoing thread's stack (the fpu-cw/mxcsr scratch saves), resolved those shared-VA stack writes through the wrong address space and clobbered a blocked thread's live stack (root-caused via hardware watchpoint: switch_to wrote garbage into a blocked parent's on-stack user_mutex slot in AS0). 4. core/lfmutex.cc + core/mmu.cc: a wait_record on a shared kernel condvar/mutex must live in the identity-mapped kernel heap (coherent in every AS) whenever a cross-address-space wake is possible. Extend the Option-A rule from 'current thread is a fork child' to also cover 'any child address space is live' (live_child_address_spaces counter), so the parent in AS0 -- woken by a child that walks the queue through its own page tables -- is safe too. libc/process/execve.cc: on a fork child, move the thread onto its own kernel stack before switching to AS0 and running the target, so reloading CR3 to AS0 cannot alias the same-VA app stack onto the parent's physical page (which would corrupt a blocked parent). The child address space is destroyed HERE, on the kernel stack, exactly once: the reap-time cleanup (libc/process/fork.cc) then sees the thread's AS is AS0 (!= child_as) and skips its own destroy, so destroy_address_space() runs once whether the child exits normally (reaper destroys) or execs (execve destroys) -- no leak, no double-free. A pre-exec existence check keeps a failed exec returning to the caller. Validated on x86-64 (KVM): tst-fork 10/10, tst-fork-cow 6/6, tst-fork-deep 0 failures, tst-execve 3/3 with clean shutdown (no hang, no double-free at reap), on smp=1 and smp=4, 5/5 repeat runs; conf_fork=0 builds and boots clean (fork excluded, tst-mmap passes). --- arch/aarch64/fork.cc | 3 +- arch/x64/arch-switch.hh | 52 +++++++++++++++--- arch/x64/fork.cc | 117 ++++++++++++++++++++++------------------ core/lfmutex.cc | 22 ++++++-- core/mmu.cc | 58 +++++++++++++++++++- include/osv/fork.hh | 17 ++++++ libc/process/execve.cc | 112 ++++++++++++++++++++++++++++++-------- libc/process/fork.cc | 31 ++++++++--- 8 files changed, 318 insertions(+), 94 deletions(-) diff --git a/arch/aarch64/fork.cc b/arch/aarch64/fork.cc index 798e8ce85a..1f76e9e3d3 100644 --- a/arch/aarch64/fork.cc +++ b/arch/aarch64/fork.cc @@ -22,8 +22,9 @@ extern "C" void __osv_run_atfork_child(); sched::thread *fork_thread(void *caller_ret, void *caller_sp, - void **out_stack_to_free) + void *resume_ctx, void **out_stack_to_free) { + (void)resume_ctx; // aarch64 uses caller_sp bias-copy (same-VA is x86-64) auto parent = sched::thread::current(); auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr; diff --git a/arch/x64/arch-switch.hh b/arch/x64/arch-switch.hh index 637accb551..31f95648c5 100644 --- a/arch/x64/arch-switch.hh +++ b/arch/x64/arch-switch.hh @@ -95,14 +95,24 @@ void thread::switch_to() set_fsbase(reinterpret_cast(_tcb)); barrier(); #if CONF_fork - // Address-space (CR3) switch for fork COW. Only touch CR3 when the target + // Address-space (CR3) switch for fork COW. Only switch when the target // thread lives in a different address space than the outgoing one, so the - // common single-address-space case pays nothing (and no TLB flush). The - // kernel half of every AS is identically mapped, so the switch code, kernel - // stacks and kernel heap remain valid across the write. - if (_current_as != old->_current_as) { - processor::write_cr3(mmu::pt_root_phys(_current_as)); - } + // common single-address-space case pays nothing (and no TLB flush). + // + // CRITICAL (same-VA fork stacks): the CR3 write must happen EXACTLY at the + // rsp/rbp swap below, NOT here. OSv runs kernel code (this switch, the + // fpu-cw/mxcsr scratch saves, the register-save asm) on the app thread's + // stack. A forked child resumes on the parent's EXACT stack VAs, so parent + // and child alias the same stack virtual addresses (backed by different + // physical pages per address space). If we loaded the incoming thread's + // CR3 here, the subsequent scratch writes to the OUTGOING thread's stack + // (fnstcw/stmxcsr at -0x34(%rbp), etc., with rsp/rbp still the old thread's) + // would resolve through the WRONG address space and clobber the incoming + // thread's physical page at that shared VA -- corrupting a blocked thread's + // live stack. So we defer the CR3 load into the asm, coincident with the + // rsp/rbp swap: old-stack writes use the old AS, new-stack writes the new. + bool switch_as = (_current_as != old->_current_as); + u64 new_cr3 = switch_as ? mmu::pt_root_phys(_current_as) : 0; #endif auto c = _detached_state->_cpu; old->_state.exception_stack = c->arch.get_exception_stack(); @@ -119,6 +129,34 @@ void thread::switch_to() c->arch._current_thread_kernel_tcb = reinterpret_cast(_tcb); auto fpucw = processor::fnstcw(); auto mxcsr = processor::stmxcsr(); +#if CONF_fork + if (switch_as) { + // Same-VA-safe switch: swap rsp/rbp to the incoming thread, THEN load + // its CR3 (mov %rax,%cr3), THEN jump. All old-stack writes above ran + // under the old AS; every access after the rsp/rbp swap is to the new + // thread's stack, now resolved through the new AS -- so a shared stack + // VA never resolves through the wrong address space. Kernel .text and + // the thread_state (heap) are identically mapped in every AS, so the + // instruction fetch across the mov-to-cr3 and the %c[rip] load are + // valid before and after the CR3 write. + asm volatile + ("mov %%rbp, %c[rbp](%0) \n\t" + "movq $1f, %c[rip](%0) \n\t" + "mov %%rsp, %c[rsp](%0) \n\t" + "mov %c[rsp](%1), %%rsp \n\t" + "mov %c[rbp](%1), %%rbp \n\t" + "mov %2, %%cr3 \n\t" + "jmpq *%c[rip](%1) \n\t" + "1: \n\t" + : + : "a"(&old->_state), "c"(&this->_state), "d"(new_cr3), + [rsp]"i"(offsetof(thread_state, rsp)), + [rbp]"i"(offsetof(thread_state, rbp)), + [rip]"i"(offsetof(thread_state, rip)) + : "rbx", "rsi", "rdi", "r8", "r9", + "r10", "r11", "r12", "r13", "r14", "r15", "memory"); + } else +#endif asm volatile ("mov %%rbp, %c[rbp](%0) \n\t" "movq $1f, %c[rip](%0) \n\t" diff --git a/arch/x64/fork.cc b/arch/x64/fork.cc index 9c422c3c32..32c8ede28b 100644 --- a/arch/x64/fork.cc +++ b/arch/x64/fork.cc @@ -4,17 +4,26 @@ * This work is open source software, licensed under the terms of the * BSD license as described in the LICENSE file in the top-level directory. * - * fork_thread(): create a child thread that resumes in fork()'s CALLER, on a - * private copy of the parent's user stack, returning 0 from fork() in the child. - * The x86-64 arch half of the fork() emulation (see documentation/fork.md). + * fork_thread(): create a child thread that resumes in fork()'s CALLER on the + * SAME user stack virtual addresses the parent had -- no stack relocation, no + * SP bias -- and with the caller's FULL callee-saved register context restored, + * exactly as a normal `ret` from fork() would leave it. The child's address + * space (built by clone_address_space) maps the parent's live stack VA range to + * freshly-allocated PRIVATE physical pages that byte-copy the parent's stack, + * so the child sees identical stack contents at identical addresses but writes + * to its own private frames. * - * fork() (libc/process/fork.cc) passes us the caller's resume point: - * caller_ret = the address fork() would return to (__builtin_return_address) - * caller_sp = the parent's SP at fork()'s return (fork()'s frame base) - * We copy the parent stack region [caller_sp .. stack_base) into a fresh stack, - * bias caller_sp into the copy, and start a child thread whose trampoline sets - * rsp=child_sp, rax=0, and jumps to caller_ret -- i.e. the child returns from - * fork() with value 0 on its own private stack, in the caller. + * Why same-VA + full register restore: OSv/x86-64 code addresses saved frame + * pointers (rbp), return addresses, and &local pointers as absolute stack VAs, + * and keeps live values in callee-saved registers (rbx, r12-r15) across calls. + * Relocating the child stack to a different VA and biasing rsp leaves every + * saved rbp/&local pointing at the PARENT's stack (off by the bias), and simply + * jmp-ing to the return address skips fork()'s epilogue so the caller resumes + * with fork()'s internal register values. Either corrupts a deep unwind. + * Keeping the child on the parent's exact VAs (private phys) AND restoring the + * full caller register context (osv::fork_resume_ctx, captured at fork() entry) + * makes the child continue in fork()'s caller as if fork() had returned 0 -- the + * only correct way to fork a deep stack (see tst-fork-deep, documentation/fork.md). */ #include "arch.hh" @@ -23,13 +32,16 @@ #include #include #include +#include // pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the // child's context before it resumes user code. extern "C" void __osv_run_atfork_child(); -sched::thread *fork_thread(void *caller_ret, void *caller_sp, void **out_stack_to_free) +sched::thread *fork_thread(void *caller_ret, void *caller_sp, + void *resume_ctx, void **out_stack_to_free) { + auto ctx = static_cast(resume_ctx); auto parent = sched::thread::current(); auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr; @@ -39,57 +51,56 @@ sched::thread *fork_thread(void *caller_ret, void *caller_sp, void **out_stack_t if (sp < static_cast(si.begin) || sp > stack_base) { return nullptr; // caller SP not within the known user stack } - size_t stack_size = si.size; - char *child_stack_mem = static_cast(malloc(stack_size)); - if (!child_stack_mem) { - return nullptr; - } - // Copy ONLY the live top of the stack, [caller_sp .. stack_base), into the - // TOP of the child buffer. App (pthread) stacks are demand-paged: only the - // used top is mapped, so copying from si.begin would fault on the first - // unmapped page. Keeping the copy at the top of the child buffer preserves - // the base-relative bias so a biased SP resolves correctly. - char *child_base = child_stack_mem + stack_size; - ptrdiff_t bias = child_base - stack_base; - size_t live = static_cast(stack_base - sp); - memcpy(child_base - live, sp, live); - char *child_sp = sp + bias; - - volatile u64 resume_sp = reinterpret_cast(child_sp); - volatile u64 resume_pc = reinterpret_cast(caller_ret); - char *stack_to_free = child_stack_mem; + // Same-VA stack: the child resumes on the parent's EXACT register context. + // No copy and no bias here -- clone_address_space() privatizes the parent's + // live stack VA range into the child's address space (fresh private pages + // that byte-copy the parent's stack), so these very addresses are valid and + // private in the child. Capture the resume context by value in the lambda + // (the parent's on-stack ctx is gone by the time the child runs). + (void)caller_ret; + osv::fork_resume_ctx rc = *ctx; // TLS handling. The child is a real OSv sched::thread, so its constructor - // already ran setup_tcb() and installed a FRESH, private OSv TLS block - // (with its own errno and all libc __thread state). Two cases: - // - // (1) The app uses OSv's libc TLS (the normal dynamically-linked path, - // app_tcb == 0): the child's own fresh TCB is exactly right -- do NOT - // touch fsbase, let the child run on its private per-thread TLS. This - // is the clean case and fork() "just works" for TLS. - // (2) The app installed its own TCB via arch_prctl(SET_FS) (app_tcb != 0, - // e.g. a glibc binary's __libc_setup_tls): the child would need a - // private COPY of that app TCB. We do not duplicate it here yet; - // the child inherits the parent's app_tcb (shared), which is the - // documented multi-process-glibc limitation. A musl app built against - // OSv's libc takes path (1) and avoids this entirely. + // already ran setup_tcb() and installed a FRESH, private OSv TLS block. If + // the parent installed its own app TCB via arch_prctl (app_tcb != 0), the + // child inherits it (shared) -- the documented multi-process-glibc limit; + // otherwise the child keeps its own private OSv per-thread TLS. u64 parent_app_tcb = parent->get_app_tcb(); - auto t = sched::thread::make([resume_sp, resume_pc, parent_app_tcb] { - // Only override the child's own (fresh) TLS if the parent had installed - // an app TCB via arch_prctl; otherwise keep the child's private OSv TCB. + auto t = sched::thread::make([rc, parent_app_tcb] { if (parent_app_tcb) { arch::set_fsbase(parent_app_tcb); } // Run pthread_atfork child handlers in the child's context (e.g. reset // the malloc arena lock) before resuming user code. __osv_run_atfork_child(); + // Restore the caller's full callee-saved register context and resume in + // fork()'s caller with return value 0, on the parent's exact stack VAs + // (private phys in the child AS). We base all loads off a scratch + // register (rax) pointing at a LOCAL copy of the context, and restore + // the base-conflicting registers (rbx, rbp) LAST, so the load sequence + // never clobbers its own base pointer. + osv::fork_resume_ctx c = rc; // local copy the asm can address stably + // %0 = &c pinned in a register that is NOT one of the restored regs + // (we let the compiler pick, but we consume it only to seed rax). We + // move &c into rax first, then load every callee-saved reg from rax- + // relative offsets, and load rsp last -- rax (the base) is never in the + // restore set, so the sequence never clobbers its own base pointer. + // Offsets match struct fork_resume_ctx { rbx,rbp,r12,r13,r14,r15,rsp,rip }. asm volatile - ("movq %0, %%rsp \n\t" // install the private copied stack - "xorq %%rax, %%rax \n\t" // fork() returns 0 in the child - "jmpq *%1 \n\t" // resume in fork()'s caller - : : "r"(resume_sp), "r"(resume_pc) : "memory"); + ("movq %0, %%rax \n\t" // rax = &c (base; not restored) + "movq 0(%%rax), %%rbx \n\t" // rbx + "movq 8(%%rax), %%rbp \n\t" // rbp + "movq 16(%%rax), %%r12 \n\t" // r12 + "movq 24(%%rax), %%r13 \n\t" // r13 + "movq 32(%%rax), %%r14 \n\t" // r14 + "movq 40(%%rax), %%r15 \n\t" // r15 + "movq 56(%%rax), %%rcx \n\t" // rcx = caller rip (scratch) + "movq 48(%%rax), %%rsp \n\t" // adopt parent's exact rsp + "xorq %%rax, %%rax \n\t" // fork() returns 0 in the child + "jmpq *%%rcx \n\t" // resume in fork()'s caller + : : "r"(&c) : "rax", "rcx", "memory"); }, sched::thread::attr(). stack(4096 * 4). // Detached: nobody join()s the fork child (the parent reaps it via the @@ -106,10 +117,10 @@ sched::thread *fork_thread(void *caller_ret, void *caller_sp, void **out_stack_t if (parent_pinned_cpu) { t->pin(parent_pinned_cpu); } - // The caller (fork.cc) owns the single cleanup; hand back the copied user - // stack so it can be freed when the child is reaped. + // Same-VA: no separate user-stack buffer to free (the child's stack pages + // are owned by its address space and freed on destroy_address_space()). if (out_stack_to_free) { - *out_stack_to_free = stack_to_free; + *out_stack_to_free = nullptr; } return t; } diff --git a/core/lfmutex.cc b/core/lfmutex.cc index fd128cf1e3..3540cd2adc 100644 --- a/core/lfmutex.cc +++ b/core/lfmutex.cc @@ -16,11 +16,27 @@ #endif #if CONF_fork -// See wait_record.hh: true iff the current thread runs in a forked-child -// (non-AS0) address space, in which case a wait_record queued on a shared -// kernel object must be heap-allocated to stay coherent cross-address-space. +namespace mmu { + // Live fork-child address-space count (defined in core/mmu.cc). When > 0, + // more than one address space exists and a stack-resident wait_record on a + // shared kernel object can be dereferenced cross-AS. + extern std::atomic live_child_address_spaces; +} + +// See wait_record.hh: with per-child same-VA private stacks, a wait_record +// queued on a SHARED kernel condvar/mutex must live in the identity-mapped +// kernel heap (same VA->phys in every address space) whenever a cross-address- +// space wake is possible. That is true when EITHER the current thread runs in +// a forked-child (non-AS0) address space (a child woken by the parent), OR any +// child address space is currently live (the parent, in AS0, woken by a child +// -- the child dereferences the parent's stack-resident wait_record through +// its own page tables and would read a stale private stack copy). Default OSv +// (no fork, single AS) always takes the zero-overhead on-stack fast path. bool fork_child_needs_heap_wait_record() { + if (mmu::live_child_address_spaces.load(std::memory_order_relaxed) > 0) { + return true; + } auto t = sched::thread::current(); return t && t->address_space() && t->address_space() != mmu::kernel_address_space(); diff --git a/core/mmu.cc b/core/mmu.cc index ba4238db9b..1ad4180f4e 100644 --- a/core/mmu.cc +++ b/core/mmu.cc @@ -222,6 +222,17 @@ pt_element<4> *current_pt_root() // --- fork COW page-table clone -------------------------------------------- // +// Number of live fork-child address spaces. A wait_record placed on a SHARED +// kernel condvar/mutex is a stack local; with per-child same-VA private stacks, +// that stack VA resolves to DIFFERENT physical pages in different address +// spaces, so a thread in one AS that walks the queue and dereferences another +// thread's stack-resident wait_record reads the wrong page. When any child AS +// is live, ANY thread's wait_record can be dereferenced cross-AS (the parent's +// by a child, a child's by the parent), so wait_records must come from the +// identity-mapped kernel heap (same VA->phys in every AS) -- see +// fork_child_needs_heap_wait_record() and include/osv/wait_record.hh. +std::atomic live_child_address_spaces{0}; + // Recursively clone the child's copy of an application-range subtree of the // parent's page table, level by level, marking most private 4K leaf pages // copy-on-write. @@ -237,6 +248,14 @@ pt_element<4> *current_pt_root() // These "share, don't COW" address ranges are passed in via cow_share_ranges. struct cow_range { uintptr_t start, end; }; static const std::vector *cow_share_ranges; +// Ranges (the FORKING thread's live stack) that must be PRIVATIZED into the +// child: fresh private physical pages that byte-copy the parent's page, mapped +// at the SAME VA in the child, writable and NOT COW (OSv runs kernel code with +// irqs off on the app stack, so a COW write-fault there is illegal) and NOT +// shared (so the child owns its stack and frees it on teardown). This is what +// lets the child resume on the parent's exact rsp/rbp (same-VA stack) while +// having private stack memory -- see arch/x64/fork.cc. +static const std::vector *cow_privatize_ranges; static bool addr_is_shared(uintptr_t va) { @@ -247,6 +266,15 @@ static bool addr_is_shared(uintptr_t va) return false; } +static bool addr_is_privatize(uintptr_t va) +{ + if (!cow_privatize_ranges) return false; + for (auto &r : *cow_privatize_ranges) { + if (va >= r.start && va < r.end) return true; + } + return false; +} + // The parent must hold vma_list_mutex for write while this runs. base_virt is // the virtual address that leaf entry 0 of this PT maps. static void clone_pt_level0(pt_element<0> *parent_pt, pt_element<0> *child_pt, @@ -259,7 +287,23 @@ static void clone_pt_level0(pt_element<0> *parent_pt, pt_element<0> *child_pt, continue; } uintptr_t va = base_virt + (uintptr_t)i * page_size; - if (addr_is_shared(va)) { + if (addr_is_privatize(va)) { + // Forking thread's live stack: give the child its OWN physical + // page (byte-copy of the parent's), mapped at the SAME VA, plain + // writable (no COW bit, no shared bit). The parent's PTE is left + // untouched -- only the child diverges -- so the parent keeps + // writing its own stack with irqs off and never faults, and the + // child owns/frees this page on address-space teardown. + void *child_page = memory::alloc_page(); + memcpy(child_page, phys_to_virt(ppte.addr()), page_size); + pt_element<0> pv = ppte; + if (pte_is_cow(pv)) { + pv = pte_mark_cow(pv, false); + } + pv.set_writable(true); + pv.set_addr(virt_to_phys(child_page), false); + child_pt[i] = pv; + } else if (addr_is_shared(va)) { // Stack / MAP_SHARED page: must stay genuinely shared + writable in // both parent and child (never COW). Clear any COW bit and grant // write so both sides see each other's writes to the same phys @@ -366,10 +410,16 @@ address_space *clone_address_space(address_space *parent) // Always share the forking thread's live stack: OSv runs kernel code // (incl. the context switch, with irqs off) on it, so it must never be // write-protected. get_stack_info() gives the current thread's stack. + std::vector privatize_ranges; { + // The FORKING thread's live stack is PRIVATIZED (not shared): the + // child gets private byte-copies at the same VA so it can resume on + // the parent's exact rsp/rbp with independent stack memory. It + // must stay plain-writable (no COW) since OSv runs kernel code with + // irqs off on it -- a COW fault there is illegal. auto si = sched::thread::current()->get_stack_info(); uintptr_t s = reinterpret_cast(si.begin); - share_ranges.push_back({s, s + si.size}); + privatize_ranges.push_back({s, s + si.size}); } // Likewise every OTHER live thread's stack: those threads keep running // in the PARENT address space and perform context switches (irqs off) on @@ -384,6 +434,7 @@ address_space *clone_address_space(address_space *parent) } }); cow_share_ranges = &share_ranges; + cow_privatize_ranges = &privatize_ranges; for (unsigned slot = 0; slot < pte_per_page; slot++) { if (slot >= pml4_app_first && slot <= pml4_app_last) { @@ -409,6 +460,8 @@ address_space *clone_address_space(address_space *parent) } } cow_share_ranges = nullptr; + cow_privatize_ranges = nullptr; + live_child_address_spaces.fetch_add(1, std::memory_order_relaxed); // Any write-protection we applied to the parent's page tables above // must be made visible on all CPUs before the parent continues. @@ -499,6 +552,7 @@ void destroy_address_space(address_space *as) memory::free_page(phys_to_virt(pml4_phys)); } delete as; + live_child_address_spaces.fetch_sub(1, std::memory_order_relaxed); } #endif // CONF_fork diff --git a/include/osv/fork.hh b/include/osv/fork.hh index 474eae7812..f7ec0fc29d 100644 --- a/include/osv/fork.hh +++ b/include/osv/fork.hh @@ -66,4 +66,21 @@ bool exit_current_child(int status); extern "C" pid_t fork(void); extern "C" pid_t vfork(void); +#ifdef __x86_64__ +namespace osv { +// x86-64 machine context a forked child must resume with so it continues in +// fork()'s caller EXACTLY as a normal `ret` from fork() would -- restoring all +// callee-saved registers (rbx, rbp, r12-r15) the caller relies on across the +// call, plus rsp and the return address. fork() jmps past its own epilogue +// (the child is a different thread), so unless these are restored the caller +// resumes with fork()'s internal register values -> a deep call chain that +// keeps live values in callee-saved regs across fork() (e.g. an accumulator) +// computes garbage. Captured at fork() entry, where the regs still hold the +// caller's values; restored by the child trampoline (arch/x64/fork.cc). +struct fork_resume_ctx { + unsigned long rbx, rbp, r12, r13, r14, r15, rsp, rip; +}; +} // namespace osv +#endif + #endif // OSV_FORK_HH diff --git a/libc/process/execve.cc b/libc/process/execve.cc index 7546b93b81..abb6b8eb5d 100644 --- a/libc/process/execve.cc +++ b/libc/process/execve.cc @@ -47,6 +47,34 @@ int execve(const char *path, char *const argv[], char *const envp[]) // state is not torn down the way a real address-space replacement would; the // new program runs in its own ELF namespace but shares the kernel heap. +// Continuation of execve() that runs the exec'd program to completion. It is +// invoked either directly (normal case) or on a fresh stack after we have left +// the same-VA fork stack (fork-child case); either way it never returns -- a +// successful exec ends the thread with the program's exit code. +static void execve_run_and_exit(const std::string *path, + const std::vector *args, + const std::unordered_map *env, + bool have_env) +{ + osv::shared_app_t child; + try { + // new_program=true => fresh ELF namespace, so the exec'd program gets + // its own globals rather than colliding with the caller's. + child = osv::application::run(*path, *args, true, have_env ? env : nullptr); + } catch (const osv::launch_error &e) { + // Could not load/exec the target - Linux returns ENOENT/ENOEXEC/EACCES. + errno = ENOENT; + _exit(127); + } + // Record the exec'd app so the fork/wait layer can reap it under the pid of + // the thread that called execve() (Linux keeps the pid across exec). + osv::fork::adopt_execed_app(child); + // A successful execve() does not return to the caller: run the program to + // completion on this thread and exit with its return code. + int rc = child->join(); + _exit(rc); +} + extern "C" int execve(const char *path, char *const argv[], char *const envp[]) { @@ -74,22 +102,71 @@ int execve(const char *path, char *const argv[], char *const envp[]) } } - osv::shared_app_t child; - // execve() replaces the address space entirely: the forked child's COW - // private memory is discarded. Move this thread back to the kernel (global) - // address space BEFORE launching the new program, so the new program's - // mappings go into the global vma_list + page table (consistent with - // get_root_pt) rather than the now-defunct child COW address space. The - // freshly created app threads inherit this thread's (now global) AS. - { - auto self = sched::thread::current(); - auto old_as = self->address_space(); - if (old_as != mmu::kernel_address_space()) { - self->set_address_space(mmu::kernel_address_space()); - mmu::switch_to_runtime_page_tables(); // reload CR3 = global root - mmu::destroy_address_space(old_as); + // execve() replaces the caller's program: the exec'd target must run in the + // GLOBAL (AS0) address space, not the forked child's COW address space, so + // its ELF mappings and demand-paged I/O behave normally (a fault during + // block-I/O completion in a COW child AS is illegal -- assert(preemptable)). + // But with same-VA fork stacks, THIS thread is currently executing on its + // app stack whose pages are PRIVATE to the child AS at VAs it shares with + // the (possibly blocked) parent. Switching CR3 to AS0 while on that stack + // would alias the stack onto the parent's physical page and destroy_address + // _space() would free the running stack. So we first move this thread onto + // its own kernel stack (allocated in the global heap, valid in every AS), + // THEN switch to AS0, tear down the child AS, and run the target. + // + // Because that stack/AS switch is a point of no return (we cannot return -1 + // to the caller afterwards), first verify the target is loadable so a + // failed exec still returns to the caller (which typically _exit()s its own + // fallback status) instead of stranding the thread. + std::string spath(path); + auto self = sched::thread::current(); + auto old_as = self->address_space(); + if (old_as && old_as != mmu::kernel_address_space()) { + if (::access(path, F_OK) != 0) { + return libc_error(ENOENT); // caller decides the fallback } + auto si = self->get_stack_info(); // the thread's own kernel stack + void *kstack_top = static_cast(si.begin) + si.size; + // 16-byte align and leave a little headroom. + uintptr_t sp = (reinterpret_cast(kstack_top) & ~uintptr_t(15)) - 256; + // Package the continuation args so the new stack frame can reach them. + static thread_local std::string s_path; + static thread_local std::vector s_args; + static thread_local std::unordered_map s_env; + static thread_local bool s_have_env; + static thread_local mmu::address_space *s_old_as; + s_path = spath; s_args = args; s_env = env; s_have_env = (envp != nullptr); + s_old_as = old_as; + // Switch to AS0 and tear down the child AS -- but do it from the kernel + // stack, so freeing the child's app-stack pages cannot pull the rug. + // We accomplish the ordering with an rsp switch: move to the kernel + // stack, then call a lambda that switches CR3, destroys old_as, and + // runs the program. It never returns. + self->set_address_space(mmu::kernel_address_space()); + asm volatile( + "movq %0, %%rsp \n\t" + "movq %0, %%rbp \n\t" + "call *%1 \n\t" + : : "r"(sp), "r"(reinterpret_cast(+[]{ + // Now on the kernel stack (valid in every AS) with this + // thread reassigned to AS0; reload CR3 to AS0, then destroy + // the child AS HERE (exactly once): we have left the child's + // same-VA app stack, so freeing its page tables is safe. + // The reap-time cleanup (libc/process/fork.cc) sees the + // thread's AS is now AS0 (!= child_as) and skips its own + // destroy, so child_as is destroyed once whether the child + // exits normally (cleanup destroys) or execs (here). Never + // returns. + mmu::switch_to_runtime_page_tables(); + mmu::destroy_address_space(s_old_as); + execve_run_and_exit(&s_path, &s_args, &s_env, s_have_env); + })) + : "memory"); + __builtin_unreachable(); } + + // Not a fork child (or already in AS0): run the target directly. + osv::shared_app_t child; try { // new_program=true => fresh ELF namespace, so the exec'd program gets // its own globals rather than colliding with the caller's. @@ -99,14 +176,7 @@ int execve(const char *path, char *const argv[], char *const envp[]) // Could not load/exec the target - Linux returns ENOENT/ENOEXEC/EACCES. return libc_error(ENOENT); } - - // Record the exec'd app so the fork/wait layer can reap it under the pid - // of the thread that called execve() (Linux keeps the pid across exec). osv::fork::adopt_execed_app(child); - - // A successful execve() does not return to the caller. Join the child app - // (run it to completion on this thread) and then exit with its return code, - // so this thread never re-enters the old program. int rc = child->join(); _exit(rc); // not reached diff --git a/libc/process/fork.cc b/libc/process/fork.cc index 1c87455439..ae5d742961 100644 --- a/libc/process/fork.cc +++ b/libc/process/fork.cc @@ -25,7 +25,7 @@ // parent's user stack, returning 0 in the child. Hands back the copied stack // via out_stack_to_free so fork() can release it when the child is reaped. extern sched::thread *fork_thread(void *caller_ret, void *caller_sp, - void **out_stack_to_free); + void *resume_ctx, void **out_stack_to_free); // atfork handler chains (defined in libc/pthread.cc). glibc/musl register // these internally; fork() must run prepare() in the parent before forking, @@ -163,18 +163,35 @@ extern "C" __attribute__((noinline)) pid_t fork(void) { + // Capture the caller's full callee-saved register context FIRST, before + // fork()'s body clobbers rbx/r12-r15. At this point (right after fork's + // prologue) those registers still hold the caller's values, and fork's rbp + // frame gives us the caller's saved rbp ([rbp]), return address ([rbp+8]), + // and post-return rsp (rbp+16). The child resumes with exactly this + // context so it continues in fork()'s caller as a normal `ret` would -- + // essential for deep call chains (see osv::fork_resume_ctx, tst-fork-deep). + fork_resume_ctx ctx; + void **fp = static_cast(__builtin_frame_address(0)); + asm volatile("movq %%rbx, %0\n\t" + "movq %%r12, %1\n\t" + "movq %%r13, %2\n\t" + "movq %%r14, %3\n\t" + "movq %%r15, %4\n\t" + : "=m"(ctx.rbx), "=m"(ctx.r12), "=m"(ctx.r13), + "=m"(ctx.r14), "=m"(ctx.r15)); + ctx.rbp = reinterpret_cast(fp[0]); // caller's rbp + ctx.rip = reinterpret_cast(__builtin_return_address(0)); + ctx.rsp = reinterpret_cast(fp + 2); // fork rbp + 16 + pid_t parent = getpid(); - // Capture the point fork() will return to in its caller, and the caller's - // stack pointer (fork()'s own frame base == caller SP at the return). The - // child thread resumes exactly there, on a private copy of the stack. - void *caller_ret = __builtin_return_address(0); - void *caller_sp = __builtin_frame_address(0); + void *caller_ret = reinterpret_cast(ctx.rip); + void *caller_sp = reinterpret_cast(ctx.rsp); void *stack_to_free = nullptr; // POSIX: run pthread_atfork prepare handlers in the parent before forking. __osv_run_atfork_prepare(); - sched::thread *child = fork_thread(caller_ret, caller_sp, &stack_to_free); + sched::thread *child = fork_thread(caller_ret, caller_sp, &ctx, &stack_to_free); if (!child) { __osv_run_atfork_parent(); // undo prepare-side locking errno = ENOMEM; From 95a650c438d2eac7025892571f6eb56df05fce6a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 23 Jul 2026 03:48:29 +0000 Subject: [PATCH 5/6] mm/fork: copy-on-write 2 MB large pages on fork (opt-in) The per-child COW address space (clone_pt_level<1>) shared any 2 MB large page verbatim between parent and child instead of copy-on-writing it: if (ppte.large()) { child_pt[i] = ppte; continue; } // 2MB: share as-is so a forked child writing a private large mapping scribbled the parent(s) too. This hit large malloc (>= 2 MB, mmap-backed), large MAP_ANON regions, and any 2 MB-backed private writable mapping in the application slots. Fix: for a WRITABLE large PD entry, split it to 4 K in the parent in place (the existing split_large_page primitive) and fall through to the normal 4 K clone path, which already makes the correct per-page decision -- private writable -> COW, MAP_SHARED -> stays shared, read-only -> shared as-is. A read-only large page can never diverge, so it is still shared verbatim. This reuses the whole 4 K COW fault machinery (clone_pt_level0 + handle_cow_write_fault); the only cost is one 4 K page table per split large page, acceptable under the opt-in CONF_fork. Verified on x86-64 (KVM microvm): - a standalone musl-PIE fork test now isolates a private >2 MB malloc across fork (child write does not change the parent; parent write does not change the child), while a MAP_SHARED region stays mutually visible; - tst-fork 10/10, tst-fork-cow 6/6, tst-fork-deep and tst-execve all pass (no regressions); - conf_fork=0 builds clean with the whole fork path (this change included) compiled out (clone_address_space absent from loader.elf). Note: this fixes large-page (application-slot) COW. OSv's small-object malloc heap lives in the shared kernel identity map (mem_area::mempool), which fork() shares by design; isolating that is a separate change. All gated behind CONF_fork (default off). Copyright (C) 2026 Greg Burd --- core/mmu.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/mmu.cc b/core/mmu.cc index 1ad4180f4e..4bc36e7f8c 100644 --- a/core/mmu.cc +++ b/core/mmu.cc @@ -334,6 +334,8 @@ static void clone_pt_level0(pt_element<0> *parent_pt, pt_element<0> *child_pt, // clone_pt_level for N in 1..2, carrying base_virt so leaf pages can be // tested against the share-ranges. (gnu++14: no if constexpr, so explicit // specialization.) +template void split_large_page(hw_ptep ptep); // defined below +template<> void split_large_page(hw_ptep<1> ptep); // defined below template static void clone_pt_level(pt_element *parent_pt, pt_element *child_pt, uintptr_t base_virt); @@ -347,7 +349,19 @@ void clone_pt_level<1>(pt_element<1> *parent_pt, pt_element<1> *child_pt, for (unsigned i = 0; i < pte_per_page; i++) { pt_element<1> ppte = parent_pt[i]; if (ppte.empty()) { child_pt[i] = make_empty_pte<1>(); continue; } - if (ppte.large()) { child_pt[i] = ppte; continue; } // 2MB: share as-is + if (ppte.large()) { + // A read-only 2 MB page can never diverge, so share it verbatim. + // A WRITABLE 2 MB large page (large malloc, MAP_SHARED, large anon) + // must NOT be shared as-is -- that lets a forked child's writes + // leak into the parent. Split it to 4 K in the PARENT in place, + // then fall through to the normal 4 K clone path, which makes the + // correct per-page decision (private-writable -> COW, MAP_SHARED -> + // stays shared, read-only -> shared as-is). Reuses the whole 4 K + // COW machinery; cost is one 4 K page table per split large page. + if (!ppte.writable()) { child_pt[i] = ppte; continue; } + split_large_page(hw_ptep<1>::force(&parent_pt[i])); + ppte = parent_pt[i]; // re-read: now a non-large intermediate pte + } void *child_sub = memory::alloc_page(); memset(child_sub, 0, page_size); auto parent_sub = phys_cast>(ppte.next_pt_addr()); From e23e2829c1fbce6419af676f02e8ee861b832eb4 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 25 Jul 2026 04:49:42 -0400 Subject: [PATCH 6/6] mm: preserve file_vma type when cloning a child address space on fork clone_address_space() rebuilt EVERY child VMA as an anon_vma, so a fork child's file-backed mappings (file_vma -- e.g. the ELF loader's file-backed .text, mapped via map_file) became anonymous in the child. A demand fault on a child file-backed page that the parent had not already resident-loaded then dispatched to the base zero-fill path instead of file_vma::fault reading the file: the child read (or executed) zeros. This is the "wild-branch" bug that blocked file-backed fork workloads such as PostgreSQL, whose checkpointer child faults large cold .text regions the postmaster never touched. Preserve each parent VMA's dynamic type across the clone: - In the VMA-clone loop (under the parent's vmas_mutex), dynamic_cast each vma to file_vma. For a file_vma, rebuild the child's copy via the file's own mmap() -- the SAME call map_file and file_vma::split use -- so the child gets the correct page_allocator (map_file_page_read for MAP_PRIVATE, map_file_page_mmap for a cached/shared mapping) and its demand faults read the file. The new file_vma holds its own fileref. Anon vmas still clone as anon_vma. file::mmap()/default_file_mmap()/map_file_mmap() are pure allocation (no locks, no I/O), matching the anon_vma allocation this loop already performs under the lock. - In destroy_address_space(), dispose the child's owned vma objects before freeing the address space. ~file_vma releases the fileref this child took (and deletes its page_allocator), closing the file-reference leak the type preservation would otherwise introduce -- and also the pre-existing anon_vma + edge-marker leak. ~vma touches no global list, so disposing here is safe. The whole change is inside #if CONF_fork, so a conf_fork=0 build is byte-identical. Adds tests/tst-fork-file-mmap.cc (gated via the fork test group): a parent mmaps a read-only ROFS file MAP_PRIVATE without faulting a mid-file page, forks, and the child faults that page first -- it must read the real file bytes, not zeros. Reproduces the bug on the unfixed tree (child reads 0x00) and passes with this fix (child reads the real byte). Signed-off-by: Greg Burd --- core/mmu.cc | 35 +++++++++- modules/tests/Makefile | 2 +- tests/tst-fork-file-mmap.cc | 130 ++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 tests/tst-fork-file-mmap.cc diff --git a/core/mmu.cc b/core/mmu.cc index 4bc36e7f8c..fe68468d8b 100644 --- a/core/mmu.cc +++ b/core/mmu.cc @@ -487,13 +487,36 @@ address_space *clone_address_space(address_space *parent) // Structurally clone the parent's VMAs into the child's vma_list so // the fault path can resolve child faults (permissions, backing). The // physical pages are already wired via the cloned page tables above. + // + // Preserve each parent vma's DYNAMIC TYPE. A file_vma (e.g. the ELF + // loader's file-backed .text, created via map_file) must clone as a + // file_vma bound to the SAME file + offset -- otherwise the child's + // demand faults dispatch to the base zero-fill path and it executes + // zeros over its own .text (the fork COW-clone wild-branch bug). A + // file-backed vma is rebuilt via the file's own mmap() -- the SAME + // call map_file and file_vma::split use -- so the child gets the + // correct page_allocator (map_file_page_read for MAP_PRIVATE, + // map_file_page_mmap for a cached/shared mapping) and its demand + // faults read the file instead of zero-filling. The new file_vma + // holds its OWN fileref; ~file_vma in destroy_address_space releases + // it. file::mmap()/default_file_mmap()/map_file_mmap() are pure + // allocation (no locks, no I/O), so calling them here under + // vmas_mutex is safe -- the same as the anon_vma allocation this loop + // already does under the lock. for (auto &v : *parent->vmas) { // Skip the edge marker VMAs (size 0); the child's vma_list_type // constructor already inserted its own edge markers. if (v.size() == 0) { continue; } - auto *nv = new anon_vma(addr_range(v.start(), v.end()), v.perm(), v.flags()); + vma *nv; + auto *fv = dynamic_cast(&v); + if (fv) { + nv = fv->file()->mmap(addr_range(v.start(), v.end()), + v.flags(), v.perm(), fv->offset()).release(); + } else { + nv = new anon_vma(addr_range(v.start(), v.end()), v.perm(), v.flags()); + } child->vmas->insert(*nv); } return child; @@ -565,6 +588,16 @@ void destroy_address_space(address_space *as) } memory::free_page(phys_to_virt(pml4_phys)); } + // Dispose the child's own vma objects (the clones built in + // clone_address_space, plus the vma_list edge markers). ~vma is a no-op + // for the base/anon case; for a file_vma it deletes the page_allocator and + // releases the fileref this child took in clone_address_space -- so the + // file-backed clone does not leak its file reference. (This also closes + // the pre-existing anon_vma + edge-marker leak.) ~vma touches no global + // list, so disposing here is safe. + if (as->owned_vmas) { + as->owned_vmas->clear_and_dispose([](vma *v) { delete v; }); + } delete as; live_child_address_spaces.fetch_sub(1, std::memory_order_relaxed); } diff --git a/modules/tests/Makefile b/modules/tests/Makefile index 2290d47a72..9b6ee65a12 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -131,7 +131,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. misc-ctxsw.so tst-read.so tst-symlink.so tst-openat.so \ tst-eventfd.so tst-remove.so misc-wake.so tst-epoll.so misc-lfring.so \ tst-fork.so tst-execve.so payload-exit7.so \ - tst-fork-cow.so tst-fork-deep.so \ + tst-fork-cow.so tst-fork-deep.so tst-fork-file-mmap.so \ misc-fsx.so tst-sleep.so tst-resolve.so tst-except.so \ misc-tcp-sendonly.so tst-tcp-nbwrite.so misc-tcp-hash-srv.so \ misc-loadbalance.so misc-scheduler.so tst-console.so tst-app.so \ diff --git a/tests/tst-fork-file-mmap.cc b/tests/tst-fork-file-mmap.cc new file mode 100644 index 0000000000..6889c95126 --- /dev/null +++ b/tests/tst-fork-file-mmap.cc @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2026 Greg Burd + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + * + * REGRESSION TEST for [W-branch]: a fork child that demand-faults a page of a + * FILE-backed (MAP_PRIVATE) mapping which the PARENT never touched before the + * fork. Before the clone_address_space type-preservation fix this returned + * ZEROS (the bug): clone_address_space rebuilt EVERY child vma as an anon_vma, + * so a file_vma (e.g. PostgreSQL's file-backed .text, mapped by the ELF loader) + * became anonymous in the child and its demand fault hit the base zero-fill + * path instead of file_vma::fault reading the file. It now PASSES (the child + * reads the real file bytes). + * + * The mapping is of a READ-ONLY ROFS file -- matching the real case (an + * executable's file-backed .text, demand-paged from the read-only image), and + * avoiding the writable-fs access-time update that would otherwise write to the + * inode during a read fault. + * + * Root cause (proven via KVM+hbreak, see /tmp/w-branch-kvm.txt): + * core/mmu.cc clone_address_space(): + * struct vma_snap { start, end, perm, flags; }; // dropped dynamic type + * snap.push_back({v.start(), v.end(), v.perm(), v.flags()}); + * auto *nv = new anon_vma(...); // ALWAYS anon_vma + * The child's file-backed VA thus faulted to a fresh ZERO anon page. The + * existing tst-fork* tests only exercise ANON memory, so they miss it; PG is + * the first fork whose child demand-faults a file-backed .text page the + * parent never resident-loaded. + * + * Mechanism the test recreates: + * - parent opens a ROFS file and reads the EXPECTED bytes of a mid-file page + * via pread(2) on a SEPARATE fd (this never faults the mapping's PTE); + * - parent mmaps the file MAP_PRIVATE but does NOT read the target page (so + * no PTE for it exists in the parent -> the COW page-table clone hands the + * child an absent PTE); + * - fork(); + * - the CHILD reads the target page FIRST -> demand fault -> must read the + * real file bytes. On buggy HEAD it reads zeros (MISMATCH). + * + * Deterministic on -smp 1. Gated CONF_fork. + * + * To reproduce (in arena-dev, CONF_fork=y build): + * ./scripts/build conf_fork=1 fs=rofs image=tests, then: + * ./scripts/run.py -c 1 -e /tests/tst-fork-file-mmap.so + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main() +{ + printf("=== tst-fork-file-mmap ===\n"); + const size_t page = (size_t)sysconf(_SC_PAGESIZE); + + // A read-only ROFS file present on the test image (~47 KiB). + const char *path = "/rofs/mmap-file-test1"; + int fd = open(path, O_RDONLY); + if (fd < 0) { + // fall back to the /tmp copy the image also ships + fd = open("/tmp/mmap-file-test1", O_RDONLY); + } + if (fd < 0) { printf("FAIL: open %s (%s)\n", path, strerror(errno)); return 1; } + + struct stat st; + if (fstat(fd, &st) != 0) { printf("FAIL: fstat\n"); return 1; } + size_t size = (size_t)st.st_size; + if (size < 6 * page) { printf("FAIL: file too small (%zu)\n", size); return 1; } + + // Target page the child will fault: a mid-file page (well past page 0) that + // the parent never touches through the mapping. + const size_t target_pg = 5; + const off_t target_off = (off_t)(target_pg * page); + + // Read the EXPECTED bytes of that page via a SEPARATE fd + pread -- this + // does not fault the mapping's PTE, so the child still demand-faults it. + unsigned char *want = new unsigned char[page]; + { + int fd2 = open(path, O_RDONLY); + if (fd2 < 0) fd2 = open("/tmp/mmap-file-test1", O_RDONLY); + if (fd2 < 0) { printf("FAIL: open2\n"); return 1; } + ssize_t n = pread(fd2, want, page, target_off); + close(fd2); + if (n != (ssize_t)page) { printf("FAIL: pread expected page (%zd)\n", n); return 1; } + } + + // mmap the file MAP_PRIVATE, read-only. Do NOT touch the target page. + unsigned char *p = (unsigned char *)mmap(nullptr, size, PROT_READ, + MAP_PRIVATE, fd, 0); + if (p == MAP_FAILED) { printf("FAIL: mmap (%s)\n", strerror(errno)); return 1; } + close(fd); // mmap holds its own fileref + + pid_t pid = fork(); + if (pid == 0) { + // CHILD: first access to the target page -> demand fault. On buggy HEAD + // this reads a zero-filled anon page; with the fix it reads file bytes. + volatile unsigned char got0 = p[target_off]; + int bad = memcmp((const void *)(p + target_off), want, page); + printf("child: page %zu byte0=0x%02x want=0x%02x %s\n", + target_pg, (unsigned)got0, (unsigned)want[0], + bad ? "MISMATCH" : "OK"); + _exit(bad ? 70 : 55); + } + + int status = 0; + waitpid(pid, &status, 0); + munmap(p, size); + delete[] want; + + if (WIFEXITED(status) && WEXITSTATUS(status) == 55) { + printf("PASS: fork child reads real file bytes from an untouched " + "file-backed mmap page\n"); + return 0; + } + if (WIFEXITED(status) && WEXITSTATUS(status) == 70) { + printf("FAIL/BUG [W-branch]: child saw ZEROS/wrong data -- file_vma " + "cloned as anon_vma (status=0x%x)\n", status); + } else { + printf("FAIL: child aborted (status=0x%x)\n", status); + } + return 1; +}