mm: per-child copy-on-write address space for fork() (opt-in, stacked on #1455)

Makefile

@@ -1057,6 +1057,9 @@ objects += arch/$(arch)/firmware.o
 objects += arch/$(arch)/hypervisor.o
 objects += arch/$(arch)/interrupt.o
 objects += arch/$(arch)/clone.o
+ifeq ($(conf_fork),1)
+objects += arch/$(arch)/fork.o
+endif
 ifeq ($(conf_drivers_pci),1)
 objects += arch/$(arch)/pci.o
 objects += arch/$(arch)/msi.o
@@ -1660,11 +1663,20 @@ musl += prng/srand48.o
 libc += random.o
 
 libc += process/execve.o
+ifeq ($(conf_fork),1)
+libc += process/fork.o
+endif
 musl += process/execle.o
 musl += process/execv.o
 musl += process/execl.o
 libc += process/waitpid.o
+# When fork() is enabled, our waitpid.cc provides wait()/waitpid()/wait4()
+# backed by the fork() child registry, superseding musl's process/wait.o.
+# When fork() is disabled, waitpid.cc provides only the ECHILD waitpid() stub,
+# so keep musl's wait.o for wait()/wait4().
+ifneq ($(conf_fork),1)
 musl += process/wait.o
+endif
 
 musl += setjmp/$(musl_arch)/setjmp.o
 musl += setjmp/$(musl_arch)/longjmp.o

arch/aarch64/fork.cc

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2026 Greg Burd
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ *
+ * fork_thread() for aarch64 -- mirrors arch/x64/fork.cc.  fork() (fork.cc)
+ * passes the caller's return address and stack pointer; we copy the parent's
+ * user stack, bias the SP into the copy, and start a child thread that installs
+ * the copied stack, sets x0=0 (fork()'s return value in the child), and returns
+ * to fork()'s caller.
+ */
+
+#include "arch.hh"
+#include <errno.h>
+#include <string.h>
+#include <cstdlib>
+#include <osv/sched.hh>
+
+// pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the
+// child's context before it resumes user code.
+extern "C" void __osv_run_atfork_child();
+
+sched::thread *fork_thread(void *caller_ret, void *caller_sp,
+                           void *resume_ctx, void **out_stack_to_free)
+{
+    (void)resume_ctx;   // aarch64 uses caller_sp bias-copy (same-VA is x86-64)
+    auto parent = sched::thread::current();
+    auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr;
+
+    auto si = parent->get_stack_info();
+    char *stack_base = static_cast<char*>(si.begin) + si.size;
+    char *sp = static_cast<char*>(caller_sp);
+    if (sp < static_cast<char*>(si.begin) || sp > stack_base) {
+        return nullptr;
+    }
+    size_t stack_size = si.size;
+
+    char *child_stack_mem = static_cast<char*>(malloc(stack_size));
+    if (!child_stack_mem) {
+        return nullptr;
+    }
+    // Copy ONLY the live top [caller_sp .. stack_base) into the top of the
+    // child buffer (app stacks are demand-paged; copying from si.begin faults).
+    char *child_base = child_stack_mem + stack_size;
+    ptrdiff_t bias = child_base - stack_base;
+    size_t live = static_cast<size_t>(stack_base - sp);
+    memcpy(child_base - live, sp, live);
+    char *child_sp = sp + bias;
+
+    volatile u64 resume_sp = reinterpret_cast<u64>(child_sp);
+    volatile u64 resume_pc = reinterpret_cast<u64>(caller_ret);
+    char *stack_to_free = child_stack_mem;
+
+    // TLS: the child is a real OSv thread with its own fresh setup_tcb() block.
+    // Only override tpidr_el0 if the parent had installed its own app TCB via
+    // arch_prctl (parent_app_tcb != 0); otherwise keep the child's private OSv
+    // TLS (the clean case for a musl app built against OSv's libc).
+    u64 parent_app_tcb = parent->get_app_tcb();
+    auto t = sched::thread::make([resume_sp, resume_pc, parent_app_tcb] {
+        if (parent_app_tcb) {
+            asm volatile ("msr tpidr_el0, %0; isb" :: "r"(parent_app_tcb) : "memory");
+        }
+        // Run pthread_atfork child handlers in the child's context before
+        // resuming user code.
+        __osv_run_atfork_child();
+        asm volatile
+          ("mov sp, %0 \n\t"    // install the private copied stack
+           "mov x0, #0 \n\t"    // fork() returns 0 in the child
+           "br %1 \n\t"         // resume in fork()'s caller
+           : : "r"(resume_sp), "r"(resume_pc) : "x0", "memory");
+    }, sched::thread::attr().
+        stack(4096 * 4).
+        // Detached: nobody join()s the fork child (the parent reaps it via the
+        // pid registry / waitpid, not sched::thread::join).  A detached thread
+        // is handed to the reaper on completion, which runs our set_cleanup()
+        // (freeing the copied stack and disposing the thread object, releasing
+        // its application_runtime reference).  Without this the thread object
+        // (and its app_runtime shared_ptr) would leak and OSv would hang at
+        // shutdown -- see the cleanup comment in libc/process/fork.cc.
+        detached(),
+        false,
+        true);
+    t->set_app_tcb(parent->get_app_tcb());
+    if (parent_pinned_cpu) {
+        t->pin(parent_pinned_cpu);
+    }
+    if (out_stack_to_free) {
+        *out_stack_to_free = stack_to_free;
+    }
+    return t;
+}

arch/x64/arch-switch.hh

@@ -13,6 +13,7 @@
 #include <osv/kernel_config_preempt.h>
 #include <osv/kernel_config_threads_default_kernel_stack_size.h>
 #include <osv/kernel_config_syscall_stack_size.h>
+#include <osv/kernel_config_fork.h>
 #include <string.h>
 #include "tls-switch.hh"
 
@@ -93,6 +94,26 @@ void thread::switch_to()
     barrier();
     set_fsbase(reinterpret_cast<u64>(_tcb));
     barrier();
+#if CONF_fork
+    // Address-space (CR3) switch for fork COW.  Only switch when the target
+    // thread lives in a different address space than the outgoing one, so the
+    // common single-address-space case pays nothing (and no TLB flush).
+    //
+    // CRITICAL (same-VA fork stacks): the CR3 write must happen EXACTLY at the
+    // rsp/rbp swap below, NOT here.  OSv runs kernel code (this switch, the
+    // fpu-cw/mxcsr scratch saves, the register-save asm) on the app thread's
+    // stack.  A forked child resumes on the parent's EXACT stack VAs, so parent
+    // and child alias the same stack virtual addresses (backed by different
+    // physical pages per address space).  If we loaded the incoming thread's
+    // CR3 here, the subsequent scratch writes to the OUTGOING thread's stack
+    // (fnstcw/stmxcsr at -0x34(%rbp), etc., with rsp/rbp still the old thread's)
+    // would resolve through the WRONG address space and clobber the incoming
+    // thread's physical page at that shared VA -- corrupting a blocked thread's
+    // live stack.  So we defer the CR3 load into the asm, coincident with the
+    // rsp/rbp swap: old-stack writes use the old AS, new-stack writes the new.
+    bool switch_as = (_current_as != old->_current_as);
+    u64 new_cr3 = switch_as ? mmu::pt_root_phys(_current_as) : 0;
+#endif
     auto c = _detached_state->_cpu;
     old->_state.exception_stack = c->arch.get_exception_stack();
     // save the old thread SYSCALL caller stack pointer in the syscall stack descriptor
@@ -108,6 +129,34 @@ void thread::switch_to()
     c->arch._current_thread_kernel_tcb = reinterpret_cast<u64>(_tcb);
     auto fpucw = processor::fnstcw();
     auto mxcsr = processor::stmxcsr();
+#if CONF_fork
+    if (switch_as) {
+        // Same-VA-safe switch: swap rsp/rbp to the incoming thread, THEN load
+        // its CR3 (mov %rax,%cr3), THEN jump.  All old-stack writes above ran
+        // under the old AS; every access after the rsp/rbp swap is to the new
+        // thread's stack, now resolved through the new AS -- so a shared stack
+        // VA never resolves through the wrong address space.  Kernel .text and
+        // the thread_state (heap) are identically mapped in every AS, so the
+        // instruction fetch across the mov-to-cr3 and the %c[rip] load are
+        // valid before and after the CR3 write.
+        asm volatile
+            ("mov %%rbp, %c[rbp](%0) \n\t"
+             "movq $1f, %c[rip](%0) \n\t"
+             "mov %%rsp, %c[rsp](%0) \n\t"
+             "mov %c[rsp](%1), %%rsp \n\t"
+             "mov %c[rbp](%1), %%rbp \n\t"
+             "mov %2, %%cr3 \n\t"
+             "jmpq *%c[rip](%1) \n\t"
+             "1: \n\t"
+             :
+             : "a"(&old->_state), "c"(&this->_state), "d"(new_cr3),
+               [rsp]"i"(offsetof(thread_state, rsp)),
+               [rbp]"i"(offsetof(thread_state, rbp)),
+               [rip]"i"(offsetof(thread_state, rip))
+             : "rbx", "rsi", "rdi", "r8", "r9",
+               "r10", "r11", "r12", "r13", "r14", "r15", "memory");
+    } else
+#endif
     asm volatile
         ("mov %%rbp, %c[rbp](%0) \n\t"
          "movq $1f, %c[rip](%0) \n\t"

arch/x64/fork.cc

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2026 Greg Burd
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ *
+ * fork_thread(): create a child thread that resumes in fork()'s CALLER on the
+ * SAME user stack virtual addresses the parent had -- no stack relocation, no
+ * SP bias -- and with the caller's FULL callee-saved register context restored,
+ * exactly as a normal `ret` from fork() would leave it.  The child's address
+ * space (built by clone_address_space) maps the parent's live stack VA range to
+ * freshly-allocated PRIVATE physical pages that byte-copy the parent's stack,
+ * so the child sees identical stack contents at identical addresses but writes
+ * to its own private frames.
+ *
+ * Why same-VA + full register restore: OSv/x86-64 code addresses saved frame
+ * pointers (rbp), return addresses, and &local pointers as absolute stack VAs,
+ * and keeps live values in callee-saved registers (rbx, r12-r15) across calls.
+ * Relocating the child stack to a different VA and biasing rsp leaves every
+ * saved rbp/&local pointing at the PARENT's stack (off by the bias), and simply
+ * jmp-ing to the return address skips fork()'s epilogue so the caller resumes
+ * with fork()'s internal register values.  Either corrupts a deep unwind.
+ * Keeping the child on the parent's exact VAs (private phys) AND restoring the
+ * full caller register context (osv::fork_resume_ctx, captured at fork() entry)
+ * makes the child continue in fork()'s caller as if fork() had returned 0 -- the
+ * only correct way to fork a deep stack (see tst-fork-deep, documentation/fork.md).
+ */
+
+#include "arch.hh"
+#include "tls-switch.hh"
+#include <errno.h>
+#include <string.h>
+#include <cstdlib>
+#include <osv/sched.hh>
+#include <osv/fork.hh>
+
+// pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the
+// child's context before it resumes user code.
+extern "C" void __osv_run_atfork_child();
+
+sched::thread *fork_thread(void *caller_ret, void *caller_sp,
+                           void *resume_ctx, void **out_stack_to_free)
+{
+    auto ctx = static_cast<osv::fork_resume_ctx*>(resume_ctx);
+    auto parent = sched::thread::current();
+    auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr;
+
+    auto si = parent->get_stack_info();
+    char *stack_base = static_cast<char*>(si.begin) + si.size;
+    char *sp = static_cast<char*>(caller_sp);
+    if (sp < static_cast<char*>(si.begin) || sp > stack_base) {
+        return nullptr;   // caller SP not within the known user stack
+    }
+
+    // Same-VA stack: the child resumes on the parent's EXACT register context.
+    // No copy and no bias here -- clone_address_space() privatizes the parent's
+    // live stack VA range into the child's address space (fresh private pages
+    // that byte-copy the parent's stack), so these very addresses are valid and
+    // private in the child.  Capture the resume context by value in the lambda
+    // (the parent's on-stack ctx is gone by the time the child runs).
+    (void)caller_ret;
+    osv::fork_resume_ctx rc = *ctx;
+
+    // TLS handling.  The child is a real OSv sched::thread, so its constructor
+    // already ran setup_tcb() and installed a FRESH, private OSv TLS block.  If
+    // the parent installed its own app TCB via arch_prctl (app_tcb != 0), the
+    // child inherits it (shared) -- the documented multi-process-glibc limit;
+    // otherwise the child keeps its own private OSv per-thread TLS.
+    u64 parent_app_tcb = parent->get_app_tcb();
+
+    auto t = sched::thread::make([rc, parent_app_tcb] {
+        if (parent_app_tcb) {
+            arch::set_fsbase(parent_app_tcb);
+        }
+        // Run pthread_atfork child handlers in the child's context (e.g. reset
+        // the malloc arena lock) before resuming user code.
+        __osv_run_atfork_child();
+        // Restore the caller's full callee-saved register context and resume in
+        // fork()'s caller with return value 0, on the parent's exact stack VAs
+        // (private phys in the child AS).  We base all loads off a scratch
+        // register (rax) pointing at a LOCAL copy of the context, and restore
+        // the base-conflicting registers (rbx, rbp) LAST, so the load sequence
+        // never clobbers its own base pointer.
+        osv::fork_resume_ctx c = rc;   // local copy the asm can address stably
+        // %0 = &c pinned in a register that is NOT one of the restored regs
+        // (we let the compiler pick, but we consume it only to seed rax).  We
+        // move &c into rax first, then load every callee-saved reg from rax-
+        // relative offsets, and load rsp last -- rax (the base) is never in the
+        // restore set, so the sequence never clobbers its own base pointer.
+        // Offsets match struct fork_resume_ctx { rbx,rbp,r12,r13,r14,r15,rsp,rip }.
+        asm volatile
+          ("movq %0, %%rax        \n\t"  // rax = &c (base; not restored)
+           "movq  0(%%rax), %%rbx \n\t"  // rbx
+           "movq  8(%%rax), %%rbp \n\t"  // rbp
+           "movq 16(%%rax), %%r12 \n\t"  // r12
+           "movq 24(%%rax), %%r13 \n\t"  // r13
+           "movq 32(%%rax), %%r14 \n\t"  // r14
+           "movq 40(%%rax), %%r15 \n\t"  // r15
+           "movq 56(%%rax), %%rcx \n\t"  // rcx = caller rip (scratch)
+           "movq 48(%%rax), %%rsp \n\t"  // adopt parent's exact rsp
+           "xorq %%rax, %%rax     \n\t"  // fork() returns 0 in the child
+           "jmpq *%%rcx           \n\t"  // resume in fork()'s caller
+           : : "r"(&c) : "rax", "rcx", "memory");
+    }, sched::thread::attr().
+        stack(4096 * 4).
+        // Detached: nobody join()s the fork child (the parent reaps it via the
+        // pid registry / waitpid, not sched::thread::join).  A detached thread
+        // is handed to the reaper on completion, which runs our set_cleanup()
+        // (freeing the copied stack and disposing the thread object, releasing
+        // its application_runtime reference).  Without this the thread object
+        // (and its app_runtime shared_ptr) would leak and OSv would hang at
+        // shutdown -- see the cleanup comment in libc/process/fork.cc.
+        detached(),
+        false,
+        true);
+    t->set_app_tcb(parent->get_app_tcb());
+    if (parent_pinned_cpu) {
+        t->pin(parent_pinned_cpu);
+    }
+    // Same-VA: no separate user-stack buffer to free (the child's stack pages
+    // are owned by its address space and freed on destroy_address_space()).
+    if (out_stack_to_free) {
+        *out_stack_to_free = nullptr;
+    }
+    return t;
+}

arch/x64/mmu.cc

@@ -120,15 +120,40 @@ void flush_tlb_all()
 
 static pt_element<4> page_table_root __attribute__((init_priority((int)init_prio::pt_root)));
 
+#if CONF_fork
+// Virtual pointer to the current thread's PML4 (child AS's private PML4 when
+// running in a forked child, else the kernel PML4).  Defined in core/mmu.cc.
+pt_element<4> *current_pt_root();
+
 pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) {
+    return current_pt_root();
+}
+
+// Virtual pointer to the kernel (AS0) PML4.
+pt_element<4> *kernel_pml4() {
     return &page_table_root;
 }
+#else
+pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) {
+    return &page_table_root;
+}
+#endif // CONF_fork
 
 void switch_to_runtime_page_tables()
 {
     processor::write_cr3(page_table_root.next_pt_addr());
 }
 
+#if CONF_fork
+// Physical address of the kernel (AS0) PML4 -- the CR3 value that maps OSv
+// text/data + the identity/phys ranges.  Used as the shared base for cloned
+// child address spaces and as AS0's pt_root.
+phys kernel_pt_root_phys()
+{
+    return page_table_root.next_pt_addr();
+}
+#endif // CONF_fork
+
 enum {
     page_fault_prot  = 1ul << 0,
     page_fault_write = 1ul << 1,

conf/Makefile

@@ -56,15 +56,15 @@ CONF_FILES := conf/base.mk conf/$(mode).mk conf/$(arch).mk conf/profiles/$(arch)
 
 #Generate the default .config, gen/config/kernel.conf, and gen/include/osv/kernel_config.h if not generated yet
 $(out)/.config: $(CONF_FILES) $(out)/kbuild/kconfig/conf
-	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config)
+	$(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config)
 
 #Generate the .config.yes, gen/include/osv/kernel_yes_config.h with all available options
 $(out)/.config.yes: $(CONF_FILES) $(out)/kbuild/kconfig/conf
 	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_yes_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config_yes/kernel_yes.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config.yes $(out)/kbuild/kconfig/conf -s conf/kconfig/main --allyesconfig, CONF_YES $(out)/.config.yes)
 
 #Synchronize gen/include/osv/kernel_config.h and gen/config/kernel.conf with .config if the latter has changed maybe by mconf
 $(out)/gen/include/osv/kernel_config.h: $(out)/.config
-	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config)
+	$(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config)
 
 #Generate gen/config/kernel_conf.mk AND individual option headers gen/include/osv/kernel_config_* based on the latest gen/config/kernel.conf and gen/include/osv/kernel_config.h
 #The gen/config/kernel_conf.mk is included by the main OSv makefile and the headers gen/include/osv/kernel_config_* included by relevant source files

conf/kconfig/threads

@@ -6,6 +6,18 @@ config lazy_stack
   prompt "Use lazy stack"
   def_bool $(shell,grep -q ^conf_lazy_stack=1 conf/base.mk && echo y || echo n)
 
+config fork
+  prompt "Include fork()/vfork() support (per-child address space, off by default)"
+  def_bool $(shell,[ "$conf_fork" = 1 ] && echo y || echo n)
+  help
+    Enable OSv's thread-backed fork()/vfork()/execve()/waitpid() emulation,
+    including the per-child address space with copy-on-write needed to give a
+    forked child private memory.  This changes OSv's usual single-address-space
+    model (address-space switches on context switch between fork children), so
+    it is OFF by default: with this disabled, none of the fork code is compiled
+    in and OSv behaves exactly as before (fork() returns ENOSYS).  Enable it
+    only for workloads that require fork() (e.g. multi-process programs).
+
 config lazy_stack_invariant
   prompt "Check lazy stack invariant"
   def_bool $(shell,grep -q ^conf_lazy_stack_invariant=1 conf/base.mk && echo y || echo n)