mm: per-child copy-on-write address space for fork() (opt-in, stacked)

Follow-up to the base fork() PR: gives a forked child its own address space
with copy-on-write of private mappings, so a forked child has real memory
isolation like Linux fork() while MAP_SHARED / shm stays shared.  Also gated
behind CONFIG_fork (default off): with it disabled none of this is compiled and
OSv's single-address-space model and context-switch path are unchanged.

With CONFIG_fork enabled:
- mmu::address_space object (page-table root + vma_list); the previous global
  becomes 'address space 0' (kernel + init app).
- Per-thread current address space; the context switch loads the target CR3
  only when the address space differs (a no-op for AS0-only workloads).  Kernel
  PML4 entries are shared across all address spaces so OSv code + the kernel
  heap work identically after a switch.
- fork() clones the parent's vmas into a child address space: PRIVATE writable
  mappings are write-protected in both and copied on the first write (reusing
  OSv's existing COW fault machinery); MAP_SHARED / shm map the same physical
  pages (truly shared).  execve() returns the thread to AS0.
- Lock/condvar wait_records for fork-child (non-AS0) threads are allocated from
  the AS-shared kernel heap instead of the thread stack, so a wait_record queued
  on a shared kernel mutex resolves to the same physical page for any waker
  across address spaces (kernel-stack coherence).

Validated: tst-fork-cow proves a forked child's private memory stays private
while MAP_SHARED stays shared; tst-fork stays 10/10.

Known limitation (documented): the child's stack is still relocated+copied
rather than same-VA COW, so a child that unwinds a very deep call chain (e.g. a
multi-process server forking backends) can still hit a stack-fidelity issue
(tst-fork-deep); the same-VA stack fix is a further follow-up.  This PR delivers
memory-isolated fork for the common cases with COW proven.

Copyright (C) 2026 Greg Burd

Author: gburd

arch/x64/arch-switch.hh

@@ -13,6 +13,7 @@
 #include <osv/kernel_config_preempt.h>
 #include <osv/kernel_config_threads_default_kernel_stack_size.h>
 #include <osv/kernel_config_syscall_stack_size.h>
+#include <osv/kernel_config_fork.h>
 #include <string.h>
 #include "tls-switch.hh"
 
@@ -93,6 +94,16 @@ void thread::switch_to()
     barrier();
     set_fsbase(reinterpret_cast<u64>(_tcb));
     barrier();
+#if CONF_fork
+    // Address-space (CR3) switch for fork COW.  Only touch CR3 when the target
+    // thread lives in a different address space than the outgoing one, so the
+    // common single-address-space case pays nothing (and no TLB flush).  The
+    // kernel half of every AS is identically mapped, so the switch code, kernel
+    // stacks and kernel heap remain valid across the write.
+    if (_current_as != old->_current_as) {
+        processor::write_cr3(mmu::pt_root_phys(_current_as));
+    }
+#endif
     auto c = _detached_state->_cpu;
     old->_state.exception_stack = c->arch.get_exception_stack();
     // save the old thread SYSCALL caller stack pointer in the syscall stack descriptor

arch/x64/mmu.cc

@@ -120,15 +120,40 @@ void flush_tlb_all()
 
 static pt_element<4> page_table_root __attribute__((init_priority((int)init_prio::pt_root)));
 
+#if CONF_fork
+// Virtual pointer to the current thread's PML4 (child AS's private PML4 when
+// running in a forked child, else the kernel PML4).  Defined in core/mmu.cc.
+pt_element<4> *current_pt_root();
+
 pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) {
+    return current_pt_root();
+}
+
+// Virtual pointer to the kernel (AS0) PML4.
+pt_element<4> *kernel_pml4() {
     return &page_table_root;
 }
+#else
+pt_element<4> *get_root_pt(uintptr_t virt __attribute__((unused))) {
+    return &page_table_root;
+}
+#endif // CONF_fork
 
 void switch_to_runtime_page_tables()
 {
     processor::write_cr3(page_table_root.next_pt_addr());
 }
 
+#if CONF_fork
+// Physical address of the kernel (AS0) PML4 -- the CR3 value that maps OSv
+// text/data + the identity/phys ranges.  Used as the shared base for cloned
+// child address spaces and as AS0's pt_root.
+phys kernel_pt_root_phys()
+{
+    return page_table_root.next_pt_addr();
+}
+#endif // CONF_fork
+
 enum {
     page_fault_prot  = 1ul << 0,
     page_fault_write = 1ul << 1,

conf/Makefile

@@ -56,15 +56,15 @@ CONF_FILES := conf/base.mk conf/$(mode).mk conf/$(arch).mk conf/profiles/$(arch)
 
 #Generate the default .config, gen/config/kernel.conf, and gen/include/osv/kernel_config.h if not generated yet
 $(out)/.config: $(CONF_FILES) $(out)/kbuild/kconfig/conf
-	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config)
+	$(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --alldefconfig, CONF_DEF $(out)/.config)
 
 #Generate the .config.yes, gen/include/osv/kernel_yes_config.h with all available options
 $(out)/.config.yes: $(CONF_FILES) $(out)/kbuild/kconfig/conf
 	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_yes_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config_yes/kernel_yes.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config.yes $(out)/kbuild/kconfig/conf -s conf/kconfig/main --allyesconfig, CONF_YES $(out)/.config.yes)
 
 #Synchronize gen/include/osv/kernel_config.h and gen/config/kernel.conf with .config if the latter has changed maybe by mconf
 $(out)/gen/include/osv/kernel_config.h: $(out)/.config
-	$(call quiet, mode=$(mode) arch=$(arch) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config)
+	$(call quiet, mode=$(mode) arch=$(arch) conf_fork=$(conf_fork) CONFIG_=CONF_ KCONFIG_AUTOHEADER=$(out)/gen/include/osv/kernel_config.h KCONFIG_AUTOCONFIG=$(out)/gen/config/kernel.conf KCONFIG_RUSTCCFG=$(out)/gen/include/osv/rustc_cfg KCONFIG_CONFIG=$(out)/.config $(out)/kbuild/kconfig/conf -s conf/kconfig/main --syncconfig, SYNC $(out)/.config)
 
 #Generate gen/config/kernel_conf.mk AND individual option headers gen/include/osv/kernel_config_* based on the latest gen/config/kernel.conf and gen/include/osv/kernel_config.h
 #The gen/config/kernel_conf.mk is included by the main OSv makefile and the headers gen/include/osv/kernel_config_* included by relevant source files

conf/kconfig/threads

@@ -8,8 +8,7 @@ config lazy_stack
 
 config fork
   prompt "Include fork()/vfork() support (per-child address space, off by default)"
-  bool
-  default n
+  def_bool $(shell,[ "$conf_fork" = 1 ] && echo y || echo n)
   help
     Enable OSv's thread-backed fork()/vfork()/execve()/waitpid() emulation,
     including the per-child address space with copy-on-write needed to give a

core/condvar.cc

@@ -13,6 +13,7 @@
 #include <osv/export.h>
 #include <osv/kernel_config_lazy_stack.h>
 #include <osv/kernel_config_lazy_stack_invariant.h>
+#include <osv/kernel_config_fork.h>
 
 TRACEPOINT(trace_condvar_wait, "%p", condvar *);
 TRACEPOINT(trace_condvar_wake_one, "%p", condvar *);
@@ -22,7 +23,15 @@ int condvar::wait(mutex* user_mutex, sched::timer* tmr)
 {
     trace_condvar_wait(this);
     int ret = 0;
+#if CONF_fork
+    // A fork child (non-AS0) heap-allocates its wait_record so it stays
+    // coherent when the parent (different address space) walks this shared
+    // condvar's queue.  AS0 keeps the on-stack fast path.
+    coherent_wait_record wr_holder(sched::thread::current());
+    wait_record &wr = wr_holder.get();
+#else
     wait_record wr(sched::thread::current());
+#endif
 
     _m.lock();
     if (!_waiters_fifo.oldest) {

core/lfmutex.cc

@@ -10,6 +10,22 @@
 #include <osv/sched.hh>
 #include <osv/wait_record.hh>
 #include <osv/export.h>
+#include <osv/kernel_config_fork.h>
+#if CONF_fork
+#include <osv/mmu.hh>
+#endif
+
+#if CONF_fork
+// See wait_record.hh: true iff the current thread runs in a forked-child
+// (non-AS0) address space, in which case a wait_record queued on a shared
+// kernel object must be heap-allocated to stay coherent cross-address-space.
+bool fork_child_needs_heap_wait_record()
+{
+    auto t = sched::thread::current();
+    return t && t->address_space() &&
+           t->address_space() != mmu::kernel_address_space();
+}
+#endif
 
 namespace lockfree {
 
@@ -51,7 +67,15 @@ void mutex::lock()
     // when another thread releases the lock.
     // Note "waiter" is on the stack, so we must not return before making sure
     // it was popped from waitqueue (by another thread or by us.)
+#if CONF_fork
+    // A fork child (non-AS0) gets a heap-allocated wait_record so it stays
+    // coherent when the parent, in a different address space, dereferences it
+    // off this shared kernel mutex.  AS0 keeps the on-stack fast path.
+    coherent_wait_record waiter_holder(current);
+    wait_record &waiter = waiter_holder.get();
+#else
     wait_record waiter(current);
+#endif
     waitqueue.push(&waiter);
 
     // The "Responsibility Hand-Off" protocol where a lock() picks from