[DRAFT / tracking - DO NOT MERGE as one unit] PostgreSQL-on-OSv fork+ZFS integration (will split into S1..S6 on #1455/#1456/#1457 + #1423)

.gitignore

@@ -53,3 +53,4 @@ modules/dl_tests/usr.manifest
 compile_commands.json
 downloaded_packages
 .firecracker
+!modules/open_zfs/patches/*.patch

.gitmodules

@@ -21,3 +21,8 @@
 [submodule "kbuild"]
 	path = kbuild
 	url = https://github.com/osvunikernel/kbuild-standalone.git
+[submodule "open_zfs"]
+	path = modules/open_zfs/openzfs
+	url = https://github.com/openzfs/zfs.git
+	branch = zfs-2.4.2
+	ignore = dirty

arch/aarch64/fork.cc

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2026 Greg Burd
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ *
+ * fork_thread() for aarch64 -- mirrors arch/x64/fork.cc.  fork() (fork.cc)
+ * passes the caller's return address and stack pointer; we copy the parent's
+ * user stack, bias the SP into the copy, and start a child thread that installs
+ * the copied stack, sets x0=0 (fork()'s return value in the child), and returns
+ * to fork()'s caller.
+ */
+
+#include "arch.hh"
+#include <errno.h>
+#include <string.h>
+#include <cstdlib>
+#include <osv/sched.hh>
+
+// pthread_atfork child-handler chain (defined in libc/pthread.cc), run in the
+// child's context before it resumes user code.
+extern "C" void __osv_run_atfork_child();
+
+sched::thread *fork_thread(void *caller_ret, void *caller_sp,
+                           void *resume_ctx, void **out_stack_to_free)
+{
+    (void)resume_ctx;   // aarch64 uses caller_sp bias-copy (same-VA is x86-64)
+    auto parent = sched::thread::current();
+    auto parent_pinned_cpu = parent->pinned() ? sched::cpu::current() : nullptr;
+
+    auto si = parent->get_stack_info();
+    char *stack_base = static_cast<char*>(si.begin) + si.size;
+    char *sp = static_cast<char*>(caller_sp);
+    if (sp < static_cast<char*>(si.begin) || sp > stack_base) {
+        return nullptr;
+    }
+    size_t stack_size = si.size;
+
+    char *child_stack_mem = static_cast<char*>(malloc(stack_size));
+    if (!child_stack_mem) {
+        return nullptr;
+    }
+    // Copy ONLY the live top [caller_sp .. stack_base) into the top of the
+    // child buffer (app stacks are demand-paged; copying from si.begin faults).
+    char *child_base = child_stack_mem + stack_size;
+    ptrdiff_t bias = child_base - stack_base;
+    size_t live = static_cast<size_t>(stack_base - sp);
+    memcpy(child_base - live, sp, live);
+    char *child_sp = sp + bias;
+
+    volatile u64 resume_sp = reinterpret_cast<u64>(child_sp);
+    volatile u64 resume_pc = reinterpret_cast<u64>(caller_ret);
+    char *stack_to_free = child_stack_mem;
+
+    // TLS: the child is a real OSv thread with its own fresh setup_tcb() block.
+    // Only override tpidr_el0 if the parent had installed its own app TCB via
+    // arch_prctl (parent_app_tcb != 0); otherwise keep the child's private OSv
+    // TLS (the clean case for a musl app built against OSv's libc).
+    u64 parent_app_tcb = parent->get_app_tcb();
+    auto t = sched::thread::make([resume_sp, resume_pc, parent_app_tcb] {
+        if (parent_app_tcb) {
+            asm volatile ("msr tpidr_el0, %0; isb" :: "r"(parent_app_tcb) : "memory");
+        }
+        // Run pthread_atfork child handlers in the child's context before
+        // resuming user code.
+        __osv_run_atfork_child();
+        asm volatile
+          ("mov sp, %0 \n\t"    // install the private copied stack
+           "mov x0, #0 \n\t"    // fork() returns 0 in the child
+           "br %1 \n\t"         // resume in fork()'s caller
+           : : "r"(resume_sp), "r"(resume_pc) : "x0", "memory");
+    }, sched::thread::attr().
+        stack(4096 * 4).
+        // Detached: nobody join()s the fork child (the parent reaps it via the
+        // pid registry / waitpid, not sched::thread::join).  A detached thread
+        // is handed to the reaper on completion, which runs our set_cleanup()
+        // (freeing the copied stack and disposing the thread object, releasing
+        // its application_runtime reference).  Without this the thread object
+        // (and its app_runtime shared_ptr) would leak and OSv would hang at
+        // shutdown -- see the cleanup comment in libc/process/fork.cc.
+        detached(),
+        false,
+        true);
+    t->set_app_tcb(parent->get_app_tcb());
+    if (parent_pinned_cpu) {
+        t->pin(parent_pinned_cpu);
+    }
+    if (out_stack_to_free) {
+        *out_stack_to_free = stack_to_free;
+    }
+    return t;
+}

arch/x64/arch-switch.hh

@@ -13,6 +13,10 @@
 #include <osv/kernel_config_preempt.h>
 #include <osv/kernel_config_threads_default_kernel_stack_size.h>
 #include <osv/kernel_config_syscall_stack_size.h>
+#include <osv/kernel_config_fork.h>
+#if CONF_fork
+#include <osv/fork_arena.hh>
+#endif
 #include <string.h>
 #include "tls-switch.hh"
 
@@ -93,6 +97,26 @@ void thread::switch_to()
     barrier();
     set_fsbase(reinterpret_cast<u64>(_tcb));
     barrier();
+#if CONF_fork
+    // Address-space (CR3) switch for fork COW.  Only switch when the target
+    // thread lives in a different address space than the outgoing one, so the
+    // common single-address-space case pays nothing (and no TLB flush).
+    //
+    // CRITICAL (same-VA fork stacks): the CR3 write must happen EXACTLY at the
+    // rsp/rbp swap below, NOT here.  OSv runs kernel code (this switch, the
+    // fpu-cw/mxcsr scratch saves, the register-save asm) on the app thread's
+    // stack.  A forked child resumes on the parent's EXACT stack VAs, so parent
+    // and child alias the same stack virtual addresses (backed by different
+    // physical pages per address space).  If we loaded the incoming thread's
+    // CR3 here, the subsequent scratch writes to the OUTGOING thread's stack
+    // (fnstcw/stmxcsr at -0x34(%rbp), etc., with rsp/rbp still the old thread's)
+    // would resolve through the WRONG address space and clobber the incoming
+    // thread's physical page at that shared VA -- corrupting a blocked thread's
+    // live stack.  So we defer the CR3 load into the asm, coincident with the
+    // rsp/rbp swap: old-stack writes use the old AS, new-stack writes the new.
+    bool switch_as = (_current_as != old->_current_as);
+    u64 new_cr3 = switch_as ? mmu::pt_root_phys(_current_as) : 0;
+#endif
     auto c = _detached_state->_cpu;
     old->_state.exception_stack = c->arch.get_exception_stack();
     // save the old thread SYSCALL caller stack pointer in the syscall stack descriptor
@@ -108,6 +132,69 @@ void thread::switch_to()
     c->arch._current_thread_kernel_tcb = reinterpret_cast<u64>(_tcb);
     auto fpucw = processor::fnstcw();
     auto mxcsr = processor::stmxcsr();
+#if CONF_fork
+    if (switch_as) {
+        // Same-VA-safe switch: swap rsp/rbp to the incoming thread, THEN load
+        // its CR3 (mov %rax,%cr3), THEN jump.  All old-stack writes above ran
+        // under the old AS; every access after the rsp/rbp swap is to the new
+        // thread's stack, now resolved through the new AS -- so a shared stack
+        // VA never resolves through the wrong address space.  Kernel .text and
+        // the thread_state (heap) are identically mapped in every AS, so the
+        // instruction fetch across the mov-to-cr3 and the %c[rip] load are
+        // valid before and after the CR3 write.
+        // [fork-stack] latent lost-wakeup / COW-fault fix (see the block that
+        // this replaces below).  The old code re-tested `switch_as` and did the
+        // FPU-control-word reload in C++ AFTER this asm returned.  The compiler
+        // spills `switch_as` (and the fpucw/mxcsr locals) to %rbp-relative
+        // stack slots, and %rbp here already points at the INCOMING fork
+        // child's stack under the just-loaded child CR3.  So the post-swap
+        // `if (switch_as)` re-read (`mov -0x50(%rbp),%rdx; cmp %rdx,-0x48(%rbp)`)
+        // and the else-branch's `movzwl -0x52(%rbp)` / `mov %ax,-0x34(%rbp)`
+        // touch the child's COW stack in this irq-off, non-preemptable window:
+        // a COW write-fault there trips assert(preemptable()) (arch/x64/mmu.cc:38),
+        // or a mis-read `switch_as` takes the wrong branch and #GPs on garbage
+        // MXCSR -> the resumed child NEVER runs again (lost wakeup: a forked PG
+        // backend parks in switch_to and is never rescheduled -> boot/DROP
+        // DATABASE hang).  Dormant when the surrounding code compiles like
+        // c8f9c82b; near-certain once kill()'s layout shifts the fork timing.
+        //
+        // Fix: do the whole FPU-control-word restore INSIDE this asm, right at
+        // the `1:` resume, from a KERNEL-IDENTITY-MAPPED static (RIP-relative
+        // .rodata, same VA->phys in every AS, never COW).  No %rbp-relative
+        // (stack) read or write happens after the CR3 swap, so there is nothing
+        // to COW-fault or mis-read.  The canonical control words are correct for
+        // every OSv thread (issue #1020; switch_to does no fxsave/fxrstor, so no
+        // per-thread FPU data is preserved across a switch regardless), so this
+        // is semantically identical to the fldcw(fpucw)/ldmxcsr(mxcsr) below for
+        // the AS-crossing case.  The switch_as asm therefore RETURNS to C++
+        // already-FPU-restored; the shared post-asm reload runs only for the
+        // non-switch (same-AS) path, whose %rbp is the outgoing==incoming stack
+        // and is safe.
+        static const unsigned short canon_fpucw = 0x37f;
+        static const unsigned int   canon_mxcsr = 0x1f80;
+        asm volatile
+            ("mov %%rbp, %c[rbp](%0) \n\t"
+             "movq $1f, %c[rip](%0) \n\t"
+             "mov %%rsp, %c[rsp](%0) \n\t"
+             "mov %c[rsp](%1), %%rsp \n\t"
+             "mov %c[rbp](%1), %%rbp \n\t"
+             "mov %2, %%cr3 \n\t"
+             "jmpq *%c[rip](%1) \n\t"
+             "1: \n\t"
+             "emms \n\t"
+             "fldcw %3 \n\t"
+             "ldmxcsr %4 \n\t"
+             :
+             : "a"(&old->_state), "c"(&this->_state), "d"(new_cr3),
+               "m"(canon_fpucw), "m"(canon_mxcsr),
+               [rsp]"i"(offsetof(thread_state, rsp)),
+               [rbp]"i"(offsetof(thread_state, rbp)),
+               [rip]"i"(offsetof(thread_state, rip))
+             : "rbx", "rsi", "rdi", "r8", "r9",
+               "r10", "r11", "r12", "r13", "r14", "r15", "memory");
+        return;
+    } else
+#endif
     asm volatile
         ("mov %%rbp, %c[rbp](%0) \n\t"
          "movq $1f, %c[rip](%0) \n\t"
@@ -123,8 +210,12 @@ void thread::switch_to()
            [rip]"i"(offsetof(thread_state, rip))
          : "rbx", "rdx", "rsi", "rdi", "r8", "r9",
            "r10", "r11", "r12", "r13", "r14", "r15", "memory");
-    // As the catch-all solution, reset FPU state and more specifically
-    // its status word. For details why we need it please see issue #1020.
+    // Same-AS (non-fork-crossing) path only: the AS-crossing switch_as case
+    // above already ran emms + the identity FPU restore INSIDE its asm and
+    // returned, so nothing here reads the incoming child's COW stack.  Here
+    // %rbp is the (same) resumed thread's stack under an unchanged CR3, so the
+    // fpucw/mxcsr rbp-relative reloads are safe.  This is the ONLY path for
+    // conf_fork=0 (no switch_as), keeping that build byte-identical to before.
     asm volatile ("emms");
     processor::fldcw(fpucw);
     processor::ldmxcsr(mxcsr);
@@ -161,6 +252,18 @@ void thread::init_stack()
         stack.size = CONF_threads_default_kernel_stack_size;
     }
     if (!stack.begin) {
+#if CONF_fork
+        // The kernel thread stack must live in the identity kernel heap, never
+        // the COW fork arena: OSv runs kernel code (incl. context switches with
+        // preemption/irqs off) on it, where a COW write fault is illegal, and
+        // the thread REAPER frees it from AS0 in ~thread -> stack.deleter ->
+        // free(begin).  An arena-resident stack has a VA that is COW-private to
+        // the forked child and unmapped/divergent in the reaper's AS, so the
+        // reaper's free() faults on an arena address (0x3000..) -> page fault in
+        // ~thread.  Force it onto the shared identity heap, like the TCB/TLS and
+        // syscall stacks below.
+        fork_arena::kernel_heap_scope kh;
+#endif
         stack.begin = malloc(stack.size);
         stack.deleter = stack.default_deleter;
     } else {
@@ -181,8 +284,15 @@ void thread::init_stack()
 
     if (is_app()) {
         //
-        // Allocate TINY syscall call stack
-        void* tiny_syscall_stack_begin = malloc(TINY_SYSCALL_STACK_SIZE);
+        // Allocate TINY syscall call stack (identity heap, not the fork arena:
+        // the kernel runs syscalls on it and must not COW-fault it).
+        void* tiny_syscall_stack_begin;
+        {
+#if CONF_fork
+            fork_arena::kernel_heap_scope kh;
+#endif
+            tiny_syscall_stack_begin = malloc(TINY_SYSCALL_STACK_SIZE);
+        }
         assert(tiny_syscall_stack_begin);
         //
         // The top of the stack needs to be 16 bytes lower to make space for
@@ -262,7 +372,18 @@ void thread::setup_tcb()
     assert(align_check(user_tls_size, (size_t)64));
 
     auto total_tls_size = kernel_tls_size + user_tls_size;
-    void* p = aligned_alloc(64, total_tls_size + sizeof(*_tcb));
+    // The TLS block must live in the identity kernel heap, never the fork
+    // arena: it holds __thread variables accessed from every context including
+    // preemption/interrupt-disabled ones, so it must be plain-writable in every
+    // address space and must never become a copy-on-write page (a COW fault
+    // with preemption off is illegal).  Force the kernel heap for this alloc.
+    void* p;
+    {
+#if CONF_fork
+        fork_arena::kernel_heap_scope kh;
+#endif
+        p = aligned_alloc(64, total_tls_size + sizeof(*_tcb));
+    }
     // First goes user TLS data
     if (user_tls_size) {
         memcpy(p, user_tls_data, user_tls_size);
@@ -296,8 +417,15 @@ void thread::setup_large_syscall_stack()
     assert(is_app());
     assert(GET_SYSCALL_STACK_TYPE_INDICATOR() == TINY_SYSCALL_STACK_INDICATOR);
     //
-    // Allocate LARGE syscall stack
-    void* large_syscall_stack_begin = malloc(LARGE_SYSCALL_STACK_SIZE);
+    // Allocate LARGE syscall stack (identity heap, not the fork arena: the
+    // kernel runs syscalls on it and must never COW-fault it).
+    void* large_syscall_stack_begin;
+    {
+#if CONF_fork
+        fork_arena::kernel_heap_scope kh;
+#endif
+        large_syscall_stack_begin = malloc(LARGE_SYSCALL_STACK_SIZE);
+    }
     void* large_syscall_stack_top = large_syscall_stack_begin + LARGE_SYSCALL_STACK_DEPTH;
     //
     // Copy all of the tiny stack to the are of last 1024 bytes of large stack.