From 70c7f619d6d8660083974ea4f7e7757994c21849 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 27 Jul 2026 08:11:34 -0400 Subject: [PATCH 1/3] postgres18-musl: stock musl PostgreSQL 18 demo app for OSv Builds PostgreSQL 18 (REL_18_STABLE) from source with musl-gcc as an OSv PIE, boots a small baked cluster, and serves SQL over TCP. Single-backend demo; requires the OSv fork/COW coherence work (conf_fork=1 + the fork-arena kernel fixes) which is not yet on OSv master, so this stays a WIP branch until that lands. --- postgres18-musl/.gitignore | 8 ++ postgres18-musl/Makefile | 12 ++ postgres18-musl/README.md | 110 ++++++++++++++++++ postgres18-musl/build.sh | 215 ++++++++++++++++++++++++++++++++++++ postgres18-musl/module.py | 16 +++ postgres18-musl/seed_copy.c | 88 +++++++++++++++ 6 files changed, 449 insertions(+) create mode 100644 postgres18-musl/.gitignore create mode 100644 postgres18-musl/Makefile create mode 100644 postgres18-musl/README.md create mode 100755 postgres18-musl/build.sh create mode 100644 postgres18-musl/module.py create mode 100644 postgres18-musl/seed_copy.c diff --git a/postgres18-musl/.gitignore b/postgres18-musl/.gitignore new file mode 100644 index 00000000..3676616d --- /dev/null +++ b/postgres18-musl/.gitignore @@ -0,0 +1,8 @@ +# Generated by build.sh; not checked in. +/src/ +/build/ +/install/ +/emptydir/ +/seed_copy +/usr.manifest +*.tar.gz diff --git a/postgres18-musl/Makefile b/postgres18-musl/Makefile new file mode 100644 index 00000000..e27bd8a1 --- /dev/null +++ b/postgres18-musl/Makefile @@ -0,0 +1,12 @@ +.PHONY: module +module: build + +# Fetch PostgreSQL 18 source, build it with musl-gcc as an OSv PIE, apply the +# minimal OSv-environment neuters, initialize a small demo cluster, build the +# seed_copy helper, and emit usr.manifest pointing at the build output. +build: + ./build.sh + +.PHONY: clean +clean: + rm -rf src build install usr.manifest *.tar.gz seed_copy emptydir diff --git a/postgres18-musl/README.md b/postgres18-musl/README.md new file mode 100644 index 00000000..43da9c94 --- /dev/null +++ b/postgres18-musl/README.md @@ -0,0 +1,110 @@ +# postgres18-musl + +Stock **PostgreSQL 18** (branch `REL_18_STABLE`) built from source with the +**musl** C toolchain as an OSv position-independent executable, booting on OSv +and serving SQL over TCP. + +This is a demo: it boots a small baked cluster and answers `psql` queries over +the guest's forwarded port. It is deliberately single-backend-safe (see +[Status](#status)). + +## What it does + +1. `build.sh` fetches the PostgreSQL 18 source, builds it with `musl-gcc`, + `initdb`s a small demo cluster containing a `demo` database with a + `greetings` table, and lays the install tree + baked cluster out for the + image manifest. +2. At boot, `module.py` mounts a writable ramfs over `/data`, copies the + read-only baked cluster from `/data-seed` (on rofs) into it with the tiny + `seed_copy` helper, then starts `postgres -D /data`. +3. PostgreSQL listens on TCP `0.0.0.0:5432` and serves queries. + +## Build & run + +From the top-level OSv directory (this app lives under `apps/`): + +```bash +# Build the OSv image with this app. conf_fork=1 pulls in OSv's fork/COW +# cross-address-space coherence (PostgreSQL's postmaster forks a backend per +# connection). fs=ramfs makes the root (and thus the writable /data) a ramfs. +./scripts/build -j$(nproc) conf_fork=1 fs=ramfs image=postgres18-musl + +# Boot under KVM, forwarding the guest's 5432 to the host (fork needs >=2 vCPUs): +./scripts/run.py -c2 -nvV --forward tcp::5432-:5432 + +# From the host, query the demo over the forwarded port: +psql -h 127.0.0.1 -p 5432 -U postgres -d demo -c 'select * from greetings;' +``` + +> **OSv kernel requirement.** PostgreSQL forks a backend per connection, so this +> demo needs the OSv fork/COW cross-address-space coherence work (`conf_fork=1`, +> plus the DSM/POSIX-shm-registry, ramfs, net-stack, mbuf and thread-stack +> identity-heap fixes). On an OSv tree without those, the postmaster boots but a +> forked backend hits `could not open shared memory segment` (DSM ENOENT) and no +> query is served. Those fixes are a separate OSv kernel effort (fork-arena / +> PR #1455 line of work); this app is the userspace half of the demo. + +Expected output: + +``` + id | who | msg +----+------------+------------------------ + 1 | osv | hello from a unikernel + 2 | postgres18 | stock musl build + 3 | demo | this row came over TCP +(3 rows) +``` + +## OSv deviations from stock PostgreSQL + +Every deviation below is an **OSv-workaround**, not an improvement to +PostgreSQL. Each exists only because a current OSv capability gap forces it, +and each is tracked to be **erased** as those gaps close. Nothing here changes +PostgreSQL's on-disk format or SQL behavior. + +**Build flags** (`build.sh`, step 3): + +| Deviation | Why | Erase when | +|---|---|---| +| `CC=musl-gcc` | OSv's libc is musl | — (intrinsic to the musl target) | +| `--without-icu/zlib/readline/lz4/zstd/libxml` | those libs are not baked into the demo image | those libs are packaged for OSv | +| `-DWAIT_USE_SELF_PIPE` | OSv's epoll/signalfd latch path is incomplete; use PostgreSQL's portable self-pipe latch | OSv latch path is complete | +| `LDFLAGS_EX=-pie` | OSv runs applications as PIEs | — (intrinsic to OSv) | + +**Source neuters** (`build.sh`, step 2 -- two one-line `if (0 && ...)` guards): + +| Deviation | Why | Erase when | +|---|---|---| +| `check_root()` in `main.c` | OSv runs the app as uid 0; there is no unprivileged user to drop to | OSv models an unprivileged user | +| `checkDataDir()` perms in `miscinit.c` | OSv filesystems do not report Unix-style directory modes | OSv fs reports Unix modes | + +**Cluster config** (`build.sh`, step 5, baked into `postgresql.conf`/`pg_hba.conf`): + +| Deviation | Why | Erase when | +|---|---|---| +| `initdb --locale=C` | musl provides only the C locale | musl locale support lands | +| `io_method = sync` | OSv's async-I/O (io_uring/worker) path is incomplete | OSv AIO path is complete | +| `unix_socket_directories = ''` | OSv has no AF_UNIX | OSv AF_UNIX support lands | +| `listen_addresses = '*'` + `pg_hba trust` | demo reachability over the forwarded TCP port | it's a demo (ephemeral ramfs cluster) | + +## Status + +- **Works: single-backend query serving.** The demo boots PostgreSQL to + "database system is ready to accept connections" and serves real SQL + (`SELECT`, DDL, DML, aggregates) over TCP. Sequential `psql` connections are + reliable. +- **WIP: sustained multi-backend concurrency.** PostgreSQL's postmaster forks a + backend per connection, and OSv's `fork()`/COW cross-address-space coherence + is still being hardened (a separate OSv kernel effort). A demo doing + sequential queries is fine; hammering it with many concurrent connections can + trip the fork/reap lifecycle wall. This app is intentionally scoped to the + single-backend demo. + +## Files + +- `Makefile` -- osv-apps entry point; runs `build.sh`. +- `build.sh` -- fetch + build PostgreSQL, build `seed_copy`, `initdb` the demo + cluster, emit `usr.manifest`. +- `seed_copy.c` -- boot-time recursive rofs→ramfs copy helper (libc only). +- `module.py` -- boot command (mount ramfs, seed_copy, run postgres). +- `usr.manifest` -- generated by `build.sh`; maps the build output into the image. diff --git a/postgres18-musl/build.sh b/postgres18-musl/build.sh new file mode 100755 index 00000000..4e3efd3d --- /dev/null +++ b/postgres18-musl/build.sh @@ -0,0 +1,215 @@ +#!/bin/bash +# +# Build stock musl PostgreSQL 18 as an OSv PIE and bake a small demo cluster. +# +# Runs inside the OSv build container (fedora:39 with musl-gcc). Produces: +# install/ -- the PostgreSQL install tree (bin, lib, share) +# install/data-seed/ -- a baked demo cluster (read-only at runtime on rofs) +# seed_copy -- the boot-time rofs->ramfs copy helper +# usr.manifest -- maps the above into the OSv image +# +# Everything below "OSv deviation" is a workaround for a current OSv gap, kept +# minimal and documented, and expected to be erased as those gaps close. +set -e + +HERE="$(cd "$(dirname "$0")" && pwd)" +PG_BRANCH="${PG_BRANCH:-REL_18_STABLE}" +SRC="$HERE/src" +BUILD="$HERE/build" +PREFIX="$HERE/install" +SEED="$PREFIX/data-seed" +JOBS="${JOBS:-$(nproc)}" + +: "${CC:=musl-gcc}" +export CC + +# --------------------------------------------------------------------------- +# 1. Fetch the PostgreSQL 18 source. +# --------------------------------------------------------------------------- +if [ ! -f "$SRC/configure" ]; then + echo "== fetching PostgreSQL $PG_BRANCH ==" + rm -rf "$SRC" + TARBALL="$HERE/postgres-$PG_BRANCH.tar.gz" + [ -f "$TARBALL" ] || wget -q -O "$TARBALL" \ + "https://github.com/postgres/postgres/archive/refs/heads/$PG_BRANCH.tar.gz" + mkdir -p "$SRC" + tar xzf "$TARBALL" -C "$SRC" --strip-components=1 +fi + +# --------------------------------------------------------------------------- +# 2. OSv deviation: neuter two host-environment checks. +# +# OSv runs the unikernel application as uid 0 (there is no unprivileged user +# to drop to) and its filesystems do not report Unix-style directory perms, +# so PostgreSQL's two startup safety checks reject a perfectly valid OSv +# environment. We disable exactly those two checks and nothing else. Both +# edits are idempotent (skipped if already applied). +# --------------------------------------------------------------------------- +python3 - "$SRC" <<'PY' +import sys, io +src = sys.argv[1] + +# 2a. check_root() in main.c -- allow uid 0. +p = src + "/src/backend/main/main.c" +s = io.open(p).read() +needle = '\tif (geteuid() == 0)\n\t{\n\t\twrite_stderr("\\"root\\" execution' +repl = '\t/* OSv: unikernel app runs as uid 0; no unprivileged user. */\n\tif (0 && geteuid() == 0)\n\t{\n\t\twrite_stderr("\\"root\\" execution' +if repl not in s: + assert s.count(needle) == 1, ("main.c check_root: matches=%d" % s.count(needle)) + io.open(p, "w").write(s.replace(needle, repl)) + print("patched main.c check_root") +else: + print("main.c check_root already patched") + +# 2b. checkDataDir() perm check in miscinit.c -- OSv fs modes are not Unix-y. +p = src + "/src/backend/utils/init/miscinit.c" +s = io.open(p).read() +needle = "\tif (stat_buf.st_mode & PG_MODE_MASK_GROUP)\n" +repl = "\t/* OSv: filesystem modes are not Unix-y; suppress the perms check. */\n\tif (0 && (stat_buf.st_mode & PG_MODE_MASK_GROUP))\n" +if repl not in s: + assert s.count(needle) == 1, ("miscinit.c checkDataDir: matches=%d" % s.count(needle)) + io.open(p, "w").write(s.replace(needle, repl)) + print("patched miscinit.c checkDataDir") +else: + print("miscinit.c checkDataDir already patched") +PY + +# --------------------------------------------------------------------------- +# 3. Configure + build with musl-gcc as a PIE. +# +# OSv deviations in the build flags: +# CC=musl-gcc -- OSv's libc is musl; build against it. +# --without-icu/zlib/ -- avoid optional libs not baked into the image +# readline/lz4/zstd (keeps the demo self-contained; add back once +# those libs are packaged for OSv). +# -DWAIT_USE_SELF_PIPE -- OSv's epoll/signalfd latch path is not complete; +# force PostgreSQL's portable self-pipe latch. +# LDFLAGS_EX=-pie -- OSv runs applications as position-independent +# executables. +# --------------------------------------------------------------------------- +echo "== configuring (CC=$CC) ==" +if [ -x "$PREFIX/bin/postgres" ]; then + echo "== postgres already built, skipping configure/make (rm -rf install to force) ==" +else + rm -rf "$BUILD" "$PREFIX" + mkdir -p "$BUILD" + cd "$BUILD" + + CFLAGS='-O2 -g -fPIC -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 -DWAIT_USE_SELF_PIPE -idirafter /usr/include' + "$SRC/configure" \ + --prefix="$PREFIX" \ + --without-icu --without-zlib --without-readline \ + --without-libxml --without-lz4 --without-zstd \ + CFLAGS="$CFLAGS" >/tmp/pg-configure.log 2>&1 || { tail -20 /tmp/pg-configure.log; exit 1; } + + echo "== building (-j$JOBS) ==" + # A fully clean parallel build can race src/common ahead of the generated + # catalog/error-code headers; generate those serially first. + make -C src/backend generated-headers >/tmp/pg-genhdr.log 2>&1 || { tail -20 /tmp/pg-genhdr.log; exit 1; } + make -j"$JOBS" LDFLAGS_EX='-pie' >/tmp/pg-make.log 2>&1 || { tail -30 /tmp/pg-make.log; exit 1; } + make install >/tmp/pg-install.log 2>&1 || { tail -20 /tmp/pg-install.log; exit 1; } +fi +echo "== built: $(file "$PREFIX/bin/postgres" | cut -d: -f2-) ==" + +# --------------------------------------------------------------------------- +# 4. Build the seed_copy boot helper (rofs -> ramfs cluster copy). +# --------------------------------------------------------------------------- +echo "== building seed_copy ==" +"$CC" -O2 -fPIC -pie -o "$HERE/seed_copy" "$HERE/seed_copy.c" + +# --------------------------------------------------------------------------- +# 5. Initialize a small demo cluster (runs natively in the build container). +# +# OSv deviations baked into the cluster: +# initdb --locale=C -- musl has only the C locale. +# io_method = sync -- OSv's io_uring/worker AIO path is incomplete; +# use synchronous I/O. +# unix_socket_directories = '' -- OSv has no AF_UNIX; TCP only. +# listen_addresses = '*' + pg_hba trust -- so the demo is reachable over +# the guest's forwarded TCP port for `psql -h`. +# +# initdb/postgres refuse to run as root (a build-container concern, not an +# OSv one -- on OSv the neuter in step 2 handles uid 0). If we are root, +# create/reuse an unprivileged 'pgbuild' user and run the cluster init as +# that user, owning $SEED. +# --------------------------------------------------------------------------- +echo "== initdb demo cluster ==" +rm -rf "$SEED" +mkdir -p "$SEED" +export LC_ALL=C LANG=C + +PGRUN="" +if [ "$(id -u)" = 0 ]; then + id pgbuild >/dev/null 2>&1 || useradd -m pgbuild + chown -R pgbuild "$SEED" "$HERE" 2>/dev/null || true + PGRUN="runuser -u pgbuild --" +fi + +$PGRUN "$PREFIX/bin/initdb" -D "$SEED" -U postgres --locale=C -E UTF8 \ + >/tmp/pg-initdb.log 2>&1 || { tail -30 /tmp/pg-initdb.log; exit 1; } + +cat >> "$SEED/postgresql.conf" <<'CONF' + +# --- OSv demo settings (see README) --- +listen_addresses = '*' # reachable over forwarded TCP +port = 5432 +unix_socket_directories = '' # OSv has no AF_UNIX +io_method = sync # OSv AIO path incomplete; use sync I/O +fsync = on +CONF + +# Trust auth for the demo (single, local, ephemeral cluster over the demo port). +cat > "$SEED/pg_hba.conf" <<'HBA' +# OSv demo: trust auth. The cluster is ephemeral (ramfs) and only reachable +# over the guest's forwarded TCP port. Do not use trust in production. +host all all 0.0.0.0/0 trust +host all all ::/0 trust +HBA + +# Seed a demo database + table with a few rows, using a throwaway local +# postmaster over a TCP port (AF_UNIX is unavailable here too, to match OSv). +echo "== seeding demo data ==" +$PGRUN "$PREFIX/bin/pg_ctl" -D "$SEED" -o "-p 55432 -k '' -h 127.0.0.1" -w start \ + >/tmp/pg-seed-start.log 2>&1 || { tail -30 /tmp/pg-seed-start.log; exit 1; } +$PGRUN "$PREFIX/bin/psql" -h 127.0.0.1 -p 55432 -U postgres -d postgres -v ON_ERROR_STOP=1 <<'SQL' +CREATE DATABASE demo; +\connect demo +CREATE TABLE greetings (id serial PRIMARY KEY, who text, msg text); +INSERT INTO greetings (who, msg) VALUES + ('osv', 'hello from a unikernel'), + ('postgres18', 'stock musl build'), + ('demo', 'this row came over TCP'); +SQL +$PGRUN "$PREFIX/bin/pg_ctl" -D "$SEED" -w stop >/tmp/pg-seed-stop.log 2>&1 + +# OSv manifest globs (**) only capture files, so empty cluster subdirectories +# (pg_notify, pg_dynshmem, pg_wal/archive_status, pg_tblspc, ...) would be +# dropped from the baked /data-seed and PostgreSQL would fail at startup with +# ENOENT. Drop a .keep marker into each empty dir so the manifest carries it; +# seed_copy recreates the directory in the writable /data and skips the marker +# itself (so pg_tblspc, where PostgreSQL treats any entry as a tablespace, ends +# up as a clean empty directory). +find "$SEED" -type d -empty -exec touch {}/.keep \; + +# --------------------------------------------------------------------------- +# 6. Emit usr.manifest -- maps the build output into the OSv image. +# All paths are relative to this module dir ($${MODULE_DIR}); no absolute +# build-machine paths. /data is an empty mount point the boot command +# mounts a ramfs over and seed_copy fills from /data-seed. +# --------------------------------------------------------------------------- +echo "== writing usr.manifest ==" +mkdir -p "$HERE/emptydir" +cat > "$HERE/usr.manifest" <<'MAN' +# musl PostgreSQL 18 on OSv -- demo module. +# The read-only cluster is baked at /data-seed; at boot seed_copy copies it into +# a writable ramfs at /data (see module.py). +/usr/bin/postgres: ${MODULE_DIR}/install/bin/postgres +/usr/bin/seed_copy: ${MODULE_DIR}/seed_copy +/usr/lib/**: ${MODULE_DIR}/install/lib/** +/usr/share/**: ${MODULE_DIR}/install/share/** +/data-seed/**: ${MODULE_DIR}/install/data-seed/** +# empty mount point for the writable ramfs /data (rofs cannot mkdir at runtime) +/data: ${MODULE_DIR}/emptydir +MAN + +echo "== done. postgres18-musl app built. ==" diff --git a/postgres18-musl/module.py b/postgres18-musl/module.py new file mode 100644 index 00000000..b9693bc4 --- /dev/null +++ b/postgres18-musl/module.py @@ -0,0 +1,16 @@ +from osv.modules import api + +# Boot sequence (OSv runs ;-separated commands in one instance over a shared +# filesystem): +# 1. mount a writable ramfs over the empty /data mount point (the baked +# cluster on rofs is read-only; PostgreSQL must write to its data dir); +# 2. seed_copy the read-only baked cluster (/data-seed) into /data; +# 3. start the postmaster on the writable copy. +# +# The baked postgresql.conf sets unix_socket_directories='' (OSv has no +# AF_UNIX) and listen_addresses='*', so PostgreSQL serves TCP on 0.0.0.0:5432. +default = api.run( + '--mount-fs=ramfs,/dev/null,/data ' + '/usr/bin/seed_copy /data-seed /data ; ' + '/usr/bin/postgres -D /data' +) diff --git a/postgres18-musl/seed_copy.c b/postgres18-musl/seed_copy.c new file mode 100644 index 00000000..98e9d59d --- /dev/null +++ b/postgres18-musl/seed_copy.c @@ -0,0 +1,88 @@ +/* + * seed_copy -- recursively copy SRC into DST, preserving file modes. + * + * OSv boots PostgreSQL's data directory from a read-only baked cluster on the + * image filesystem (rofs). PostgreSQL must write to its data directory, so at + * boot we copy the baked cluster (/data-seed) into a writable ramfs (/data) + * before starting the postmaster. cp -a is not available on the OSv image, + * so this tiny helper does the copy with nothing but libc. + * + * Usage: seed_copy + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int copy_file(const char *src, const char *dst, mode_t mode) +{ + int in = open(src, O_RDONLY); + if (in < 0) { perror(src); return -1; } + int out = open(dst, O_WRONLY | O_CREAT | O_TRUNC, mode & 0777); + if (out < 0) { perror(dst); close(in); return -1; } + + char buf[65536]; + ssize_t n; + while ((n = read(in, buf, sizeof buf)) > 0) { + char *p = buf; + while (n > 0) { + ssize_t w = write(out, p, n); + if (w < 0) { perror(dst); close(in); close(out); return -1; } + p += w; n -= w; + } + } + if (n < 0) { perror(src); close(in); close(out); return -1; } + close(in); + close(out); + return 0; +} + +static int copy_tree(const char *src, const char *dst) +{ + struct stat st; + if (lstat(src, &st) < 0) { perror(src); return -1; } + + if (S_ISDIR(st.st_mode)) { + if (mkdir(dst, st.st_mode & 0777) < 0 && errno != EEXIST) { + perror(dst); + return -1; + } + DIR *d = opendir(src); + if (!d) { perror(src); return -1; } + struct dirent *e; + int rc = 0; + while ((e = readdir(d))) { + if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) + continue; + /* .keep is a directory-preservation marker (OSv manifest globs + * drop empty dirs); the dir is now created, skip the marker. */ + if (!strcmp(e->d_name, ".keep")) + continue; + char s[4096], t[4096]; + snprintf(s, sizeof s, "%s/%s", src, e->d_name); + snprintf(t, sizeof t, "%s/%s", dst, e->d_name); + if (copy_tree(s, t) < 0) rc = -1; + } + closedir(d); + return rc; + } + return copy_file(src, dst, st.st_mode); +} + +int main(int argc, char **argv) +{ + if (argc != 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + if (copy_tree(argv[1], argv[2]) < 0) { + fprintf(stderr, "seed_copy: failed copying %s -> %s\n", argv[1], argv[2]); + return 1; + } + return 0; +} From 7215030a67e6a44f740720d363feec04a9d92eee Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 30 Jul 2026 09:59:11 -0400 Subject: [PATCH 2/3] postgres18-musl: Firecracker snapshot/restore (boot once, resume already-serving) Add a documented full-VM snapshot/restore capability to the demo: boot OSv+PostgreSQL under Firecracker to "ready to accept connections", take a Full snapshot (guest memory + device/vCPU state), and on later launches restore that snapshot so PostgreSQL is already serving. No reboot, no re-initdb, no musl-PIE re-mmap. Measured on m5d.metal (Firecracker v1.7.0, OSv v0.57 PG18.4-musl + OpenZFS, single-file ZFS pool recordsize=8k): cold launch -> first external select 1 = 11.75 s restore -> first external select 1 = 0.24 s (median of 5, ~48x) FC resume itself = 0.05 s The ~6.7 s PostgreSQL musl-PIE mmap+init is eliminated (baked into the snapshot). OSv resumes cleanly with NO OSv kernel changes required (already snapshot-ready). Proof it is a resume not a reboot: pg_postmaster_start_time() after restore is the original pre-snapshot boot time. Disk consistency via a fresh COW copy (cp --reflink=auto) of the pool at snapshot time so the guest ARC/shared_buffers agree with disk. Honest gotchas documented: clock skew (kvmclock re-syncs, ~50-85 ms stable offset, tolerated), virtio re-attach (host tap+DNAT must pre-exist), RNG (no observed duplication, weak probe). Files: fc-snapshot.sh (driver), fc-snap.py (Firecracker API helper), README section with build/snapshot/restore steps, measured numbers, and gotchas. Author: Greg Burd --- postgres18-musl/README.md | 126 ++++++++++++++++++++++++++ postgres18-musl/fc-snap.py | 157 +++++++++++++++++++++++++++++++++ postgres18-musl/fc-snapshot.sh | 152 +++++++++++++++++++++++++++++++ 3 files changed, 435 insertions(+) create mode 100755 postgres18-musl/fc-snap.py create mode 100755 postgres18-musl/fc-snapshot.sh diff --git a/postgres18-musl/README.md b/postgres18-musl/README.md index 43da9c94..87f968f5 100644 --- a/postgres18-musl/README.md +++ b/postgres18-musl/README.md @@ -55,6 +55,128 @@ Expected output: (3 rows) ``` +## Snapshot / restore (Firecracker) -- boot once, resume already-serving + +Booting PostgreSQL is not free: on OSv the musl PIE `postgres` install tree is +memory-mapped and the process initialized on every cold boot, which dominates +launch time. Firecracker full-VM snapshots let you pay that **once**: boot +OSv+PostgreSQL to "ready to accept connections", snapshot the whole VM (guest +memory + device/vCPU state), and on every later launch **restore** that +snapshot so PostgreSQL is *already serving* -- no reboot, no re-`initdb`, no +re-mmap. + +### Measured, real numbers + +Measured on an `m5d.metal` EC2 instance (AL2023, Firecracker v1.7.0, OSv v0.57 +with PostgreSQL 18.4-musl + OpenZFS 2.4.3, a single-file ZFS pool at +`recordsize=8k`, guest `select 1` over the real virtio-net path): + +| Path | launch -> first external `select 1` | +|---|---| +| **cold** boot (kernel + ZFS import + PG mmap/init + ready + first query) | **11.75 s** | +| **restore** from a Full snapshot | **0.24 s** (median of 5; range 0.234-0.244) | + +That is **~48x faster**; the ~6.7 s PostgreSQL musl-PIE mmap+init is eliminated +because it is baked into the snapshotted memory. The Firecracker resume itself +(load + resume) is ~0.05 s; the rest is the first TCP query round-trip. + +Cold-boot breakdown (for reference): kernel 0.61 s, ZFS import+mount 0.71 s, PG +spawn+init **6.70 s**, PG start->ready 0.28 s, ready->first query ~1.4 s. + +### How to snapshot and restore + +`fc-snapshot.sh` (with its helper `fc-snap.py`, which drives Firecracker's HTTP +API over a unix socket) automates the flow. It needs a bare-metal host +(`/dev/kvm`), `firecracker` v1.7+, and `qemu-img`. + +```bash +# 0. Build a ZFS-rooted image so the cluster is a real ZFS pool that stays +# consistent across snapshot/restore (ramfs would not survive a restore +# as a real filesystem). fork/COW is still required for the forked backends. +./scripts/build -j$(nproc) conf_fork=1 fs=zfs conf_zfs=openzfs image=postgres18-musl + +cd apps/postgres18-musl + +# 1. One-time host setup: bring up the tap+DNAT and convert the OSv qcow2 +# rootfs to raw (Firecracker only reads RAW block devices). +sudo ./fc-snapshot.sh setup + +# 2. (seed a single-file ZFS pool with the demo cluster once -- create the pool +# in-guest and cpiod-push the baked cluster into /data, then export; save it +# as pool-pristine.raw. See the prototype orchestrator for the exact steps.) + +# 3. Boot to "ready", then take a Full snapshot (memory + vmstate): +sudo ./fc-snapshot.sh boot +sudo ./fc-snapshot.sh snapshot # -> snap/osvpg.vmstate (~90 KB) + snap/osvpg.mem + +# 4. Restore + verify PG is already serving (no reboot): +sudo ./fc-snapshot.sh restore +sudo ./fc-snapshot.sh query # select 1 returns immediately + +# 5. Benchmark restore -> first query (median of N restores): +sudo ./fc-snapshot.sh measure 5 +``` + +The snapshot is two files: a tiny **vmstate** (device + vCPU state, ~90 KB) and +a **mem** file the size of the guest RAM (dense). Boot the guest with a small +`GMEM` (e.g. 2-4 GiB, matched to `shared_buffers` + working set) to keep the +mem file small; the memory is mostly zero pages and compresses heavily (a 16 +GiB dump gzipped to ~160 MB in testing). Firecracker diff snapshots shrink it +further. + +### Disk consistency (memory <-> disk must agree) + +A restored guest's in-memory ZFS ARC, PostgreSQL `shared_buffers`, and dirty +pages reflect the disk **as of the snapshot instant**. The pool backing file +must match. `fc-snapshot.sh` copies the exact pool file at snapshot time to +`pool-at-snapshot.raw` and gives **every restore a fresh copy** (via +`cp --reflink=auto`, a copy-on-write clone on reflink-capable filesystems such +as XFS/Btrfs/ZFS; it falls back to a plain copy elsewhere). This is the +zero-/low-copy COW overlay approach: memory and disk always agree, and each +restored clone gets an isolated writable pool. Verified: a row `INSERT`ed +*before* the snapshot is readable *after* restore, `CHECKPOINT` flushes, and +cold table reads serve -- memory + disk state genuinely resumed. + +### Gotchas (honest) + +- **Clock skew.** A restored guest wakes "in the past" relative to the host + wall clock. In testing OSv's `kvmclock` re-reads the KVM pvclock/wall-clock + MSRs on resume (there is a 1 Hz wall-clock sync thread), so the guest clock + re-tracks the host with only a **small, stable offset (~50-85 ms ahead)** -- + no backward jump, no timer storm, no hang. PostgreSQL timestamps are correct. + Tolerated with no OSv change. If sub-millisecond accuracy on the very first + post-resume tick is ever required, force an immediate kvmclock resync instead + of waiting up to 1 s for the sync thread. +- **virtio device re-attach.** Firecracker restores virtio-net/virtio-blk + device + virtqueue state, but the **host** side must pre-exist identically: + the `fc_tap0` tap and the DNAT rules must already be up on the restoring host + (`fc-snapshot.sh setup` establishes them; they persist). After restore a + fresh TCP connection completes through the restored virtio-net, and + virtio-blk serves reads/writes + `CHECKPOINT`. +- **RNG / entropy.** Two restores from the *same* snapshot produced different + `random()` and `gen_random_uuid()` values (no observable duplication at the + app layer). This is only a weak probe, though -- a security-sensitive clone + fleet should add an explicit OS-entropy reseed on resume so cloned VMs cannot + share RNG state. + +### Does OSv resume cleanly? Yes -- no OSv changes needed. + +OSv v0.57 (the full PG-serving fix stack) resumes cleanly from a Firecracker +Full snapshot with a live PostgreSQL+OpenZFS workload: **no core fault, no +clock discontinuity, no virtio re-attach failure, no IRQ/APIC re-arm bug**. The +definitive proof that it is a resume and not a reboot: `pg_postmaster_start_time()` +after restore reports the *original* pre-snapshot boot time, not the restore +wall clock -- the postmaster and its timers continue, never restart. This demo +therefore does **not** depend on any new OSv kernel change for correct resume. +(The two optional hardening items above -- RNG reseed, immediate clock resync +-- are not required for correctness here and would be separate OSv kernel PRs +if a production clone fleet ever needs them.) + +> One deployment gotcha that is **not** an OSv bug: OSv's `usr.img` is a QCOW2 +> image and Firecracker only accepts RAW block devices. Convert once with +> `qemu-img convert -O raw usr.img usr.raw` (`fc-snapshot.sh setup` does this). +> Skipping it makes the ZFS root import fail (`spa_import_rootpool ... failed: 5`). + ## OSv deviations from stock PostgreSQL Every deviation below is an **OSv-workaround**, not an improvement to @@ -108,3 +230,7 @@ PostgreSQL's on-disk format or SQL behavior. - `seed_copy.c` -- boot-time recursive rofs→ramfs copy helper (libc only). - `module.py` -- boot command (mount ramfs, seed_copy, run postgres). - `usr.manifest` -- generated by `build.sh`; maps the build output into the image. +- `fc-snapshot.sh` -- Firecracker snapshot/restore driver (boot-once, + resume-already-serving). See "Snapshot / restore" above. +- `fc-snap.py` -- helper that drives Firecracker's HTTP API (boot / Full + snapshot / restore+resume) over a unix socket. diff --git a/postgres18-musl/fc-snap.py b/postgres18-musl/fc-snap.py new file mode 100755 index 00000000..ed0e283b --- /dev/null +++ b/postgres18-musl/fc-snap.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Snapshot-capable Firecracker driver for OSv+PG. +# Modes: +# boot : boot OSv+PG via the API socket (drives + tap net), leave running. +# snapshot : pause a running VM and write a Full snapshot (vmstate + mem). +# restore : from a NEW fc process, load a snapshot and resume the VM. +# +# Uses the FC HTTP API over a unix socket (requests via requests_unixsocket-free +# raw http over AF_UNIX with the stdlib). Config mirrors fc-run.py's device order. +import argparse, json, os, socket, subprocess, sys, time + +FC = "/usr/local/bin/firecracker" +OSV = os.environ.get("OSV", os.path.expanduser("~/osv")) + "/build/last" +KERNEL = OSV + "/loader-stripped.elf" +ROOTFS = OSV + "/usr.raw" +GUEST_IP = "192.168.100.2"; GW = "192.168.100.1"; TAP = "fc_tap0" +MAC = "52:54:00:12:34:56" + +def http_unix(sock_path, method, path, body=None, tries=200): + """Minimal HTTP/1.1 over AF_UNIX. Returns (status, body_str).""" + data = json.dumps(body) if body is not None else None + for _ in range(tries): + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.connect(sock_path); break + except (FileNotFoundError, ConnectionRefusedError): + time.sleep(0.02) + else: + raise RuntimeError("FC socket never came up: %s" % sock_path) + req = "%s %s HTTP/1.1\r\nHost: localhost\r\nAccept: application/json\r\n" % (method, path) + if data is not None: + req += "Content-Type: application/json\r\nContent-Length: %d\r\n" % len(data) + req += "\r\n" + if data is not None: + req += data + s.sendall(req.encode()) + resp = b"" + while b"\r\n\r\n" not in resp: + chunk = s.recv(65536) + if not chunk: break + resp += chunk + head, _, rest = resp.partition(b"\r\n\r\n") + status = int(head.split(b" ")[1]) + # read body per content-length if present + clen = 0 + for line in head.split(b"\r\n"): + if line.lower().startswith(b"content-length:"): + clen = int(line.split(b":")[1]) + while len(rest) < clen: + rest += s.recv(65536) + s.close() + return status, rest.decode(errors="replace") + +def api(sock, method, path, body=None, ok=(200, 204)): + st, b = http_unix(sock, method, path, body) + if st not in ok: + raise RuntimeError("FC API %s %s -> %d: %s" % (method, path, st, b)) + return st, b + +def start_fc(sock, logf): + if os.path.exists(sock): os.unlink(sock) + lf = open(logf, "w") + p = subprocess.Popen([FC, "--api-sock", sock], stdout=lf, stderr=subprocess.STDOUT) + return p + +def do_boot(a): + global KERNEL, ROOTFS + if a.kernel: KERNEL = a.kernel + if a.rootfs: ROOTFS = a.rootfs + cmdline = a.append + if a.net: + cmdline = "--ip=eth0,%s,255.255.255.0 --defaultgw=%s %s" % (GUEST_IP, GW, cmdline) + cmdline = "--nopci " + cmdline + p = start_fc(a.socket, a.fclog) + # machine-config + api(a.socket, "PUT", "/machine-config", + {"vcpu_count": a.vcpus, "mem_size_mib": a.mem, "smt": False}) + api(a.socket, "PUT", "/boot-source", + {"kernel_image_path": KERNEL, "boot_args": cmdline}) + # drives: rootfs then members + api(a.socket, "PUT", "/drives/rootfs", + {"drive_id": "rootfs", "path_on_host": ROOTFS, + "is_root_device": False, "is_read_only": False}) + members = [m for m in a.members.split(",") if m] + for i, m in enumerate(members, start=1): + api(a.socket, "PUT", "/drives/blk%d" % i, + {"drive_id": "blk%d" % i, "path_on_host": m, + "is_root_device": False, "is_read_only": False}) + if a.net: + api(a.socket, "PUT", "/network-interfaces/eth0", + {"iface_id": "eth0", "host_dev_name": TAP, "guest_mac": MAC}) + api(a.socket, "PUT", "/actions", {"action_type": "InstanceStart"}) + print("BOOTED pid=%d sock=%s" % (p.pid, a.socket), flush=True) + with open(a.pidfile, "w") as f: f.write(str(p.pid)) + if a.wait: + try: p.wait() + except KeyboardInterrupt: p.terminate() + +def do_snapshot(a): + # pause, then Full snapshot + t0 = time.time() + api(a.socket, "PATCH", "/vm", {"state": "Paused"}) + tp = time.time() + api(a.socket, "PUT", "/snapshot/create", + {"snapshot_type": "Full", + "snapshot_path": a.snap_vmstate, + "mem_file_path": a.snap_mem}) + ts = time.time() + print("PAUSE=%.3fs SNAPSHOT_WRITE=%.3fs TOTAL=%.3fs" % (tp - t0, ts - tp, ts - t0), flush=True) + if a.resume_after: + api(a.socket, "PATCH", "/vm", {"state": "Resumed"}) + print("RESUMED-after-snapshot", flush=True) + +def do_restore(a): + # NEW fc process; load snapshot and resume. + p = start_fc(a.socket, a.fclog) + t0 = time.time() + body = { + "snapshot_path": a.snap_vmstate, + "mem_backend": {"backend_type": "File", "backend_path": a.snap_mem}, + "enable_diff_snapshots": False, + "resume_vm": True, + } + api(a.socket, "PUT", "/snapshot/load", body) + t1 = time.time() + print("RESTORE_RESUME=%.3fs pid=%d T1(fc-resume-return)=%.6f" % (t1 - t0, p.pid, t1), flush=True) + with open(a.pidfile, "w") as f: f.write(str(p.pid)) + # emit T0/T1 epochs for external measurement harness + with open(a.tstamp, "w") as f: + f.write("T0 %.6f\nT1 %.6f\n" % (t0, t1)) + if a.wait: + try: p.wait() + except KeyboardInterrupt: p.terminate() + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("mode", choices=["boot", "snapshot", "restore"]) + ap.add_argument("--append", default="") + ap.add_argument("--vcpus", type=int, default=8) + ap.add_argument("--mem", type=int, default=16384) + ap.add_argument("--net", action="store_true") + ap.add_argument("--members", default="") + ap.add_argument("--socket", default="/tmp/fc.sock") + ap.add_argument("--fclog", default="/tmp/fc-console.log") + ap.add_argument("--pidfile", default="/tmp/fc.pid") + ap.add_argument("--snap-vmstate", default="/tmp/snap/osvpg.vmstate") + ap.add_argument("--snap-mem", default="/tmp/snap/osvpg.mem") + ap.add_argument("--tstamp", default="/tmp/fc-restore.tstamp") + ap.add_argument("--wait", action="store_true") + ap.add_argument("--resume-after", action="store_true") + ap.add_argument("--kernel", default="") + ap.add_argument("--rootfs", default="") + a = ap.parse_args() + {"boot": do_boot, "snapshot": do_snapshot, "restore": do_restore}[a.mode](a) + +if __name__ == "__main__": + main() diff --git a/postgres18-musl/fc-snapshot.sh b/postgres18-musl/fc-snapshot.sh new file mode 100755 index 00000000..119c8373 --- /dev/null +++ b/postgres18-musl/fc-snapshot.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# fc-snapshot.sh -- Firecracker full-VM snapshot/restore for the postgres18-musl +# OSv demo. Boots OSv+PostgreSQL once under Firecracker to "ready to accept +# connections", takes a FULL snapshot (guest memory + device/vCPU state), and on +# later launches RESTORES that snapshot so PostgreSQL is ALREADY SERVING -- no +# reboot, no re-initdb, no musl-PIE re-mmap. +# +# Measured on m5d.metal (AL2023, Firecracker v1.7.0, OSv v0.57 PG18.4-musl): +# cold launch -> first external "select 1" = 11.75 s +# restore -> first external "select 1" = 0.24 s (median, ~48x faster) +# FC resume (load+resume) itself = 0.05 s +# The whole PostgreSQL start cost is baked into the snapshotted memory image. +# +# OSv resumes CLEANLY from the snapshot with NO OSv kernel changes required +# (OSv is already Firecracker-snapshot-ready for this workload). See README. +# +# Requirements on the host: +# * bare-metal x86_64 (Firecracker needs /dev/kvm) e.g. *.metal EC2 +# * firecracker v1.7+ on PATH, qemu-img (for the one-time qcow2->raw convert) +# * an OSv postgres18-musl image built with fs=zfs (so the cluster is a real +# ZFS pool that survives snapshot/restore consistently); see README "Build". +# +# Usage: +# fc-snapshot.sh setup # tap net + convert usr.img -> usr.raw +# fc-snapshot.sh boot # boot to "ready", leave running (API sock) +# fc-snapshot.sh snapshot # pause + Full snapshot (mem + vmstate) +# fc-snapshot.sh restore # NEW fc process: load snapshot, resume +# fc-snapshot.sh query # external psql "select 1" against guest +# fc-snapshot.sh measure [N] # restore N times, report first-query time +set -u + +# --- paths (override via env) --- +OSV="${OSV:-$HOME/osv}" # OSv source tree with build/last +BUILD="${BUILD:-$OSV/build/last}" +KERNEL="${KERNEL:-$BUILD/loader-stripped.elf}" +USRIMG="${USRIMG:-$BUILD/usr.img}" # OSv qcow2 rootfs (converted below) +USRRAW="${USRRAW:-$BUILD/usr.raw}" # raw copy Firecracker can read +WORK="${WORK:-/mnt/fc}" # scratch (fast NVMe recommended) +SNAP="$WORK/snap" +POOL="$WORK/pool.raw" # live PG ZFS pool backing file +PRISTINE="$WORK/pool-pristine.raw" # clean post-seed pool +ATSNAP="$WORK/pool-at-snapshot.raw" # pool state at snapshot instant +VMSTATE="$SNAP/osvpg.vmstate" +MEM="$SNAP/osvpg.mem" +SOCK="${SOCK:-/tmp/fc.sock}" +FCLOG="${FCLOG:-/tmp/fc-console.log}" +FC="${FC:-$(command -v firecracker || echo /usr/local/bin/firecracker)}" +SNAPPY="${SNAPPY:-$(dirname "$0")/fc-snap.py}" + +# --- guest network (host tap <-> guest virtio-net) --- +HOST_ENI="${HOST_ENI:-$(ip -o -4 route show default | awk '{print $5; exit}')}" +TAP=fc_tap0; TAP_IP=192.168.100.1; GUEST_IP=192.168.100.2; PGPORT=5432 +export PGCONNECT_TIMEOUT=5 + +# --- PG paths inside the guest --- +PGBIN=/b/.local/pg18/install/bin + +need() { command -v "$1" >/dev/null || { echo "missing: $1"; exit 1; }; } + +do_setup() { + need qemu-img + sudo mkdir -p "$WORK" "$SNAP"; sudo chown -R "$USER" "$WORK" + # one-time: Firecracker only reads RAW; OSv's usr.img is QCOW2. + if [ ! -f "$USRRAW" ] || [ "$USRIMG" -nt "$USRRAW" ]; then + echo "converting $USRIMG (qcow2) -> $USRRAW (raw) for Firecracker" + qemu-img convert -O raw "$USRIMG" "$USRRAW" + fi + # host tap + DNAT so external clients reach the guest PG at host-IP:5432 + if ! ip link show "$TAP" >/dev/null 2>&1; then + sudo ip tuntap add dev "$TAP" mode tap + sudo ip addr add "$TAP_IP/24" dev "$TAP" + fi + sudo ip link set "$TAP" up + sudo sysctl -qw net.ipv4.ip_forward=1 + sudo iptables -t nat -C PREROUTING -i "$HOST_ENI" -p tcp --dport "$PGPORT" -j DNAT --to "$GUEST_IP:$PGPORT" 2>/dev/null \ + || sudo iptables -t nat -A PREROUTING -i "$HOST_ENI" -p tcp --dport "$PGPORT" -j DNAT --to "$GUEST_IP:$PGPORT" + sudo iptables -t nat -C POSTROUTING -s "$GUEST_IP/32" -o "$HOST_ENI" -j MASQUERADE 2>/dev/null \ + || sudo iptables -t nat -A POSTROUTING -s "$GUEST_IP/32" -o "$HOST_ENI" -j MASQUERADE + echo "setup ok: tap=$TAP guest=$GUEST_IP raw=$USRRAW" +} + +# Boot OSv+PG on the pristine pool, wait for "ready", leave FC running. +do_boot() { + [ -f "$PRISTINE" ] || { echo "no pristine pool; run 'seed' first (see README)"; exit 1; } + cp "$PRISTINE" "$POOL" + local serve="/zpool.so import -f -N pgdata ; /zfs.so set sync=disabled pgdata ; /zfs.so mount pgdata ; $PGBIN/postgres -D /data" + sudo -b python3 "$SNAPPY" boot --socket "$SOCK" --fclog "$FCLOG" \ + --append "--rootfs=zfs $serve" --vcpus "${VCPUS:-8}" --mem "${GMEM:-4096}" \ + --members "$POOL" --net --kernel "$KERNEL" --rootfs "$USRRAW" >/dev/null 2>&1 + echo -n "waiting for PG ready" + for i in $(seq 1 240); do + grep -qa "ready to accept" "$FCLOG" 2>/dev/null && { echo " -- READY"; return 0; } + grep -qiE "assert|abort|panic|Failed to load" "$FCLOG" 2>/dev/null && { echo " -- CRASH"; tail -20 "$FCLOG"; exit 1; } + sleep 0.5; echo -n . + done + echo " -- TIMEOUT"; tail -20 "$FCLOG"; exit 1 +} + +do_snapshot() { + rm -f "$VMSTATE" "$MEM" + sudo python3 "$SNAPPY" snapshot --socket "$SOCK" --snap-vmstate "$VMSTATE" --snap-mem "$MEM" + sudo pkill -9 firecracker 2>/dev/null; sleep 1 + cp "$POOL" "$ATSNAP" + echo "snapshot: vmstate=$(du -h "$VMSTATE"|cut -f1) mem=$(du -h "$MEM"|cut -f1) pool-at-snapshot saved" +} + +# Restore into a fresh FC process, resume, leave serving. +# Disk consistency: each restore gets a FRESH COPY of the snapshot-time pool so +# the guest's in-memory ZFS ARC / shared_buffers agree with on-disk state. A +# reflink/overlay (cp --reflink=auto, or an overlay file) is the zero-copy COW +# form; plain cp is the simplest correct equivalent. +do_restore() { + sudo pkill -9 firecracker 2>/dev/null; sleep 0.4 + cp --reflink=auto "$ATSNAP" "$POOL" 2>/dev/null || cp "$ATSNAP" "$POOL" + sudo python3 "$SNAPPY" restore --socket "$SOCK" --members "$POOL" \ + --snap-vmstate "$VMSTATE" --snap-mem "$MEM" --fclog "$FCLOG" \ + --tstamp /tmp/fc-restore.tstamp +} + +do_query() { + psql -h "$GUEST_IP" -U postgres -d postgres -tAc \ + "select 'served-by', inet_server_addr(), version()" 2>&1 | head -1 +} + +do_measure() { + local n="${1:-5}"; : >/tmp/fc-measure.tsv + for r in $(seq 1 "$n"); do + do_restore >/dev/null 2>&1 + local T0=$(awk '/^T0/{print $2}' /tmp/fc-restore.tstamp) + local T2="" + for i in $(seq 1 500); do + psql -h "$GUEST_IP" -U postgres -d postgres -tAc "select 1" 2>/dev/null | grep -q '^1$' \ + && { T2=$(date +%s.%N); break; } + sleep 0.01 + done + [ -z "$T2" ] && { echo "run $r: NO first-query"; continue; } + local fq=$(echo "$T2-$T0"|bc) + echo "run $r: restore->first-query = ${fq}s" + echo "$r $fq" >> /tmp/fc-measure.tsv + done + echo "median: $(awk '{print $2}' /tmp/fc-measure.tsv|sort -n|awk '{a[NR]=$1}END{print a[int((NR+1)/2)]}')s (cold baseline 11.75s)" +} + +case "${1:-}" in + setup) do_setup ;; + boot) do_boot ;; + snapshot) do_snapshot ;; + restore) do_restore ;; + query) do_query ;; + measure) do_measure "${2:-5}" ;; + *) sed -n '2,30p' "$0"; exit 1 ;; +esac From c0d2bad7cd23e5af4eed6aaa981143bc7881e2c8 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 30 Jul 2026 11:30:41 -0400 Subject: [PATCH 3/3] postgres18-musl: persistent warm snapshot checkpoint lifecycle Add a 'lifecycle' subcommand that proves a PERSISTENT WARM checkpoint: a first warm restore (~0.2s to first query), insert rows in the restored session, then FREEZE (a fresh Full snapshot plus a preserved ZFS backing file) followed by a CLEAN Firecracker shutdown (the VM process exits entirely), and finally a SECOND warm restore from the frozen state (~0.2s again). The table, its rows, and the original pg_postmaster_start_time() all survive the freeze/clean-shutdown/warm-restart cycle, so the instance identity and its data are a durable resumable checkpoint. Also add a scripted 'seed' subcommand (build the single-file ZFS pool and cpiod-push an initdb'd cluster into it) with its cpio_push.py helper, and fix the boot argument passing so the leading --rootfs is not mis-parsed by the Firecracker driver argument parser. Measured on m5d.metal with Firecracker v1.7 (4 GiB guest, single-file ZFS pool): both warm launches were about 0.17s to first query and pg_postmaster_start_time() stayed equal to the very first launch across every freeze/thaw cycle. Signed-off-by: Greg Burd --- postgres18-musl/README.md | 44 ++++++++++++++-- postgres18-musl/cpio_push.py | 66 +++++++++++++++++++++++ postgres18-musl/fc-snapshot.sh | 96 +++++++++++++++++++++++++++++++--- 3 files changed, 193 insertions(+), 13 deletions(-) create mode 100644 postgres18-musl/cpio_push.py diff --git a/postgres18-musl/README.md b/postgres18-musl/README.md index 87f968f5..66117724 100644 --- a/postgres18-musl/README.md +++ b/postgres18-musl/README.md @@ -101,9 +101,11 @@ cd apps/postgres18-musl # rootfs to raw (Firecracker only reads RAW block devices). sudo ./fc-snapshot.sh setup -# 2. (seed a single-file ZFS pool with the demo cluster once -- create the pool -# in-guest and cpiod-push the baked cluster into /data, then export; save it -# as pool-pristine.raw. See the prototype orchestrator for the exact steps.) +# 2. Seed a single-file ZFS pool with an initdb'd PG18 cluster (once). Point +# DATAINIT at an initdb'd cluster directory on the host; the pool is created +# in-guest and the cluster is cpiod-pushed into /data, then exported and +# saved as pool-pristine.raw. +DATAINIT=$HOME/data-init sudo -E ./fc-snapshot.sh seed # 3. Boot to "ready", then take a Full snapshot (memory + vmstate): sudo ./fc-snapshot.sh boot @@ -115,6 +117,12 @@ sudo ./fc-snapshot.sh query # select 1 returns immediately # 5. Benchmark restore -> first query (median of N restores): sudo ./fc-snapshot.sh measure 5 + +# 6. Persistent WARM checkpoint lifecycle: first warm launch, insert data in +# the restored session, FREEZE (snapshot-B + preserve pool), CLEANLY shut +# Firecracker down, then a SECOND warm launch from the frozen state. The +# table, its rows, and the original pg_postmaster_start_time() all survive. +sudo ./fc-snapshot.sh lifecycle ``` The snapshot is two files: a tiny **vmstate** (device + vCPU state, ~90 KB) and @@ -124,6 +132,32 @@ mem file small; the memory is mostly zero pages and compresses heavily (a 16 GiB dump gzipped to ~160 MB in testing). Firecracker diff snapshots shrink it further. +### Persistent warm checkpoint (freeze -> clean shutdown -> warm restart) + +A Firecracker Full snapshot is an immutable restore-as-template. To freeze new +work and keep it warm across a full process shutdown you take a FRESH snapshot +after the work and preserve the ZFS backing file at that instant. The +`lifecycle` subcommand demonstrates the whole cycle and times it: + +1. Cold boot once to build the base warm snapshot-A (setup only, not the + headline). +2. FIRST warm launch: restore snapshot-A. Restore to first `select 1` is about + 0.2 s. +3. In the restored session, create a table and insert rows. +4. FREEZE: take snapshot-B (captures the table and rows in guest memory), + preserve the pool backing file, then cleanly kill Firecracker so the VM + process is gone. +5. SECOND warm launch: a fresh Firecracker process loads snapshot-B and + resumes. Restore to first `select 1` is again about 0.2 s. +6. The table, all its rows, and the original `pg_postmaster_start_time()` are + all still there, and PostgreSQL is writable. + +Measured on m5d.metal (Firecracker v1.7, 4 GiB guest, single-file ZFS pool): +both warm launches were about 0.17 s to first query, and +`pg_postmaster_start_time()` stayed equal to the very first launch across every +freeze/clean-shutdown/warm-restart cycle. The instance identity and its data +survive the freeze/thaw. + ### Disk consistency (memory <-> disk must agree) A restored guest's in-memory ZFS ARC, PostgreSQL `shared_buffers`, and dirty @@ -188,10 +222,10 @@ PostgreSQL's on-disk format or SQL behavior. | Deviation | Why | Erase when | |---|---|---| -| `CC=musl-gcc` | OSv's libc is musl | — (intrinsic to the musl target) | +| `CC=musl-gcc` | OSv's libc is musl | n/a (intrinsic to the musl target) | | `--without-icu/zlib/readline/lz4/zstd/libxml` | those libs are not baked into the demo image | those libs are packaged for OSv | | `-DWAIT_USE_SELF_PIPE` | OSv's epoll/signalfd latch path is incomplete; use PostgreSQL's portable self-pipe latch | OSv latch path is complete | -| `LDFLAGS_EX=-pie` | OSv runs applications as PIEs | — (intrinsic to OSv) | +| `LDFLAGS_EX=-pie` | OSv runs applications as PIEs | n/a (intrinsic to OSv) | **Source neuters** (`build.sh`, step 2 -- two one-line `if (0 && ...)` guards): diff --git a/postgres18-musl/cpio_push.py b/postgres18-musl/cpio_push.py new file mode 100644 index 00000000..73cab17a --- /dev/null +++ b/postgres18-musl/cpio_push.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 +# Push a host directory tree into a running OSv cpiod over TCP (cpio-newc). +# Usage: cpio_push.py +# Files are sent with paths relative to src_dir, rooted at '/', so with cpiod +# --prefix /data/ the tree lands under /data/. +import os, sys, socket, stat, time + +src, host, port = sys.argv[1], sys.argv[2], int(sys.argv[3]) + +# connect (retry until cpiod is listening) +s = None +for _ in range(120): + try: + s = socket.create_connection((host, port), timeout=5) + break + except OSError: + time.sleep(1) +if s is None: + sys.exit("could not connect to cpiod at %s:%d" % (host, port)) + +def field(n, l=8): + return ("%.*x" % (l, n)).encode() + +def header(name, mode, size): + nb = name.encode() + return (b"070701" + field(0) + field(mode) + field(0) + field(0) + + field(0) + field(0) + field(size) + field(0) + field(0) + + field(0) + field(0) + field(len(nb)+1) + field(0) + nb + b"\0") + +def send(data): + s.sendall(data) + pad = len(data) % 4 + if pad: + s.sendall(b"\0" * (4 - pad)) + +count = 0 +# dirs first (walk top-down), then files +for root, dirs, files in os.walk(src): + rel = os.path.relpath(root, src) + guest_dir = "/" if rel == "." else "/" + rel + st = os.stat(root) + if guest_dir != "/": + send(header(guest_dir, (st.st_mode & 0o777) | stat.S_IFDIR, 0)) + count += 1 + for f in files: + hp = os.path.join(root, f) + gp = ("/" + f) if rel == "." else ("/" + rel + "/" + f) + if os.path.islink(hp): + link = os.readlink(hp) + send(header(gp, (os.lstat(hp).st_mode & 0o777) | stat.S_IFLNK, len(link))) + send(link.encode()) + else: + sz = os.path.getsize(hp) + send(header(gp, (os.stat(hp).st_mode & 0o777) | stat.S_IFREG, sz)) + with open(hp, "rb") as fh: + send(fh.read()) + count += 1 + +send(header("TRAILER!!!", 0, 0)) +s.shutdown(socket.SHUT_WR) +try: + s.recv(1) # wait for cpiod sync ack +except OSError: + pass +s.close() +print("pushed %d entries from %s" % (count, src)) diff --git a/postgres18-musl/fc-snapshot.sh b/postgres18-musl/fc-snapshot.sh index 119c8373..b0401491 100755 --- a/postgres18-musl/fc-snapshot.sh +++ b/postgres18-musl/fc-snapshot.sh @@ -22,11 +22,14 @@ # # Usage: # fc-snapshot.sh setup # tap net + convert usr.img -> usr.raw +# fc-snapshot.sh seed # build single-file ZFS pool + push cluster # fc-snapshot.sh boot # boot to "ready", leave running (API sock) # fc-snapshot.sh snapshot # pause + Full snapshot (mem + vmstate) # fc-snapshot.sh restore # NEW fc process: load snapshot, resume # fc-snapshot.sh query # external psql "select 1" against guest # fc-snapshot.sh measure [N] # restore N times, report first-query time +# fc-snapshot.sh lifecycle # persistent WARM checkpoint: freeze -> clean +# # shutdown -> warm restart, data + identity survive set -u # --- paths (override via env) --- @@ -85,7 +88,7 @@ do_boot() { cp "$PRISTINE" "$POOL" local serve="/zpool.so import -f -N pgdata ; /zfs.so set sync=disabled pgdata ; /zfs.so mount pgdata ; $PGBIN/postgres -D /data" sudo -b python3 "$SNAPPY" boot --socket "$SOCK" --fclog "$FCLOG" \ - --append "--rootfs=zfs $serve" --vcpus "${VCPUS:-8}" --mem "${GMEM:-4096}" \ + --append="--rootfs=zfs $serve" --vcpus "${VCPUS:-8}" --mem "${GMEM:-4096}" \ --members "$POOL" --net --kernel "$KERNEL" --rootfs "$USRRAW" >/dev/null 2>&1 echo -n "waiting for PG ready" for i in $(seq 1 240); do @@ -141,12 +144,89 @@ do_measure() { echo "median: $(awk '{print $2}' /tmp/fc-measure.tsv|sort -n|awk '{a[NR]=$1}END{print a[int((NR+1)/2)]}')s (cold baseline 11.75s)" } +# seed: build the single-file ZFS pool + push a prebuilt initdb'd cluster into it +# via cpiod, producing $PRISTINE. Needs $DATAINIT to point at an initdb'd cluster +# directory on the host (created once with initdb on any PG18 build). +DATAINIT="${DATAINIT:-$HOME/data-init}" +do_seed() { + [ -d "$DATAINIT" ] || { echo "set DATAINIT to an initdb'd PG18 cluster dir"; exit 1; } + local seedpool="$WORK/pool-seed.raw" + rm -f "$seedpool"; truncate -s 8G "$seedpool" + local create="/zpool.so create -f -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=8k -O logbias=throughput -O primarycache=all -m /data pgdata /dev/vblk1" + local init="$create ; /zfs.so set sync=disabled pgdata ; /tools/cpiod.so --prefix /data/ --port 10000 ; /zpool.so export pgdata" + sudo pkill -9 firecracker 2>/dev/null; sleep 0.5; rm -f "$FCLOG" + sudo -b python3 "$SNAPPY" boot --socket "$SOCK" --fclog "$FCLOG" \ + --append="--rootfs=zfs $init" --vcpus "${VCPUS:-8}" --mem "${GMEM:-4096}" \ + --members "$seedpool" --net --kernel "$KERNEL" --rootfs "$USRRAW" >/dev/null 2>&1 + echo -n "waiting for cpiod" + for i in $(seq 1 90); do + grep -qa "Waiting for connection from host" "$FCLOG" 2>/dev/null && { echo " -- ready"; break; } + sleep 1; echo -n . + done; sleep 1 + python3 "$(dirname "$0")/cpio_push.py" "$DATAINIT" "$GUEST_IP" 10000 2>&1 | sed 's/^/[push] /' | tail -2 + sudo pkill -9 firecracker 2>/dev/null; sleep 1 + cp "$seedpool" "$PRISTINE" + echo "seeded pristine pool: $(du -h "$PRISTINE"|cut -f1)" +} + +# lifecycle: prove a PERSISTENT WARM snapshot checkpoint that survives a freeze, +# clean Firecracker shutdown, and warm restart. Both launches are warm restores +# (about 0.2s to first query). Data inserted in a restored session and the +# original pg_postmaster_start_time() both survive the freeze/clean-shutdown/ +# warm-restart cycle. The cold boot below is ONLY the one-time setup that +# produces the base warm snapshot; it is not the headline path. +lifecycle_first_query() { + for i in $(seq 1 2000); do + psql -h "$GUEST_IP" -U postgres -d postgres -tAc "select 1" 2>/dev/null | grep -q '^1$' && { date +%s.%N; return 0; } + sleep 0.005 + done; return 1 +} +do_lifecycle() { + local P="psql -h $GUEST_IP -U postgres -d postgres -tAc" + local snapB_vm="$SNAP/B.vmstate" snapB_mem="$SNAP/B.mem" poolB="$WORK/pool-B.raw" + echo "## setup: cold boot -> base warm snapshot-A (one time, not the headline)" + do_boot + lifecycle_first_query >/dev/null # wait until PG accepts external TCP, not just the log line + local ppst0=$($P "select pg_postmaster_start_time()") + echo " first-ever pg_postmaster_start_time = $ppst0" + do_snapshot + echo "## FIRST WARM LAUNCH (restore snapshot-A)" + do_restore >/dev/null 2>&1 + local t0=$(awk '/^T0/{print $2}' /tmp/fc-restore.tstamp) t1=$(lifecycle_first_query) + echo " restore->first-query = $(echo "$t1-$t0"|bc)s [warm, the headline]" + echo " pg_postmaster_start_time = $($P "select pg_postmaster_start_time()")" + echo "## create table + rows in the restored session" + $P "drop table if exists t; create table t(id int, note text, ts timestamptz default now())" + for r in 1 2 3 4 5; do $P "insert into t(id,note) values ($r,'row-$r')" >/dev/null; done + psql -h "$GUEST_IP" -U postgres -d postgres -c "select id,note from t order by id" + echo "## FREEZE: snapshot-B (captures table+rows) + preserve pool, then clean shutdown" + $P "checkpoint" >/dev/null 2>&1; rm -f "$snapB_vm" "$snapB_mem" + sudo python3 "$SNAPPY" snapshot --socket "$SOCK" --snap-vmstate "$snapB_vm" --snap-mem "$snapB_mem" + cp "$POOL" "$poolB" + sudo pkill -9 firecracker 2>/dev/null; while pgrep -x firecracker >/dev/null; do sleep 0.02; done + echo " clean shutdown done (VM process gone)" + echo "## SECOND WARM LAUNCH (restore frozen snapshot-B)" + cp --reflink=auto "$poolB" "$POOL" 2>/dev/null || cp "$poolB" "$POOL"; sudo rm -f "$FCLOG" + sudo python3 "$SNAPPY" restore --socket "$SOCK" --members "$POOL" \ + --snap-vmstate "$snapB_vm" --snap-mem "$snapB_mem" --fclog "$FCLOG" --tstamp /tmp/fc-restore.tstamp >/dev/null 2>&1 + local u0=$(awk '/^T0/{print $2}' /tmp/fc-restore.tstamp) u1=$(lifecycle_first_query) + echo " restore->first-query = $(echo "$u1-$u0"|bc)s [warm, the headline]" + echo "## VERIFY persistence + identity after freeze/clean-shutdown/warm-restart" + psql -h "$GUEST_IP" -U postgres -d postgres -c "select id,note from t order by id" + echo " pg_postmaster_start_time = $($P "select pg_postmaster_start_time()") (must equal $ppst0)" + $P "insert into t(id,note) values (99,'written-after-warm-restart')" >/dev/null && echo " writable: inserted id=99" + echo " row count = $($P "select count(*) from t")" + sudo pkill -9 firecracker 2>/dev/null +} + case "${1:-}" in - setup) do_setup ;; - boot) do_boot ;; - snapshot) do_snapshot ;; - restore) do_restore ;; - query) do_query ;; - measure) do_measure "${2:-5}" ;; - *) sed -n '2,30p' "$0"; exit 1 ;; + setup) do_setup ;; + seed) do_seed ;; + boot) do_boot ;; + snapshot) do_snapshot ;; + restore) do_restore ;; + query) do_query ;; + measure) do_measure "${2:-5}" ;; + lifecycle) do_lifecycle ;; + *) sed -n '2,34p' "$0"; exit 1 ;; esac