From 4390a95b086c7573930f8d96608f53ef70f50bac Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sun, 12 Jul 2026 11:50:50 -0400 Subject: [PATCH 1/9] libext: real fsync + page-cache bridge for ext2/3/4 Two gaps in the ext (lwext4) filesystem module for the Postgres-over-ext goal. 1. fsync durability. ext mounts with lwext4 block-cache write-back enabled, so a write only reaches the disk when the block cache is flushed -- but vop_fsync was vop_nullop, so fsync(2)/fdatasync(2) on an ext file persisted nothing. Implement ext_fsync() to flush the device's block cache (like ext_sync does at unmount), making written data durable. The cache is shared per device, so this persists the file along with any other dirty buffers, which is correct if slightly more than the theoretical per-inode minimum. 2. Page-cache bridge (vop_cache). The vop_cache slot was null, so mmap faults on ext files went through the block layer on every fault, unlike ROFS and ZFS which populate the shared page cache. Add ext_map_cached_page(): on a VOP_CACHE call it reads one page-aligned page of file data into a freshly allocated page and hands it to pagecache::map_read_cached_page(), warming the read cache so subsequent faults and readahead are served from it. This is an allocate-and-copy bridge (lwext4's block-cache buffers are not page-aligned/shareable the way ROFS's read-around cache is); a zero-copy borrow-and-pin bridge like the ZFS ARC one can follow if it shows up hot. Because the module is built with -fno-rtti and pulls in (which uses typeid), the two page-cache symbols we need are declared minimally instead of including the heavy header (the uio carries the hashkey opaquely, so its layout is never needed). Add tests/tst-ext4-rw.cc: mmap a pre-populated ext4 file and verify the pattern survives the vop_cache bridge (first fault + cached re-read), then write a file, fsync it, and read it back (plus fdatasync and a fresh re-open). Verified on OSv under KVM with an ext4 second disk (created with mkfs.ext4 -b 4096 -O ^64bit,^metadata_csum, which lwext4 supports). Known pre-existing limitation (not introduced here, out of scope for this PR): libext's inode-delete path does not set the inode dtime, so Linux e2fsck flags "deleted inode has zero dtime" on a disk after OSv deletes a file. A fresh disk that OSv only reads/writes+fsyncs (no delete) fscks clean. Tracked as a follow-up in the ext write-path correctness work. --- .gitignore | 1 + modules/libext/ext_vnops.cc | 89 ++++++++++++++++++++++++++++++++++-- modules/tests/Makefile | 1 + tests/tst-ext4-rw.cc | 90 +++++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 tests/tst-ext4-rw.cc diff --git a/.gitignore b/.gitignore index 8ba40c5079..25e538de1b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ modules/dl_tests/usr.manifest compile_commands.json downloaded_packages .firecracker +ext_images/ diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index 063fe1e465..9c41e796ba 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -19,6 +19,18 @@ //The libext does not implement journal (we can integrate it later and make it optional) //nor xattr which is not even supported by OSv VFS layer. +// The libext module is built with -fno-rtti, but pulls in +// -> which uses typeid. We only need a couple of +// symbols from the page cache to warm it in ext_map_cached_page(), so declare +// them minimally here instead of including the heavy headers. The uio carries +// the pagecache hashkey opaquely, so we never need its layout. +#include // memory::alloc_page / free_page (light header) +namespace pagecache { + struct hashkey; + void map_read_cached_page(hashkey *key, void *page); +} +static const unsigned long EXT_PAGE_SIZE = 4096; + extern "C" { #define USE_C_INTERFACE 1 #include @@ -37,6 +49,7 @@ void free_contiguous_aligned(void* p); #include #include #include +#include #include #include @@ -596,7 +609,21 @@ ext_ioctl(vnode_t *vp, file_t *fp, u_long com, void *data) return (EINVAL); } -#define ext_fsync ((vnop_fsync_t)vop_nullop) +static int +ext_fsync(struct vnode *vp, struct file *fp) +{ + // lwext4 mounts with block-cache write-back enabled (ext_vfsops.cc), so a + // write only reaches the disk when the block cache is flushed. fsync(2)/ + // fdatasync(2) must make the file's data durable, so flush the device's + // block cache here (the cache is shared per device, so this persists this + // file's dirty buffers along with any others -- correct, if slightly more + // than the minimum a per-inode flush would do). + struct ext4_fs *fs = (struct ext4_fs *)vp->v_mount->m_data; + fs->bcache_lock(); + int r = ext4_block_cache_flush(fs->bdev); + fs->bcache_unlock(); + return r; +} static int ext_readdir(struct vnode *dvp, struct file *fp, struct dirent *dir) @@ -1425,7 +1452,63 @@ ext_inactive(vnode_t *vp) return EOK; } -#define ext_arc ((vnop_cache_t)nullptr) +// vop_cache: warm the shared page cache with one page of file data so mmap +// faults (and readahead) on ext files can be served from the page cache like +// ROFS and ZFS, rather than each fault re-reading through the block layer. +// +// The uio's iov_base carries the pagecache hashkey; we read exactly one +// page-sized, page-aligned chunk at uio_offset into a freshly allocated page +// and hand it to pagecache::map_read_cached_page(), which takes ownership. +// +// ponytail: allocate-and-copy from lwext4, not a zero-copy borrow-and-pin. +// lwext4's block cache buffers are not page-aligned/shareable the way ROFS's +// read-around cache is, so copying is the correct first step; a borrow-and-pin +// bridge like the ZFS ARC one can come later if it shows up hot. +static int +ext_map_cached_page(struct vnode *vp, struct file *fp, struct uio *uio) +{ + if (vp->v_type == VDIR) + return EISDIR; + if (vp->v_type != VREG) + return EINVAL; + if (uio->uio_offset < 0) + return EINVAL; + if (uio->uio_offset >= (off_t)vp->v_size) + return 0; + if (uio->uio_resid != EXT_PAGE_SIZE) + return EINVAL; + if (uio->uio_offset % EXT_PAGE_SIZE) + return EINVAL; + + struct ext4_fs *fs = (struct ext4_fs *)vp->v_mount->m_data; + auto_inode_ref inode_ref(fs, vp->v_ino); + if (inode_ref._r != EOK) { + return inode_ref._r; + } + + void *page = memory::alloc_page(); + if (!page) { + return ENOMEM; + } + // Read up to a page; if the file's last page is short, the tail stays zero + // (alloc_page does not zero, so zero it first to avoid leaking stale data). + memset(page, 0, EXT_PAGE_SIZE); + uint64_t fsize = ext4_inode_get_size(&fs->sb, inode_ref._ref.inode); + size_t want = std::min((uint64_t)EXT_PAGE_SIZE, + fsize - (uint64_t)uio->uio_offset); + size_t got = 0; + int ret = ext_internal_read(fs, &inode_ref._ref, uio->uio_offset, page, + want, &got); + if (ret) { + memory::free_page(page); + return ret; + } + pagecache::map_read_cached_page((pagecache::hashkey *)uio->uio_iov->iov_base, + page); + uio->uio_resid = 0; + return 0; +} + #define ext_seek ((vnop_seek_t)vop_nullop) struct vnops ext_vnops = { @@ -1448,7 +1531,7 @@ struct vnops ext_vnops = { ext_inactive, /* inactive */ ext_truncate, /* truncate */ ext_link, /* link */ - ext_arc, /* arc */ + ext_map_cached_page, /* cache (vop_cache) */ ext_fallocate, /* fallocate */ ext_readlink, /* read link */ ext_symlink, /* symbolic link */ diff --git a/modules/tests/Makefile b/modules/tests/Makefile index ff66fcd931..1de957a365 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -124,6 +124,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. tst-pthread-timedlock.so \ tst-signal-fills.so \ tst-epoll-pwait2.so \ + tst-ext4-rw.so \ tst-prctl.so \ tst-mmap-file-cow.so \ tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \ diff --git a/tests/tst-ext4-rw.cc b/tests/tst-ext4-rw.cc new file mode 100644 index 0000000000..23da14a6f9 --- /dev/null +++ b/tests/tst-ext4-rw.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +// Exercises the ext4 (libext) fsync durability fix and the vop_cache pagecache +// bridge. Boot with an ext4 second disk mounted at /data, e.g.: +// +// scripts/run.py --second-disk-image ./ext_images/ext4.img \ +// --execute='--mount-fs=ext,/dev/vblk1,/data tests/tst-ext4-rw.so' +// +// The disk is expected to contain /data/readme.dat: 12288 bytes of the pattern +// (i*7+3)&0xff. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static const char *DATA = "/data"; + +int main() +{ + std::cerr << "Running ext4-rw tests\n"; + + // ---- vop_cache bridge: mmap a pre-populated ext4 file and verify it ---- + std::string readme = std::string(DATA) + "/readme.dat"; + int fd = open(readme.c_str(), O_RDONLY); + assert(fd >= 0); + struct stat st; + assert(fstat(fd, &st) == 0); + const size_t N = 12288; // 3 pages, known pattern + assert((size_t)st.st_size == N); + + // MAP_SHARED read mmap: the fault path calls VOP_CACHE (ext_map_cached_page) + // to warm the page cache, then serves the page. Read all three pages and + // verify the known pattern survives the bridge intact. + unsigned char *p = (unsigned char *)mmap(nullptr, N, PROT_READ, MAP_SHARED, fd, 0); + assert(p != MAP_FAILED); + for (size_t i = 0; i < N; i++) { + assert(p[i] == (unsigned char)((i * 7 + 3) & 0xff)); + } + // Read again (now served from the page cache) - still correct. + for (size_t i = 0; i < N; i += 512) { + assert(p[i] == (unsigned char)((i * 7 + 3) & 0xff)); + } + assert(munmap(p, N) == 0); + close(fd); + + // ---- fsync durability: write a file, fsync it, read it back ---- + std::string out = std::string(DATA) + "/written.dat"; + fd = open(out.c_str(), O_CREAT | O_TRUNC | O_RDWR, 0644); + assert(fd >= 0); + const size_t W = 8000; + std::string data(W, 0); + for (size_t i = 0; i < W; i++) { + data[i] = (char)((i * 11 + 5) & 0xff); + } + assert(write(fd, data.data(), W) == (ssize_t)W); + // fsync must now be a real flush (was a no-op before this change). It must + // return 0 and not error. + assert(fsync(fd) == 0); + // Read it back and verify. + std::string check(W, 0); + assert(pread(fd, &check[0], W, 0) == (ssize_t)W); + assert(check == data); + // fdatasync also works. + assert(write(fd, data.data(), 100) == 100); + assert(fdatasync(fd) == 0); + close(fd); + + // Re-open and verify the fsync'd data is present. + fd = open(out.c_str(), O_RDONLY); + assert(fd >= 0); + assert(pread(fd, &check[0], W, 0) == (ssize_t)W); + assert(check == data); + close(fd); + unlink(out.c_str()); + + std::cerr << "ext4-rw tests PASSED\n"; + return 0; +} From cea5adf95721c415de007b42c5391ead6fdb8d0e Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sun, 12 Jul 2026 12:11:25 -0400 Subject: [PATCH 2/9] libext: set inode deletion time so deleted files fsck clean Follow-up to the fsync/page-cache work: libext's inode-delete path freed the inode from the bitmap but never set its on-disk deletion time (dtime), so after OSv created and deleted a file, Linux e2fsck flagged "Deleted inode NN has zero dtime" and reported the filesystem as still having errors. lwext4's own delete path marks the inode with ext4_inode_set_del_time(inode, -1L) before ext4_fs_free_inode(), but that symbol is not exported from liblwext4.so. Add a small ext_mark_inode_deleted() helper that sets inode->deletion_time = 0xffffffff directly (byte-order invariant, so no to_le32() needed) and call it before each ext4_fs_free_inode() in the module (unlink, rmdir, delete-on-last-close, delete-outstanding-on-unmount, and the dir_link allocation-rollback path). Verified: after OSv boots an ext4 second disk, mmaps/reads a file, writes and fsyncs another, then deletes it, `e2fsck -n -f` on the disk now exits 0 (clean), where before it reported the zero-dtime error. tst-ext4-rw still passes. --- modules/libext/ext_vnops.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index 9c41e796ba..ba4a827294 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -68,6 +68,16 @@ void free_contiguous_aligned(void* p); #define ext_debug(...) #endif +// Mark an inode as deleted with a non-zero deletion time before it is freed, so +// Linux e2fsck does not flag "deleted inode has zero dtime". lwext4's own +// delete path uses ext4_inode_set_del_time(inode, -1L), but that symbol is not +// exported from liblwext4.so, so set the field directly. 0xffffffff is +// byte-order invariant, so no to_le32() is needed. +static inline void ext_mark_inode_deleted(struct ext4_inode *inode) +{ + inode->deletion_time = 0xffffffff; +} + //Simple RAII struct to automate release of i-node reference //when it goes out of scope. struct auto_inode_ref { @@ -168,6 +178,10 @@ void ext_delete_outstanding_inodes(struct ext4_fs *fs) if (inode._r != EOK) { continue; } + // Set a non-zero deletion time before freeing so Linux e2fsck does not + // flag "deleted inode has zero dtime" (lwext4's own delete path does the + // same with -1L). + ext_mark_inode_deleted(inode._ref.inode); ext4_fs_free_inode(&inode._ref); ext_debug("delete: i-node=%ld when unmounting\n", inode_no); } @@ -195,6 +209,7 @@ ext_close(vnode_t *vp, file_t *fp) if (inode._r != EOK) { return inode._r; } + ext_mark_inode_deleted(inode._ref.inode); ext4_fs_free_inode(&inode._ref); remove_inode_from_being_deleted(vp->v_ino); @@ -885,6 +900,7 @@ ext_dir_link(struct vnode *dvp, char *name, int file_type, mode_t mode, uint32_t ext_debug("dir_link: created %s under i-node=%li with filetype:%d\n", name, dvp->v_ino, file_type); } else { if (!inode_no) { + ext_mark_inode_deleted(child_ref.inode); ext4_fs_free_inode(&child_ref); } //We do not want to write new inode. But block has to be released. @@ -1065,6 +1081,7 @@ ext_dir_remove_entry(struct vnode *dvp, struct vnode *vp, char *name) if (links_cnt == 1) {//Zero now if (vdata->ref_count == 0) { + ext_mark_inode_deleted(child._ref.inode); ext4_fs_free_inode(&child._ref); } else { ext_debug("dir_remove_entry: should remove i-node=%ld of %s on last close\n", vp->v_ino, name); @@ -1075,6 +1092,7 @@ ext_dir_remove_entry(struct vnode *dvp, struct vnode *vp, char *name) } } else { if (vdata->ref_count == 0) { + ext_mark_inode_deleted(child._ref.inode); ext4_fs_free_inode(&child._ref); } else { ext_debug("dir_remove_entry: should remove i-node=%ld of %s on last close\n", vp->v_ino, name); From 920f58894cc8eee626a29b910622a39dbe83a6f2 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 16:03:17 -0400 Subject: [PATCH 3/9] tests: add ext4 read/write throughput benchmark (D2.3) A filesystem-agnostic micro-benchmark (tst-ext4-bench) measuring sequential write+fsync, sequential read, and 4K random read throughput in MB/s, so the ext4/libext path can be compared A/B against raw virtio-blk and against Linux ext4 on the same storage. Point its argument at any mount (ext, zfs, rofs) for a cross-filesystem comparison. --- modules/tests/Makefile | 1 + tests/tst-ext4-bench.cc | 124 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 tests/tst-ext4-bench.cc diff --git a/modules/tests/Makefile b/modules/tests/Makefile index 1de957a365..ac5fb42714 100644 --- a/modules/tests/Makefile +++ b/modules/tests/Makefile @@ -125,6 +125,7 @@ tests := tst-pthread.so misc-ramdisk.so tst-vblk.so tst-mq-smoke.so tst-bsd-evh. tst-signal-fills.so \ tst-epoll-pwait2.so \ tst-ext4-rw.so \ + tst-ext4-bench.so \ tst-prctl.so \ tst-mmap-file-cow.so \ tst-elf-permissions.so misc-mutex.so misc-sockets.so tst-condvar.so \ diff --git a/tests/tst-ext4-bench.cc b/tests/tst-ext4-bench.cc new file mode 100644 index 0000000000..c8c2115e79 --- /dev/null +++ b/tests/tst-ext4-bench.cc @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2026 Waldemar Kozaczuk + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ + +// ext4 (libext/lwext4) read/write throughput micro-benchmark for the D2.3 A/B. +// +// Measures sequential write, sequential read (cold + warm), and random read +// throughput against a file on a mounted filesystem, reporting MB/s so the +// ext4 path can be compared to raw virtio-blk and to Linux ext4 on the same +// storage. It is filesystem-agnostic: point --dir at an ext mount for the ext4 +// numbers, or at a zfs/rofs mount for a cross-fs comparison. +// +// Boot (ext4 second disk mounted at /data): +// scripts/run.py --second-disk-image ./ext_images/ext4.img +// --execute='--mount-fs=ext,/dev/vblk1,/data tests/tst-ext4-bench.so /data' + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using clk = std::chrono::steady_clock; + +static double secs(clk::time_point a, clk::time_point b) +{ + return std::chrono::duration(b - a).count(); +} + +// Write `total` bytes to `path` in `bs`-sized chunks; return MB/s. +static double bench_write(const char *path, size_t total, size_t bs, bool do_fsync) +{ + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { perror("open(write)"); return -1; } + std::vector buf(bs); + for (size_t i = 0; i < bs; i++) buf[i] = (char)((i * 7 + 3) & 0xff); + + auto t0 = clk::now(); + size_t written = 0; + while (written < total) { + ssize_t n = write(fd, buf.data(), bs); + if (n != (ssize_t)bs) { perror("write"); close(fd); return -1; } + written += n; + } + if (do_fsync && fsync(fd) != 0) { perror("fsync"); } + auto t1 = clk::now(); + close(fd); + return (total / (1024.0 * 1024.0)) / secs(t0, t1); +} + +// Sequential read of `total` bytes in `bs` chunks; return MB/s. +static double bench_read_seq(const char *path, size_t total, size_t bs) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) { perror("open(read)"); return -1; } + std::vector buf(bs); + auto t0 = clk::now(); + size_t rd = 0; + while (rd < total) { + ssize_t n = read(fd, buf.data(), bs); + if (n <= 0) break; + rd += n; + } + auto t1 = clk::now(); + close(fd); + return (rd / (1024.0 * 1024.0)) / secs(t0, t1); +} + +// Random read: `count` reads of `bs` at random block offsets; return MB/s. +static double bench_read_rand(const char *path, size_t file_size, size_t bs, int count) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) { perror("open(rand)"); return -1; } + std::vector buf(bs); + std::mt19937_64 rng(12345); + size_t nblocks = file_size / bs; + std::uniform_int_distribution dist(0, nblocks ? nblocks - 1 : 0); + + auto t0 = clk::now(); + size_t rd = 0; + for (int i = 0; i < count; i++) { + off_t off = (off_t)dist(rng) * bs; + ssize_t n = pread(fd, buf.data(), bs, off); + if (n <= 0) break; + rd += n; + } + auto t1 = clk::now(); + close(fd); + return (rd / (1024.0 * 1024.0)) / secs(t0, t1); +} + +int main(int argc, char **argv) +{ + const char *dir = (argc > 1) ? argv[1] : "/data"; + // Sizes: modest by default so it runs on small images; override via argv. + size_t total = (argc > 2) ? strtoull(argv[2], nullptr, 0) : (64UL << 20); // 64 MiB + size_t bs = (argc > 3) ? strtoull(argv[3], nullptr, 0) : (128UL << 10); // 128 KiB + + char path[512]; + snprintf(path, sizeof(path), "%s/bench.dat", dir); + + fprintf(stderr, "ext4-bench: dir=%s total=%zuMiB bs=%zuKiB\n", + dir, total >> 20, bs >> 10); + + double w = bench_write(path, total, bs, /*fsync=*/true); + double rs = bench_read_seq(path, total, bs); // warm (just written) + double rr = bench_read_rand(path, total, 4096, 4096); // 4K random, 4096 ops + + fprintf(stderr, "RESULT seq_write_fsync %.1f MB/s\n", w); + fprintf(stderr, "RESULT seq_read %.1f MB/s\n", rs); + fprintf(stderr, "RESULT rand_read_4k %.1f MB/s\n", rr); + + unlink(path); + fprintf(stderr, "ext4-bench done\n"); + return (w > 0 && rs > 0 && rr > 0) ? 0 : 1; +} From ea71139bfe619bb02fe6e01fe75968f2f173e222 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 16:57:55 -0400 Subject: [PATCH 4/9] libext: read/write directly into user buffer, skip bounce copy ext_read allocated a full-size aligned bounce buffer, had lwext4 read into it, then uiomove()'d it to the caller - a per-read malloc plus a second full-size memcpy on every read; ext_write mirrored this in reverse. On the common path (a single contiguous iovec) hand lwext4 the caller's buffer directly. This removes the allocation and one memcpy per read/write and fixes a latent free()/free_contiguous_aligned() mismatch in both error paths. It is a correctness/cleanup change: an A/B on local NVMe showed throughput unchanged (the memcpy is not the bottleneck), which confirms the sequential-read gap vs Linux is the absence of async read-ahead, not copy overhead - that is a separate, larger change (see the ext4 perf notes). --- modules/libext/ext_vnops.cc | 44 +++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index ba4a827294..cd531d0c43 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -363,14 +363,38 @@ ext_read(vnode_t *vp, struct file *fp, uio_t *uio, int ioflag) // Total read amount is what they requested, or what is left uint64_t fsize = ext4_inode_get_size(&fs->sb, inode_ref._ref.inode); + if ((uint64_t)uio->uio_offset >= fsize) { + return 0; + } uint64_t read_amt = std::min(fsize - uio->uio_offset, (uint64_t)uio->uio_resid); + + // Fast path: a single contiguous user iovec lets lwext4 read straight into + // the caller's buffer, avoiding a full-size bounce allocation and the + // extra uiomove() copy that dominated the old read path. + if (uio->uio_iovcnt == 1 && uio->uio_iov[0].iov_len >= read_amt) { + size_t read_count = 0; + int ret = ext_internal_read(fs, &inode_ref._ref, uio->uio_offset, + uio->uio_iov[0].iov_base, read_amt, &read_count); + if (ret) { + kprintf("[ext_read] Error reading data\n"); + return ret; + } + // Advance the uio to reflect what we consumed (mirrors uiomove). + uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_count; + uio->uio_iov[0].iov_len -= read_count; + uio->uio_resid -= read_count; + uio->uio_offset += read_count; + return 0; + } + + // Slow path (scatter/gather or short iovec): bounce through a temp buffer. void *buf = alloc_contiguous_aligned(read_amt, alignof(std::max_align_t)); size_t read_count = 0; int ret = ext_internal_read(fs, &inode_ref._ref, uio->uio_offset, buf, read_amt, &read_count); if (ret) { kprintf("[ext_read] Error reading data\n"); - free(buf); + free_contiguous_aligned(buf); return ret; } @@ -597,12 +621,28 @@ ext_write(vnode_t *vp, uio_t *uio, int ioflag) ext_debug("write: %ld bytes at offset:%ld to file i-node=%ld\n", uio->uio_resid, uio->uio_offset, vp->v_ino); + // Fast path: a single contiguous user iovec lets lwext4 write straight from + // the caller's buffer, avoiding a full-size bounce allocation and the extra + // uiomove() copy. + if (uio->uio_iovcnt == 1 && (size_t)uio->uio_iov[0].iov_len >= (size_t)uio->uio_resid) { + size_t want = uio->uio_resid; + size_t write_count = 0; + int ret = ext_internal_write(fs, &inode_ref._ref, uio->uio_offset, + uio->uio_iov[0].iov_base, want, &write_count); + uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + write_count; + uio->uio_iov[0].iov_len -= write_count; + uio->uio_resid -= write_count; + uio->uio_offset += write_count; + vp->v_size = ext4_inode_get_size(&fs->sb, inode_ref._ref.inode); + return ret; + } + uio_t uio_copy = *uio; void *buf = alloc_contiguous_aligned(uio->uio_resid, alignof(std::max_align_t)); int ret = uiomove(buf, uio->uio_resid, &uio_copy); if (ret) { kprintf("[ext_write] Error copying data\n"); - free(buf); + free_contiguous_aligned(buf); return ret; } From e2c21705834f37f44e1924ad16d5250c3e273056 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 17:19:57 -0400 Subject: [PATCH 5/9] libext: sequential read-ahead cache for ext4 reads The local-NVMe A/B showed ext4 sequential reads at ~33% of Linux while writes were at parity and random reads ~86%. The gap is that ext_read issues one synchronous bio per read() with no prefetch (ext4_blocks_get_direct bypasses lwext4's block cache), so a stream of small sequential reads cannot keep the device pipeline busy - larger reads were measurably faster purely from amortizing the per-read() round-trip. Add a per-vnode sequential read-ahead cache (in ext_vdata): on a detected sequential single-iovec read smaller than the window, fill a 1 MiB window from disk once and serve that read and subsequent in-window reads from it (a memcpy, no I/O). This turns N small synchronous reads into 1 large read + N copies. The window is invalidated on any write to the file and freed when the vnode goes inactive; access is serialized by a per-vnode mutex. tst-ext4-bench now writes an absolute-offset pattern and verifies every byte on the sequential read, so the benchmark doubles as a read-ahead correctness test (no VERIFY FAIL = the cache returns correct data across window boundaries). --- modules/libext/ext_vnops.cc | 88 +++++++++++++++++++++++++++++++++++++ tests/tst-ext4-bench.cc | 24 ++++++++-- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index cd531d0c43..ea51799eb0 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -40,6 +40,7 @@ extern "C" { #include #include #include +#include void* alloc_contiguous_aligned(size_t size, size_t align); void free_contiguous_aligned(void* p); @@ -98,9 +99,32 @@ struct ext_vdata { int ref_count; bool delete_on_last_close; + // Sequential read-ahead cache. ext4_blocks_get_direct issues one + // synchronous bio per read() with no prefetch, so many small sequential + // reads can't keep the device busy (the seq-read gap vs Linux). When a + // sequential pattern is detected we read a large window once and serve the + // following reads from this buffer (a memcpy, no I/O). Guarded by ra_lock; + // invalidated on any write to the file. + mutex_t ra_lock; + char *ra_buf; // owned, alloc_contiguous_aligned; nullptr = empty + uint64_t ra_start; // file offset of ra_buf[0] + size_t ra_len; // valid bytes in ra_buf + uint64_t ra_next; // expected offset of the next sequential read + ext_vdata() { ref_count = 0; delete_on_last_close = false; + mutex_init(&ra_lock); + ra_buf = nullptr; + ra_start = 0; + ra_len = 0; + ra_next = 0; + } + + ~ext_vdata() { + if (ra_buf) { + free_contiguous_aligned(ra_buf); + } } }; @@ -368,6 +392,59 @@ ext_read(vnode_t *vp, struct file *fp, uio_t *uio, int ioflag) } uint64_t read_amt = std::min(fsize - uio->uio_offset, (uint64_t)uio->uio_resid); + // Sequential read-ahead: for a single-iovec read smaller than the window, + // consult (and populate) a per-vnode read-ahead buffer. On a sequential + // access pattern this turns many small synchronous per-read() bios into one + // large read plus cheap memcpys, which is the dominant cost for sequential + // reads (a single small read() cannot keep the device pipeline busy). + static const size_t RA_WINDOW = 1 << 20; // 1 MiB read-ahead window + ext_vdata *vdata = (ext_vdata *)vp->v_data; + if (vdata && uio->uio_iovcnt == 1 && read_amt <= RA_WINDOW / 2) { + uint64_t off = uio->uio_offset; + mutex_lock(&vdata->ra_lock); + // Serve from the cached window if the request lies fully inside it. + if (vdata->ra_buf && off >= vdata->ra_start && + off + read_amt <= vdata->ra_start + vdata->ra_len) { + memcpy(uio->uio_iov[0].iov_base, + vdata->ra_buf + (off - vdata->ra_start), read_amt); + vdata->ra_next = off + read_amt; + mutex_unlock(&vdata->ra_lock); + uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; + uio->uio_iov[0].iov_len -= read_amt; + uio->uio_resid -= read_amt; + uio->uio_offset += read_amt; + return 0; + } + // Miss. If this looks sequential (or is the first read), fill the + // window from disk once and serve from it. + if (off == vdata->ra_next || vdata->ra_buf == nullptr) { + size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - off); + if (!vdata->ra_buf) { + vdata->ra_buf = (char *)alloc_contiguous_aligned( + RA_WINDOW, alignof(std::max_align_t)); + } + if (vdata->ra_buf) { + size_t got = 0; + int rr = ext_internal_read(fs, &inode_ref._ref, off, + vdata->ra_buf, win, &got); + if (rr == 0 && got >= read_amt) { + vdata->ra_start = off; + vdata->ra_len = got; + memcpy(uio->uio_iov[0].iov_base, vdata->ra_buf, read_amt); + vdata->ra_next = off + read_amt; + mutex_unlock(&vdata->ra_lock); + uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; + uio->uio_iov[0].iov_len -= read_amt; + uio->uio_resid -= read_amt; + uio->uio_offset += read_amt; + return 0; + } + } + } + vdata->ra_next = off + read_amt; // track for next-time detection + mutex_unlock(&vdata->ra_lock); + } + // Fast path: a single contiguous user iovec lets lwext4 read straight into // the caller's buffer, avoiding a full-size bounce allocation and the // extra uiomove() copy that dominated the old read path. @@ -615,6 +692,17 @@ ext_write(vnode_t *vp, uio_t *uio, int ioflag) return inode_ref._r; } + // A write invalidates any cached read-ahead window for this file. + { + ext_vdata *vdata = (ext_vdata *)vp->v_data; + if (vdata) { + mutex_lock(&vdata->ra_lock); + vdata->ra_len = 0; + vdata->ra_next = 0; + mutex_unlock(&vdata->ra_lock); + } + } + if (ioflag & IO_APPEND) { uio->uio_offset = ext4_inode_get_size(&fs->sb, inode_ref._ref.inode); } diff --git a/tests/tst-ext4-bench.cc b/tests/tst-ext4-bench.cc index c8c2115e79..5828c974c5 100644 --- a/tests/tst-ext4-bench.cc +++ b/tests/tst-ext4-bench.cc @@ -41,11 +41,14 @@ static double bench_write(const char *path, size_t total, size_t bs, bool do_fsy int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (fd < 0) { perror("open(write)"); return -1; } std::vector buf(bs); - for (size_t i = 0; i < bs; i++) buf[i] = (char)((i * 7 + 3) & 0xff); auto t0 = clk::now(); size_t written = 0; while (written < total) { + // Absolute-offset pattern so a sequential read can verify integrity + // regardless of chunk size / read-ahead window boundaries. + for (size_t i = 0; i < bs; i++) + buf[i] = (char)(((written + i) * 7 + 3) & 0xff); ssize_t n = write(fd, buf.data(), bs); if (n != (ssize_t)bs) { perror("write"); close(fd); return -1; } written += n; @@ -56,8 +59,10 @@ static double bench_write(const char *path, size_t total, size_t bs, bool do_fsy return (total / (1024.0 * 1024.0)) / secs(t0, t1); } -// Sequential read of `total` bytes in `bs` chunks; return MB/s. -static double bench_read_seq(const char *path, size_t total, size_t bs) +// Sequential read of `total` bytes in `bs` chunks; return MB/s. If verify is +// set, checks each byte matches the write pattern (i*7+3)&0xff, proving the +// read path (incl. read-ahead) returns correct data. +static double bench_read_seq(const char *path, size_t total, size_t bs, bool verify = false) { int fd = open(path, O_RDONLY); if (fd < 0) { perror("open(read)"); return -1; } @@ -67,6 +72,17 @@ static double bench_read_seq(const char *path, size_t total, size_t bs) while (rd < total) { ssize_t n = read(fd, buf.data(), bs); if (n <= 0) break; + if (verify) { + for (ssize_t i = 0; i < n; i++) { + char expect = (char)(((rd + i) * 7 + 3) & 0xff); + if (buf[i] != expect) { + fprintf(stderr, "VERIFY FAIL at byte %zu: got %d want %d\n", + rd + i, (unsigned char)buf[i], (unsigned char)expect); + close(fd); + return -2; + } + } + } rd += n; } auto t1 = clk::now(); @@ -111,7 +127,7 @@ int main(int argc, char **argv) dir, total >> 20, bs >> 10); double w = bench_write(path, total, bs, /*fsync=*/true); - double rs = bench_read_seq(path, total, bs); // warm (just written) + double rs = bench_read_seq(path, total, bs, /*verify=*/true); // warm + integrity check double rr = bench_read_rand(path, total, 4096, 4096); // 4K random, 4096 ops fprintf(stderr, "RESULT seq_write_fsync %.1f MB/s\n", w); From a6c33cf7f0b1d4ab5a5fe1818e5ee1c37df7636a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 13 Jul 2026 17:51:23 -0400 Subject: [PATCH 6/9] libext: async double-buffered read-ahead for ext4 sequential reads The synchronous 1 MiB read-ahead (6a7990d7) reads a window, lets the app consume it, then reads the next window - so the NVMe queue goes idle during the app's compute. Linux keeps the device busy via async prefetch. Add double buffering: two 1 MiB windows (cur + next) and a per-vnode worker pthread. While the app consumes cur (served by memcpy), the worker prefetches next in the background. When a read crosses out of cur, next is promoted to cur and the following window is kicked off. On a sequential pattern the next window is already in memory, keeping the device queue full and hiding read latency behind compute. Correctness: writes and truncation cancel any in-flight prefetch and drop both windows (ext_ra_invalidate); the worker uses its own inode_ref (lwext4 locks its block cache internally); ~ext_vdata joins the worker and frees both buffers on vnode inactive. The tst-ext4-bench byte-for-byte verify passes (no VERIFY FAIL) for 64K/128K/256K reads. --- modules/libext/ext_vnops.cc | 373 +++++++++++++++++++++++++++++++----- 1 file changed, 324 insertions(+), 49 deletions(-) diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index ea51799eb0..2c3a98c74b 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -58,6 +58,7 @@ void free_contiguous_aligned(void* p); #include #include #include +#include #include #include @@ -99,31 +100,102 @@ struct ext_vdata { int ref_count; bool delete_on_last_close; - // Sequential read-ahead cache. ext4_blocks_get_direct issues one + // Double-buffered ASYNC sequential read-ahead. lwext4's read path issues one // synchronous bio per read() with no prefetch, so many small sequential - // reads can't keep the device busy (the seq-read gap vs Linux). When a - // sequential pattern is detected we read a large window once and serve the - // following reads from this buffer (a memcpy, no I/O). Guarded by ra_lock; - // invalidated on any write to the file. - mutex_t ra_lock; - char *ra_buf; // owned, alloc_contiguous_aligned; nullptr = empty - uint64_t ra_start; // file offset of ra_buf[0] - size_t ra_len; // valid bytes in ra_buf - uint64_t ra_next; // expected offset of the next sequential read + // reads leave the NVMe queue idle during the app's compute (the seq-read + // gap vs Linux, which keeps the device busy via async prefetch). + // + // Design: two 1 MiB windows, "cur" (being served to the app) and "next" + // (being prefetched). A dedicated worker thread fills "next" while the app + // consumes "cur". When a read crosses out of "cur", we swap next->cur and + // ask the worker to prefetch the following window. So by the time the app + // needs the next bytes they are usually already in memory, and the device + // stays busy. All state is under ra_lock. Any write/truncate invalidates + // both windows; ~ext_vdata joins and frees the worker. + // + // ponytail: one worker pthread per open vnode, not a shared pool. Correct + // and keeps the device busy; a bounded pool is the upgrade if thousands of + // files are read sequentially at once. + mutex_t ra_lock; // protects all fields below except the worker + // handshake fields (those use ra_cvmtx) + char *cur_buf; // owned; window currently served to the app + uint64_t cur_start; + size_t cur_len; + char *next_buf; // owned; window being / to be prefetched + uint64_t next_start; // file offset the worker should prefetch + size_t next_len; // valid bytes in next_buf once ready + bool next_ready; // worker finished filling next_buf + bool next_pending; // worker asked to fill next_buf (in flight) + uint64_t ra_next; // expected offset of next sequential read + uint32_t inode_no; // for the worker to get its own inode_ref + struct ext4_fs *fs; // filesystem the worker reads from + + // Worker thread handshake (separate mutex/cond so the worker never blocks + // on ra_lock while the read path holds it, and the read path never blocks + // on the worker while holding ra_lock). + pthread_t worker; + bool worker_started; + pthread_mutex_t ra_cvmtx; + pthread_cond_t ra_req_cv; // signalled to give the worker work + pthread_cond_t ra_done_cv; // signalled by the worker when done + bool req_work; // there is a prefetch request for the worker + bool worker_busy; // worker is currently reading + bool shutdown; // tell the worker to exit + // Prefetch request the read path hands to the worker: + char *req_buf; + uint64_t req_off; + size_t req_len; + // Result the worker hands back: + size_t req_got; + int req_err; ext_vdata() { ref_count = 0; delete_on_last_close = false; mutex_init(&ra_lock); - ra_buf = nullptr; - ra_start = 0; - ra_len = 0; + cur_buf = nullptr; + cur_start = 0; + cur_len = 0; + next_buf = nullptr; + next_start = 0; + next_len = 0; + next_ready = false; + next_pending = false; ra_next = 0; + inode_no = 0; + fs = nullptr; + worker_started = false; + pthread_mutex_init(&ra_cvmtx, nullptr); + pthread_cond_init(&ra_req_cv, nullptr); + pthread_cond_init(&ra_done_cv, nullptr); + req_work = false; + worker_busy = false; + shutdown = false; + req_buf = nullptr; + req_off = 0; + req_len = 0; + req_got = 0; + req_err = 0; } ~ext_vdata() { - if (ra_buf) { - free_contiguous_aligned(ra_buf); + // Tell the worker to exit and wait for it, so nothing touches the + // buffers after we free them. + if (worker_started) { + pthread_mutex_lock(&ra_cvmtx); + shutdown = true; + pthread_cond_signal(&ra_req_cv); + pthread_mutex_unlock(&ra_cvmtx); + pthread_join(worker, nullptr); + } + pthread_cond_destroy(&ra_req_cv); + pthread_cond_destroy(&ra_done_cv); + pthread_mutex_destroy(&ra_cvmtx); + if (cur_buf) { + free_contiguous_aligned(cur_buf); + } + if (next_buf) { + free_contiguous_aligned(next_buf); } } }; @@ -358,6 +430,161 @@ ext_internal_read(struct ext4_fs *fs, struct ext4_inode_ref *ref, uint64_t offse return r; } +static const size_t RA_WINDOW = 1 << 20; // 1 MiB read-ahead window + +// Prefetch worker: one per open vnode, spawned lazily. Sleeps until the read +// path hands it a request (req_buf/req_off/req_len), fills the buffer with its +// own inode_ref (concurrent reads of the same inode are safe: lwext4 locks its +// block cache internally and inode refs are per-caller), then reports the +// result. This keeps the NVMe queue busy while the app consumes the current +// window. +static void *ext_prefetch_worker(void *arg) +{ + ext_vdata *vdata = (ext_vdata *)arg; + pthread_mutex_lock(&vdata->ra_cvmtx); + for (;;) { + while (!vdata->req_work && !vdata->shutdown) { + pthread_cond_wait(&vdata->ra_req_cv, &vdata->ra_cvmtx); + } + if (vdata->shutdown) { + break; + } + // Take the request. Copy out under the handshake lock, then read + // without holding it so the read path can keep serving from cur_buf. + char *buf = vdata->req_buf; + uint64_t off = vdata->req_off; + size_t len = vdata->req_len; + vdata->req_work = false; + vdata->worker_busy = true; + pthread_mutex_unlock(&vdata->ra_cvmtx); + + size_t got = 0; + int err = EIO; + struct ext4_fs *fs = vdata->fs; + if (fs && buf) { + auto_inode_ref ref(fs, vdata->inode_no); + if (ref._r == EOK) { + err = ext_internal_read(fs, &ref._ref, off, buf, len, &got); + } else { + err = ref._r; + } + } + + pthread_mutex_lock(&vdata->ra_cvmtx); + vdata->req_got = got; + vdata->req_err = err; + vdata->worker_busy = false; + pthread_cond_signal(&vdata->ra_done_cv); + } + pthread_mutex_unlock(&vdata->ra_cvmtx); + return nullptr; +} + +// Ask the worker to prefetch [off, off+len) into next_buf. Called with ra_lock +// held. Non-blocking: just posts the request and marks next_pending. +static void ext_kick_prefetch(ext_vdata *vdata, uint64_t off, size_t len) +{ + if (!vdata->next_buf) { + vdata->next_buf = (char *)alloc_contiguous_aligned( + RA_WINDOW, alignof(std::max_align_t)); + if (!vdata->next_buf) { + return; + } + } + pthread_mutex_lock(&vdata->ra_cvmtx); + // Don't queue a new request while the worker is still busy with the last + // one; the read path harvests completion before kicking again. + if (vdata->worker_busy || vdata->req_work) { + pthread_mutex_unlock(&vdata->ra_cvmtx); + return; + } + vdata->req_buf = vdata->next_buf; + vdata->req_off = off; + vdata->req_len = len; + vdata->req_work = true; + vdata->next_start = off; + vdata->next_len = 0; + vdata->next_ready = false; + vdata->next_pending = true; + pthread_cond_signal(&vdata->ra_req_cv); + pthread_mutex_unlock(&vdata->ra_cvmtx); +} + +// Harvest a finished prefetch into next_ready/next_len. Called with ra_lock +// held. If block is true, wait for the worker to finish (used when the app +// needs the next window right now); otherwise just poll. +static void ext_reap_prefetch(ext_vdata *vdata, bool block) +{ + if (!vdata->next_pending) { + return; + } + pthread_mutex_lock(&vdata->ra_cvmtx); + while (block && (vdata->worker_busy || vdata->req_work)) { + pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); + } + if (!vdata->worker_busy && !vdata->req_work) { + // Completed. + vdata->next_pending = false; + if (vdata->req_err == EOK) { + vdata->next_len = vdata->req_got; + vdata->next_ready = true; + } else { + vdata->next_len = 0; + vdata->next_ready = false; + } + } + pthread_mutex_unlock(&vdata->ra_cvmtx); +} + +// Cancel any in-flight prefetch and drop both windows. Called with ra_lock +// held on write/truncate invalidation. Blocks until the worker is idle so its +// buffer is not being written when we discard it. +static void ext_ra_invalidate(ext_vdata *vdata) +{ + if (vdata->next_pending) { + pthread_mutex_lock(&vdata->ra_cvmtx); + while (vdata->worker_busy || vdata->req_work) { + pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); + } + pthread_mutex_unlock(&vdata->ra_cvmtx); + vdata->next_pending = false; + } + vdata->cur_len = 0; + vdata->cur_start = 0; + vdata->next_len = 0; + vdata->next_ready = false; + vdata->ra_next = 0; +} + +// Serve read_amt bytes at cur_start-relative offset `off` from cur_buf into the +// uio, then advance the uio. Caller guarantees the range is inside cur_buf. +static void ext_serve_from_cur(ext_vdata *vdata, uio_t *uio, uint64_t off, + uint64_t read_amt) +{ + memcpy(uio->uio_iov[0].iov_base, + vdata->cur_buf + (off - vdata->cur_start), read_amt); + uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; + uio->uio_iov[0].iov_len -= read_amt; + uio->uio_resid -= read_amt; + uio->uio_offset += read_amt; + vdata->ra_next = off + read_amt; +} + +// Ensure the worker thread exists. Called with ra_lock held. +static bool ext_ensure_worker(ext_vdata *vdata, struct ext4_fs *fs, + uint32_t inode_no) +{ + if (vdata->worker_started) { + return true; + } + vdata->fs = fs; + vdata->inode_no = inode_no; + if (pthread_create(&vdata->worker, nullptr, ext_prefetch_worker, vdata) == 0) { + vdata->worker_started = true; + } + return vdata->worker_started; +} + static int ext_read(vnode_t *vp, struct file *fp, uio_t *uio, int ioflag) { @@ -392,51 +619,89 @@ ext_read(vnode_t *vp, struct file *fp, uio_t *uio, int ioflag) } uint64_t read_amt = std::min(fsize - uio->uio_offset, (uint64_t)uio->uio_resid); - // Sequential read-ahead: for a single-iovec read smaller than the window, - // consult (and populate) a per-vnode read-ahead buffer. On a sequential - // access pattern this turns many small synchronous per-read() bios into one - // large read plus cheap memcpys, which is the dominant cost for sequential - // reads (a single small read() cannot keep the device pipeline busy). - static const size_t RA_WINDOW = 1 << 20; // 1 MiB read-ahead window + // ASYNC double-buffered sequential read-ahead. For a single-iovec read + // smaller than half a window, serve from cur_buf (a memcpy) and keep the + // device busy by prefetching the next window in the background. On a + // sequential pattern this hides read latency behind the app's compute. ext_vdata *vdata = (ext_vdata *)vp->v_data; if (vdata && uio->uio_iovcnt == 1 && read_amt <= RA_WINDOW / 2) { uint64_t off = uio->uio_offset; mutex_lock(&vdata->ra_lock); - // Serve from the cached window if the request lies fully inside it. - if (vdata->ra_buf && off >= vdata->ra_start && - off + read_amt <= vdata->ra_start + vdata->ra_len) { - memcpy(uio->uio_iov[0].iov_base, - vdata->ra_buf + (off - vdata->ra_start), read_amt); - vdata->ra_next = off + read_amt; + + // 1) Fully inside the current window -> serve it, then top up next. + if (vdata->cur_buf && vdata->cur_len && off >= vdata->cur_start && + off + read_amt <= vdata->cur_start + vdata->cur_len) { + ext_serve_from_cur(vdata, uio, off, read_amt); + // If the app is near the end of cur, make sure a prefetch of the + // window after next is in flight (kick once per window). + uint64_t cur_end = vdata->cur_start + vdata->cur_len; + if (vdata->ra_next >= vdata->cur_start + vdata->cur_len / 2 && + !vdata->next_pending && !vdata->next_ready && + cur_end < fsize) { + size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - cur_end); + if (ext_ensure_worker(vdata, fs, vp->v_ino)) { + ext_kick_prefetch(vdata, cur_end, win); + } + } mutex_unlock(&vdata->ra_lock); - uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; - uio->uio_iov[0].iov_len -= read_amt; - uio->uio_resid -= read_amt; - uio->uio_offset += read_amt; return 0; } - // Miss. If this looks sequential (or is the first read), fill the - // window from disk once and serve from it. - if (off == vdata->ra_next || vdata->ra_buf == nullptr) { + + // 2) Crossed out of cur. If a prefetched "next" window covers this + // offset, promote it to cur (waiting for it if still in flight), + // serve, and kick the following window. + if (off == vdata->ra_next && vdata->next_pending) { + ext_reap_prefetch(vdata, /*block=*/true); + } + if (vdata->next_ready && off >= vdata->next_start && + off + read_amt <= vdata->next_start + vdata->next_len) { + // Swap next -> cur. + std::swap(vdata->cur_buf, vdata->next_buf); + vdata->cur_start = vdata->next_start; + vdata->cur_len = vdata->next_len; + vdata->next_ready = false; + vdata->next_len = 0; + ext_serve_from_cur(vdata, uio, off, read_amt); + uint64_t cur_end = vdata->cur_start + vdata->cur_len; + if (cur_end < fsize) { + size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - cur_end); + if (ext_ensure_worker(vdata, fs, vp->v_ino)) { + ext_kick_prefetch(vdata, cur_end, win); + } + } + mutex_unlock(&vdata->ra_lock); + return 0; + } + + // 3) Cold miss / seek. Fill cur synchronously (as before) and, if this + // looks sequential, immediately kick prefetch of the next window. + if (off == vdata->ra_next || vdata->cur_len == 0) { size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - off); - if (!vdata->ra_buf) { - vdata->ra_buf = (char *)alloc_contiguous_aligned( + if (!vdata->cur_buf) { + vdata->cur_buf = (char *)alloc_contiguous_aligned( RA_WINDOW, alignof(std::max_align_t)); } - if (vdata->ra_buf) { + if (vdata->cur_buf) { + // Any stale in-flight prefetch is invalid after a seek. + ext_reap_prefetch(vdata, /*block=*/true); + vdata->next_ready = false; + vdata->next_len = 0; size_t got = 0; int rr = ext_internal_read(fs, &inode_ref._ref, off, - vdata->ra_buf, win, &got); + vdata->cur_buf, win, &got); if (rr == 0 && got >= read_amt) { - vdata->ra_start = off; - vdata->ra_len = got; - memcpy(uio->uio_iov[0].iov_base, vdata->ra_buf, read_amt); - vdata->ra_next = off + read_amt; + vdata->cur_start = off; + vdata->cur_len = got; + ext_serve_from_cur(vdata, uio, off, read_amt); + uint64_t cur_end = off + got; + if (cur_end < fsize) { + size_t win2 = (size_t)std::min((uint64_t)RA_WINDOW, + fsize - cur_end); + if (ext_ensure_worker(vdata, fs, vp->v_ino)) { + ext_kick_prefetch(vdata, cur_end, win2); + } + } mutex_unlock(&vdata->ra_lock); - uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; - uio->uio_iov[0].iov_len -= read_amt; - uio->uio_resid -= read_amt; - uio->uio_offset += read_amt; return 0; } } @@ -692,13 +957,13 @@ ext_write(vnode_t *vp, uio_t *uio, int ioflag) return inode_ref._r; } - // A write invalidates any cached read-ahead window for this file. + // A write invalidates any cached read-ahead window for this file, and must + // cancel any in-flight prefetch before it can free/reuse those buffers. { ext_vdata *vdata = (ext_vdata *)vp->v_data; if (vdata) { mutex_lock(&vdata->ra_lock); - vdata->ra_len = 0; - vdata->ra_next = 0; + ext_ra_invalidate(vdata); mutex_unlock(&vdata->ra_lock); } } @@ -1448,6 +1713,16 @@ ext_truncate(struct vnode *vp, off_t new_size) { ext_debug("truncate i-node=%ld, new_size:%ld\n", vp->v_ino, new_size); struct ext4_fs *fs = (struct ext4_fs *)vp->v_mount->m_data; + // Truncation changes the file's block layout; drop any read-ahead windows + // and cancel in-flight prefetch first. + { + ext_vdata *vdata = (ext_vdata *)vp->v_data; + if (vdata) { + mutex_lock(&vdata->ra_lock); + ext_ra_invalidate(vdata); + mutex_unlock(&vdata->ra_lock); + } + } bool update_cmtimed = false; int r = ext_trunc_inode(fs, vp->v_ino, new_size, &update_cmtimed); if (update_cmtimed) { From 2292f011565446aad5d3d89f673da2070088e52d Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 14 Jul 2026 07:01:07 -0400 Subject: [PATCH 7/9] copyright: correct new-file attribution to the actual author --- tests/tst-ext4-bench.cc | 2 +- tests/tst-ext4-rw.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tst-ext4-bench.cc b/tests/tst-ext4-bench.cc index 5828c974c5..46dfa62993 100644 --- a/tests/tst-ext4-bench.cc +++ b/tests/tst-ext4-bench.cc @@ -1,5 +1,5 @@ /* - * Copyright (C) 2026 Waldemar Kozaczuk + * Copyright (C) 2026 Greg Burd * * This work is open source software, licensed under the terms of the * BSD license as described in the LICENSE file in the top-level directory. diff --git a/tests/tst-ext4-rw.cc b/tests/tst-ext4-rw.cc index 23da14a6f9..182c63b50c 100644 --- a/tests/tst-ext4-rw.cc +++ b/tests/tst-ext4-rw.cc @@ -1,5 +1,5 @@ /* - * Copyright (C) 2026 Waldemar Kozaczuk + * Copyright (C) 2026 Greg Burd * * This work is open source software, licensed under the terms of the * BSD license as described in the LICENSE file in the top-level directory. From 02a5d53d705baafa9699ce0e87de0f733acfb649 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 14 Jul 2026 07:27:03 -0400 Subject: [PATCH 8/9] libext: deep multi-window prefetch ring for ext4 sequential reads The async double-buffered read-ahead only kept ~1-2 windows ahead of the consumer, so a single sequential stream drove the NVMe queue to depth ~1-2 and topped out ~1.25 GB/s vs Linux's ~2.3 GB/s (Linux issues many concurrent readahead requests, keeping the device queue deep). Replace the two-window (cur/next) scheme with a ring of RA_WINDOWS windows (256 KiB x 8 = 2 MiB per open file, same memory as before) filled by a pool of RA_WORKERS worker pthreads. Each worker calls the existing synchronous ext_internal_read() into a ring slot, so RA_WORKERS fills are in flight at once -> device queue depth ~RA_WORKERS. As soon as the consumer drains a window the ring slides forward and re-arms that slot for the window RA_WINDOWS ahead, keeping the pipeline full. Correctness: per-slot state (EMPTY/FILLING/READY) + a ring generation. All worker-shared fields are under ra_cvmtx; the read path holds ra_lock (outer) and takes ra_cvmtx to inspect slots. Workers never take ra_lock, so no deadlock. Seek/write/truncate bump the generation and drain in-flight fills before reusing buffers (no use-after-free); stale worker results whose generation no longer matches are discarded. Reads larger than a window or spanning two windows fall through to the direct read path. ~ext_vdata broadcasts shutdown, joins all workers, then frees the buffers. Reuses lwext4's existing bio path (ext_internal_read -> ext4_blocks_get_direct -> one bio + bio_wait) rather than issuing raw bios, which would mean re-implementing lwext4's extent->block mapping; a worker pool blocked in bio_wait maps directly to device queue depth and keeps all that code correct. Verified with tests/tst-ext4-bench.cc (absolute-offset byte verification) at bs 4K/64K/128K/256K/262143/1M and files smaller than the ring: no VERIFY FAIL. --- modules/libext/ext_vnops.cc | 510 +++++++++++++++++++----------------- 1 file changed, 264 insertions(+), 246 deletions(-) diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index 2c3a98c74b..4d26f13c49 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -96,106 +96,122 @@ struct auto_inode_ref { } }; +// DEEP multi-window ASYNC sequential read-ahead. +// +// lwext4's read path issues one synchronous bio per read() (ext_internal_read +// -> ext4_blocks_get_direct -> blockdev_bread -> one bio + bio_wait). With a +// single window ahead the NVMe queue only ever sees depth ~1-2, so a single +// sequential stream never fills the device queue the way Linux does with its +// multi-request readahead. The measured result was ~1.25 GB/s vs Linux ~2.3. +// +// Design: a ring of RA_WINDOWS windows of RA_WINDOW bytes each. A pool of +// RA_WORKERS worker pthreads pull fill jobs off a queue and each calls +// ext_internal_read() synchronously -- so up to RA_WORKERS bios are in flight +// at the device at once (queue depth ~RA_WORKERS). Because we keep RA_WINDOWS +// windows armed ahead of the consumer, as soon as the app drains one window we +// re-arm it for the window RA_WINDOWS ahead, keeping the pipeline full. +// +// Each ring slot maps to file offset (base + i*RA_WINDOW). "base" is the +// window-aligned start of the current sequential run; head is the slot the +// consumer is reading from. Slot state: EMPTY (needs a fill), FILLING (a +// worker owns the buffer -- do not touch), READY (bytes valid, len set). +// +// All ring state is under ra_lock (an OSv mutex). The worker job queue and +// completion use a separate pthread mutex/cond (ra_cvmtx) so a worker never +// blocks on ra_lock and the read path never blocks on a worker while holding +// ra_lock. A slot in FILLING is never freed/reused until its worker completes; +// invalidate/seek/teardown all drain in-flight fills first (no use-after-free). +// +// Memory: RA_WINDOW * RA_WINDOWS per open file (256K * 8 = 2 MiB). Buffers are +// allocated lazily as slots are first armed, so a file shorter than the ring +// only allocates the windows it actually uses. +static const size_t RA_WINDOW = 256 * 1024; // per-window bytes +static const unsigned RA_WINDOWS = 8; // ring depth (windows ahead) +static const unsigned RA_WORKERS = 4; // concurrent fill threads => QD + +enum ra_slot_state { RA_EMPTY = 0, RA_FILLING, RA_READY }; + +struct ra_job { + unsigned slot; // ring slot this job fills + uint64_t off; // file offset to read + size_t len; // bytes to read + uint64_t gen; // ring generation; stale jobs (gen mismatch) are dropped +}; + struct ext_vdata { int ref_count; bool delete_on_last_close; - // Double-buffered ASYNC sequential read-ahead. lwext4's read path issues one - // synchronous bio per read() with no prefetch, so many small sequential - // reads leave the NVMe queue idle during the app's compute (the seq-read - // gap vs Linux, which keeps the device busy via async prefetch). - // - // Design: two 1 MiB windows, "cur" (being served to the app) and "next" - // (being prefetched). A dedicated worker thread fills "next" while the app - // consumes "cur". When a read crosses out of "cur", we swap next->cur and - // ask the worker to prefetch the following window. So by the time the app - // needs the next bytes they are usually already in memory, and the device - // stays busy. All state is under ra_lock. Any write/truncate invalidates - // both windows; ~ext_vdata joins and frees the worker. - // - // ponytail: one worker pthread per open vnode, not a shared pool. Correct - // and keeps the device busy; a bounded pool is the upgrade if thousands of - // files are read sequentially at once. - mutex_t ra_lock; // protects all fields below except the worker - // handshake fields (those use ra_cvmtx) - char *cur_buf; // owned; window currently served to the app - uint64_t cur_start; - size_t cur_len; - char *next_buf; // owned; window being / to be prefetched - uint64_t next_start; // file offset the worker should prefetch - size_t next_len; // valid bytes in next_buf once ready - bool next_ready; // worker finished filling next_buf - bool next_pending; // worker asked to fill next_buf (in flight) - uint64_t ra_next; // expected offset of next sequential read - uint32_t inode_no; // for the worker to get its own inode_ref - struct ext4_fs *fs; // filesystem the worker reads from - - // Worker thread handshake (separate mutex/cond so the worker never blocks - // on ra_lock while the read path holds it, and the read path never blocks - // on the worker while holding ra_lock). - pthread_t worker; - bool worker_started; + mutex_t ra_lock; // protects all ring fields below + char *win_buf[RA_WINDOWS]; // owned; lazily allocated + uint8_t win_state[RA_WINDOWS]; // ra_slot_state + size_t win_len[RA_WINDOWS]; // valid bytes when READY + uint64_t base; // file offset of ring slot 0's stream + unsigned head; // slot currently being consumed + bool ring_active; // ring is armed for a sequential run + uint64_t ra_next; // expected offset of next sequential read + uint64_t gen; // bumped on seek/invalidate; stale jobs die + unsigned inflight; // slots currently FILLING (workers busy) + uint32_t inode_no; + struct ext4_fs *fs; + + // Worker pool + job queue (pthread primitives; module is -fno-rtti so no + // sched::thread). + pthread_t workers[RA_WORKERS]; + unsigned nworkers; + bool workers_started; pthread_mutex_t ra_cvmtx; - pthread_cond_t ra_req_cv; // signalled to give the worker work - pthread_cond_t ra_done_cv; // signalled by the worker when done - bool req_work; // there is a prefetch request for the worker - bool worker_busy; // worker is currently reading - bool shutdown; // tell the worker to exit - // Prefetch request the read path hands to the worker: - char *req_buf; - uint64_t req_off; - size_t req_len; - // Result the worker hands back: - size_t req_got; - int req_err; + pthread_cond_t ra_job_cv; // signalled when a job is queued + pthread_cond_t ra_done_cv; // signalled when a fill completes + ra_job jobq[RA_WINDOWS]; // at most one pending job per slot + unsigned jobq_n; + bool shutdown; ext_vdata() { ref_count = 0; delete_on_last_close = false; mutex_init(&ra_lock); - cur_buf = nullptr; - cur_start = 0; - cur_len = 0; - next_buf = nullptr; - next_start = 0; - next_len = 0; - next_ready = false; - next_pending = false; + for (unsigned i = 0; i < RA_WINDOWS; i++) { + win_buf[i] = nullptr; + win_state[i] = RA_EMPTY; + win_len[i] = 0; + } + base = 0; + head = 0; + ring_active = false; ra_next = 0; + gen = 0; + inflight = 0; inode_no = 0; fs = nullptr; - worker_started = false; + nworkers = 0; + workers_started = false; pthread_mutex_init(&ra_cvmtx, nullptr); - pthread_cond_init(&ra_req_cv, nullptr); + pthread_cond_init(&ra_job_cv, nullptr); pthread_cond_init(&ra_done_cv, nullptr); - req_work = false; - worker_busy = false; + jobq_n = 0; shutdown = false; - req_buf = nullptr; - req_off = 0; - req_len = 0; - req_got = 0; - req_err = 0; } ~ext_vdata() { - // Tell the worker to exit and wait for it, so nothing touches the + // Tell the workers to exit and wait for them, so nothing touches the // buffers after we free them. - if (worker_started) { + if (workers_started) { pthread_mutex_lock(&ra_cvmtx); shutdown = true; - pthread_cond_signal(&ra_req_cv); + pthread_cond_broadcast(&ra_job_cv); pthread_mutex_unlock(&ra_cvmtx); - pthread_join(worker, nullptr); + for (unsigned i = 0; i < nworkers; i++) { + pthread_join(workers[i], nullptr); + } } - pthread_cond_destroy(&ra_req_cv); + pthread_cond_destroy(&ra_job_cv); pthread_cond_destroy(&ra_done_cv); pthread_mutex_destroy(&ra_cvmtx); - if (cur_buf) { - free_contiguous_aligned(cur_buf); - } - if (next_buf) { - free_contiguous_aligned(next_buf); + for (unsigned i = 0; i < RA_WINDOWS; i++) { + if (win_buf[i]) { + free_contiguous_aligned(win_buf[i]); + } } } }; @@ -430,139 +446,140 @@ ext_internal_read(struct ext4_fs *fs, struct ext4_inode_ref *ref, uint64_t offse return r; } -static const size_t RA_WINDOW = 1 << 20; // 1 MiB read-ahead window - -// Prefetch worker: one per open vnode, spawned lazily. Sleeps until the read -// path hands it a request (req_buf/req_off/req_len), fills the buffer with its +// Worker pool: each thread pulls a fill job off the queue and calls +// ext_internal_read() synchronously into the job's ring slot buffer, using its // own inode_ref (concurrent reads of the same inode are safe: lwext4 locks its -// block cache internally and inode refs are per-caller), then reports the -// result. This keeps the NVMe queue busy while the app consumes the current -// window. +// block cache internally and inode refs are per-caller). Each blocked worker +// holds one bio at the device, so RA_WORKERS threads give queue depth +// ~RA_WORKERS. Jobs carry the ring generation; if it no longer matches when +// the fill finishes, a seek/invalidate happened and the result is discarded +// (the slot may have been re-armed for a different offset), so we never publish +// stale bytes. static void *ext_prefetch_worker(void *arg) { ext_vdata *vdata = (ext_vdata *)arg; pthread_mutex_lock(&vdata->ra_cvmtx); for (;;) { - while (!vdata->req_work && !vdata->shutdown) { - pthread_cond_wait(&vdata->ra_req_cv, &vdata->ra_cvmtx); + while (vdata->jobq_n == 0 && !vdata->shutdown) { + pthread_cond_wait(&vdata->ra_job_cv, &vdata->ra_cvmtx); } if (vdata->shutdown) { break; } - // Take the request. Copy out under the handshake lock, then read - // without holding it so the read path can keep serving from cur_buf. - char *buf = vdata->req_buf; - uint64_t off = vdata->req_off; - size_t len = vdata->req_len; - vdata->req_work = false; - vdata->worker_busy = true; + // Pop a job (LIFO is fine; order within the armed set does not matter + // for correctness, only that each gets filled). + ra_job job = vdata->jobq[--vdata->jobq_n]; pthread_mutex_unlock(&vdata->ra_cvmtx); size_t got = 0; int err = EIO; struct ext4_fs *fs = vdata->fs; + char *buf = vdata->win_buf[job.slot]; if (fs && buf) { auto_inode_ref ref(fs, vdata->inode_no); if (ref._r == EOK) { - err = ext_internal_read(fs, &ref._ref, off, buf, len, &got); + err = ext_internal_read(fs, &ref._ref, job.off, buf, job.len, &got); } else { err = ref._r; } } pthread_mutex_lock(&vdata->ra_cvmtx); - vdata->req_got = got; - vdata->req_err = err; - vdata->worker_busy = false; - pthread_cond_signal(&vdata->ra_done_cv); + vdata->inflight--; + // Publish only if the ring generation still matches; otherwise a + // seek/invalidate reclaimed this slot and its buffer content is now + // meaningless -- leave whatever state the invalidate set. + if (job.gen == vdata->gen && vdata->win_state[job.slot] == RA_FILLING) { + if (err == EOK) { + vdata->win_len[job.slot] = got; + vdata->win_state[job.slot] = RA_READY; + } else { + vdata->win_len[job.slot] = 0; + vdata->win_state[job.slot] = RA_EMPTY; + } + } + pthread_cond_broadcast(&vdata->ra_done_cv); } pthread_mutex_unlock(&vdata->ra_cvmtx); return nullptr; } -// Ask the worker to prefetch [off, off+len) into next_buf. Called with ra_lock -// held. Non-blocking: just posts the request and marks next_pending. -static void ext_kick_prefetch(ext_vdata *vdata, uint64_t off, size_t len) +// Ensure the worker pool exists. Called with ra_lock held. +static bool ext_ensure_workers(ext_vdata *vdata, struct ext4_fs *fs, + uint32_t inode_no) { - if (!vdata->next_buf) { - vdata->next_buf = (char *)alloc_contiguous_aligned( - RA_WINDOW, alignof(std::max_align_t)); - if (!vdata->next_buf) { - return; - } + if (vdata->workers_started) { + return true; } - pthread_mutex_lock(&vdata->ra_cvmtx); - // Don't queue a new request while the worker is still busy with the last - // one; the read path harvests completion before kicking again. - if (vdata->worker_busy || vdata->req_work) { - pthread_mutex_unlock(&vdata->ra_cvmtx); - return; + vdata->fs = fs; + vdata->inode_no = inode_no; + for (unsigned i = 0; i < RA_WORKERS; i++) { + if (pthread_create(&vdata->workers[i], nullptr, + ext_prefetch_worker, vdata) == 0) { + vdata->nworkers++; + } } - vdata->req_buf = vdata->next_buf; - vdata->req_off = off; - vdata->req_len = len; - vdata->req_work = true; - vdata->next_start = off; - vdata->next_len = 0; - vdata->next_ready = false; - vdata->next_pending = true; - pthread_cond_signal(&vdata->ra_req_cv); - pthread_mutex_unlock(&vdata->ra_cvmtx); + vdata->workers_started = (vdata->nworkers > 0); + return vdata->workers_started; } -// Harvest a finished prefetch into next_ready/next_len. Called with ra_lock -// held. If block is true, wait for the worker to finish (used when the app -// needs the next window right now); otherwise just poll. -static void ext_reap_prefetch(ext_vdata *vdata, bool block) +// Arm ring slot `slot` (which must currently be EMPTY) to be filled for file +// offset `off`. Allocates the buffer lazily. Called with ra_lock AND ra_cvmtx +// held; queues a job and signals a worker. off must be < fsize. +static void ext_arm_slot(ext_vdata *vdata, unsigned slot, uint64_t off, + uint64_t fsize) { - if (!vdata->next_pending) { + if (off >= fsize) { return; } - pthread_mutex_lock(&vdata->ra_cvmtx); - while (block && (vdata->worker_busy || vdata->req_work)) { - pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); - } - if (!vdata->worker_busy && !vdata->req_work) { - // Completed. - vdata->next_pending = false; - if (vdata->req_err == EOK) { - vdata->next_len = vdata->req_got; - vdata->next_ready = true; - } else { - vdata->next_len = 0; - vdata->next_ready = false; + if (!vdata->win_buf[slot]) { + vdata->win_buf[slot] = (char *)alloc_contiguous_aligned( + RA_WINDOW, alignof(std::max_align_t)); + if (!vdata->win_buf[slot]) { + return; } } - pthread_mutex_unlock(&vdata->ra_cvmtx); + size_t len = (size_t)std::min((uint64_t)RA_WINDOW, fsize - off); + vdata->win_state[slot] = RA_FILLING; + vdata->win_len[slot] = 0; + vdata->inflight++; + ra_job &j = vdata->jobq[vdata->jobq_n++]; + j.slot = slot; + j.off = off; + j.len = len; + j.gen = vdata->gen; + pthread_cond_signal(&vdata->ra_job_cv); } -// Cancel any in-flight prefetch and drop both windows. Called with ra_lock -// held on write/truncate invalidation. Blocks until the worker is idle so its -// buffer is not being written when we discard it. +// Drop the whole ring and cancel in-flight fills. Called with ra_lock held on +// write/truncate invalidation and on a seek. Bumps the generation so any +// worker result that lands after this is discarded, drains the workers that are +// mid-read (so their buffers are not being written when reused), then marks all +// slots EMPTY. static void ext_ra_invalidate(ext_vdata *vdata) { - if (vdata->next_pending) { - pthread_mutex_lock(&vdata->ra_cvmtx); - while (vdata->worker_busy || vdata->req_work) { - pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); - } - pthread_mutex_unlock(&vdata->ra_cvmtx); - vdata->next_pending = false; + pthread_mutex_lock(&vdata->ra_cvmtx); + vdata->gen++; + while (vdata->inflight > 0) { + pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); + } + for (unsigned i = 0; i < RA_WINDOWS; i++) { + vdata->win_state[i] = RA_EMPTY; + vdata->win_len[i] = 0; } - vdata->cur_len = 0; - vdata->cur_start = 0; - vdata->next_len = 0; - vdata->next_ready = false; + pthread_mutex_unlock(&vdata->ra_cvmtx); + vdata->head = 0; + vdata->base = 0; + vdata->ring_active = false; vdata->ra_next = 0; } -// Serve read_amt bytes at cur_start-relative offset `off` from cur_buf into the -// uio, then advance the uio. Caller guarantees the range is inside cur_buf. -static void ext_serve_from_cur(ext_vdata *vdata, uio_t *uio, uint64_t off, - uint64_t read_amt) +// Copy read_amt bytes from ring slot `slot` at slot-relative `rel` into the +// uio's single iovec and advance it. +static void ext_serve_from_slot(ext_vdata *vdata, uio_t *uio, unsigned slot, + size_t rel, uint64_t read_amt, uint64_t off) { - memcpy(uio->uio_iov[0].iov_base, - vdata->cur_buf + (off - vdata->cur_start), read_amt); + memcpy(uio->uio_iov[0].iov_base, vdata->win_buf[slot] + rel, read_amt); uio->uio_iov[0].iov_base = (char *)uio->uio_iov[0].iov_base + read_amt; uio->uio_iov[0].iov_len -= read_amt; uio->uio_resid -= read_amt; @@ -570,19 +587,47 @@ static void ext_serve_from_cur(ext_vdata *vdata, uio_t *uio, uint64_t off, vdata->ra_next = off + read_amt; } -// Ensure the worker thread exists. Called with ra_lock held. -static bool ext_ensure_worker(ext_vdata *vdata, struct ext4_fs *fs, - uint32_t inode_no) +// Advance the ring so slot 0 corresponds to the window containing `off`, +// re-arming freed trailing slots for the windows now furthest ahead. Called +// with ra_lock held. Keeps up to RA_WINDOWS windows armed ahead of `off`. +// win_state/win_len/inflight/jobq are worker-shared, so hold ra_cvmtx while +// touching them (workers only ever hold ra_cvmtx, never ra_lock -> no deadlock). +static void ext_ring_refill(ext_vdata *vdata, uint64_t off, uint64_t fsize, + struct ext4_fs *fs, uint32_t inode_no) { - if (vdata->worker_started) { - return true; + if (!ext_ensure_workers(vdata, fs, inode_no)) { + return; } - vdata->fs = fs; - vdata->inode_no = inode_no; - if (pthread_create(&vdata->worker, nullptr, ext_prefetch_worker, vdata) == 0) { - vdata->worker_started = true; + pthread_mutex_lock(&vdata->ra_cvmtx); + // Which window (relative to base) does off fall in? + uint64_t rel_win = (off - vdata->base) / RA_WINDOW; + // Slide head forward by rel_win, re-arming each slot we pass for the window + // RA_WINDOWS ahead of where it was, keeping the pipeline full. + while (rel_win > 0) { + unsigned slot = vdata->head; + // Only reclaim a slot the consumer has passed; it must not be FILLING + // (a worker owns the buffer). If it is still filling, stop advancing + // and let a later read retry -- correctness first. + if (vdata->win_state[slot] == RA_FILLING) { + break; + } + uint64_t next_off = vdata->base + (uint64_t)RA_WINDOWS * RA_WINDOW; + vdata->win_state[slot] = RA_EMPTY; + vdata->win_len[slot] = 0; + vdata->head = (vdata->head + 1) % RA_WINDOWS; + vdata->base += RA_WINDOW; + ext_arm_slot(vdata, slot, next_off, fsize); + rel_win--; + } + // Arm any EMPTY slots within the window ahead (initial fill / catch-up). + for (unsigned i = 0; i < RA_WINDOWS; i++) { + unsigned slot = (vdata->head + i) % RA_WINDOWS; + if (vdata->win_state[slot] == RA_EMPTY) { + uint64_t slot_off = vdata->base + (uint64_t)i * RA_WINDOW; + ext_arm_slot(vdata, slot, slot_off, fsize); + } } - return vdata->worker_started; + pthread_mutex_unlock(&vdata->ra_cvmtx); } static int @@ -619,94 +664,67 @@ ext_read(vnode_t *vp, struct file *fp, uio_t *uio, int ioflag) } uint64_t read_amt = std::min(fsize - uio->uio_offset, (uint64_t)uio->uio_resid); - // ASYNC double-buffered sequential read-ahead. For a single-iovec read - // smaller than half a window, serve from cur_buf (a memcpy) and keep the - // device busy by prefetching the next window in the background. On a - // sequential pattern this hides read latency behind the app's compute. + // DEEP async sequential read-ahead. For a single-iovec read that fits + // inside one prefetch window, serve it from the ring (a memcpy) and keep + // RA_WINDOWS windows armed ahead via the worker pool, so the NVMe queue + // stays deep (~RA_WORKERS bios in flight). Only sequential access uses the + // ring; a seek resets it. Reads spanning two windows fall through to the + // direct path (rare: only for reads not aligned within a window). ext_vdata *vdata = (ext_vdata *)vp->v_data; - if (vdata && uio->uio_iovcnt == 1 && read_amt <= RA_WINDOW / 2) { + if (vdata && uio->uio_iovcnt == 1 && read_amt <= RA_WINDOW && + (uio->uio_offset / RA_WINDOW) == + ((uio->uio_offset + read_amt - 1) / RA_WINDOW)) { uint64_t off = uio->uio_offset; mutex_lock(&vdata->ra_lock); - // 1) Fully inside the current window -> serve it, then top up next. - if (vdata->cur_buf && vdata->cur_len && off >= vdata->cur_start && - off + read_amt <= vdata->cur_start + vdata->cur_len) { - ext_serve_from_cur(vdata, uio, off, read_amt); - // If the app is near the end of cur, make sure a prefetch of the - // window after next is in flight (kick once per window). - uint64_t cur_end = vdata->cur_start + vdata->cur_len; - if (vdata->ra_next >= vdata->cur_start + vdata->cur_len / 2 && - !vdata->next_pending && !vdata->next_ready && - cur_end < fsize) { - size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - cur_end); - if (ext_ensure_worker(vdata, fs, vp->v_ino)) { - ext_kick_prefetch(vdata, cur_end, win); - } - } - mutex_unlock(&vdata->ra_lock); - return 0; + bool sequential = (off == vdata->ra_next) || !vdata->ring_active; + bool in_ring = false; + if (sequential && vdata->ring_active && off >= vdata->base && + off < vdata->base + (uint64_t)RA_WINDOWS * RA_WINDOW) { + in_ring = true; } - // 2) Crossed out of cur. If a prefetched "next" window covers this - // offset, promote it to cur (waiting for it if still in flight), - // serve, and kick the following window. - if (off == vdata->ra_next && vdata->next_pending) { - ext_reap_prefetch(vdata, /*block=*/true); - } - if (vdata->next_ready && off >= vdata->next_start && - off + read_amt <= vdata->next_start + vdata->next_len) { - // Swap next -> cur. - std::swap(vdata->cur_buf, vdata->next_buf); - vdata->cur_start = vdata->next_start; - vdata->cur_len = vdata->next_len; - vdata->next_ready = false; - vdata->next_len = 0; - ext_serve_from_cur(vdata, uio, off, read_amt); - uint64_t cur_end = vdata->cur_start + vdata->cur_len; - if (cur_end < fsize) { - size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - cur_end); - if (ext_ensure_worker(vdata, fs, vp->v_ino)) { - ext_kick_prefetch(vdata, cur_end, win); - } - } - mutex_unlock(&vdata->ra_lock); - return 0; + if (!in_ring) { + // Cold start or seek: reset the ring to a window-aligned base at + // `off` and arm the pipeline. Drain first so no worker writes a + // buffer we are about to repoint. + ext_ra_invalidate(vdata); + vdata->base = off - (off % RA_WINDOW); + vdata->head = 0; + vdata->ring_active = true; + ext_ring_refill(vdata, off, fsize, fs, vp->v_ino); } - // 3) Cold miss / seek. Fill cur synchronously (as before) and, if this - // looks sequential, immediately kick prefetch of the next window. - if (off == vdata->ra_next || vdata->cur_len == 0) { - size_t win = (size_t)std::min((uint64_t)RA_WINDOW, fsize - off); - if (!vdata->cur_buf) { - vdata->cur_buf = (char *)alloc_contiguous_aligned( - RA_WINDOW, alignof(std::max_align_t)); + // Locate the slot holding `off`. + uint64_t rel_win = (off - vdata->base) / RA_WINDOW; + if (rel_win < RA_WINDOWS) { + unsigned slot = (vdata->head + rel_win) % RA_WINDOWS; + // Snapshot the slot's worker-shared state/len under ra_cvmtx, + // waiting for an in-flight fill to complete first. + pthread_mutex_lock(&vdata->ra_cvmtx); + while (vdata->win_state[slot] == RA_FILLING) { + pthread_cond_wait(&vdata->ra_done_cv, &vdata->ra_cvmtx); } - if (vdata->cur_buf) { - // Any stale in-flight prefetch is invalid after a seek. - ext_reap_prefetch(vdata, /*block=*/true); - vdata->next_ready = false; - vdata->next_len = 0; - size_t got = 0; - int rr = ext_internal_read(fs, &inode_ref._ref, off, - vdata->cur_buf, win, &got); - if (rr == 0 && got >= read_amt) { - vdata->cur_start = off; - vdata->cur_len = got; - ext_serve_from_cur(vdata, uio, off, read_amt); - uint64_t cur_end = off + got; - if (cur_end < fsize) { - size_t win2 = (size_t)std::min((uint64_t)RA_WINDOW, - fsize - cur_end); - if (ext_ensure_worker(vdata, fs, vp->v_ino)) { - ext_kick_prefetch(vdata, cur_end, win2); - } - } - mutex_unlock(&vdata->ra_lock); - return 0; - } + uint8_t st = vdata->win_state[slot]; + size_t wlen = vdata->win_len[slot]; + pthread_mutex_unlock(&vdata->ra_cvmtx); + + uint64_t win_start = vdata->base + rel_win * RA_WINDOW; + if (st == RA_READY && vdata->win_buf[slot] && + off >= win_start && + off + read_amt <= win_start + wlen) { + size_t rel = (size_t)(off - win_start); + ext_serve_from_slot(vdata, uio, slot, rel, read_amt, off); + // Slide the ring forward and top up windows ahead. + ext_ring_refill(vdata, off + read_amt, fsize, fs, vp->v_ino); + mutex_unlock(&vdata->ra_lock); + return 0; } } - vdata->ra_next = off + read_amt; // track for next-time detection + // Ring miss (fill failed / short file / race): fall through to the + // direct read below, but keep ra_next in sync so the next read is seen + // as sequential. + vdata->ra_next = off + read_amt; mutex_unlock(&vdata->ra_lock); } From 3e8cfd57cc4a7c805decf54d60b790bfdd7a4c28 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Tue, 14 Jul 2026 08:13:28 -0400 Subject: [PATCH 9/9] libext: tune prefetch ring to N=4/M=4 from a device-queue-depth sweep Sweep on a local-NVMe host (, ext4 4K/no-journal, O_DIRECT so reads hit the device, 512 MiB, median of 3): config 64K 128K 256K prefetch mem/file N=1 M=1 ~0.69 ~0.85 ~0.85 256 KiB (~QD1) N=4 M=2 ~1.20 - ~1.28 1 MiB N=4 M=4 ~1.26 ~1.23 ~1.29 1 MiB <- knee N=8 M=4 ~1.28 ~1.28 ~1.31 2 MiB N=8 M=8 ~1.28 ~1.28 ~1.29 2 MiB N=16 M=16 ~1.25 - ~1.21 4 MiB Throughput climbs steeply from N=1/M=1 to the knee at N=4/M=4 (~1.28 GB/s) and then plateaus: deeper rings or more workers give no further gain. The cap is downstream of this code -- OSv's virtio-blk make_request() serialises submission under a single _lock and a single virtqueue, so effective device queue depth tops out at ~2-4 regardless of how many prefetch windows are in flight. (Linux fio O_DIRECT on the same raw NVMe: ~1.32 GB/s @128K QD1, ~1.68 GB/s @128K QD>=2, i.e. the device itself saturates at QD2; the remaining OSv gap is the guest virtio-blk path, not prefetch depth.) So N=4/M=4 is the sweet spot: same throughput as the deeper configs at half the memory (1 MiB vs 2 MiB per open file). It also matches the prior 2-window async design (~1.25 GB/s here) while using a general N-window ring. Make N and M build-time tunables (-DRA_WINDOWS_N / -DRA_WORKERS_M via $(RA_FLAGS)) so the sweep is reproducible when the virtio-blk submission path is later parallelised. No VERIFY FAIL at any config or block size (4K/64K/128K/256K/262143/1M, files smaller than the ring). --- modules/libext/Makefile | 2 +- modules/libext/ext_vnops.cc | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/modules/libext/Makefile b/modules/libext/Makefile index f1db54fe61..a7ca86868c 100644 --- a/modules/libext/Makefile +++ b/modules/libext/Makefile @@ -3,7 +3,7 @@ include ../common.gmk module_out := $(out)/modules/libext CXXFLAGS = -fPIC -std=gnu++11 $(INCLUDES) -I../lwext4/upstream/lwext4/include -I../lwext4/upstream/lwext4/build_lib_only/include \ - -D_KERNEL -D_GNU_SOURCE -Wall -fno-exceptions -fno-rtti -O2 + -D_KERNEL -D_GNU_SOURCE -Wall -fno-exceptions -fno-rtti -O2 $(RA_FLAGS) # the build target executable: TARGET = libext diff --git a/modules/libext/ext_vnops.cc b/modules/libext/ext_vnops.cc index 4d26f13c49..0a437c4e29 100644 --- a/modules/libext/ext_vnops.cc +++ b/modules/libext/ext_vnops.cc @@ -122,12 +122,27 @@ struct auto_inode_ref { // ra_lock. A slot in FILLING is never freed/reused until its worker completes; // invalidate/seek/teardown all drain in-flight fills first (no use-after-free). // -// Memory: RA_WINDOW * RA_WINDOWS per open file (256K * 8 = 2 MiB). Buffers are +// Memory: RA_WINDOW * RA_WINDOWS per open file (256K * 4 = 1 MiB by default). +// Buffers are // allocated lazily as slots are first armed, so a file shorter than the ring // only allocates the windows it actually uses. static const size_t RA_WINDOW = 256 * 1024; // per-window bytes -static const unsigned RA_WINDOWS = 8; // ring depth (windows ahead) -static const unsigned RA_WORKERS = 4; // concurrent fill threads => QD +// N (ring depth) and M (worker count = device queue depth) are the two tuning +// knobs; override at build time with -DRA_WINDOWS_N= -DRA_WORKERS_M= for +// the sweep. Defaults from a device-queue-depth sweep: throughput climbs from +// ~0.69 GB/s at N=1/M=1 to a ~1.28 GB/s plateau at N=4/M=4; going deeper +// (N=8/M=8, N=16/M=16) yields no further gain because OSv's virtio-blk +// make_request() serialises submission under a single _lock/virtqueue, capping +// the effective device queue depth at ~2-4. N=4/M=4 sits at the knee and uses +// only 1 MiB of prefetch memory per open file (4 x 256 KiB). +#ifndef RA_WINDOWS_N +#define RA_WINDOWS_N 4 +#endif +#ifndef RA_WORKERS_M +#define RA_WORKERS_M 4 +#endif +static const unsigned RA_WINDOWS = RA_WINDOWS_N; // ring depth (windows ahead) +static const unsigned RA_WORKERS = RA_WORKERS_M; // concurrent fill threads => QD enum ra_slot_state { RA_EMPTY = 0, RA_FILLING, RA_READY }; @@ -1899,7 +1914,7 @@ ext_inactive(vnode_t *vp) // page-sized, page-aligned chunk at uio_offset into a freshly allocated page // and hand it to pagecache::map_read_cached_page(), which takes ownership. // -// ponytail: allocate-and-copy from lwext4, not a zero-copy borrow-and-pin. +// note: allocate-and-copy from lwext4, not a zero-copy borrow-and-pin. // lwext4's block cache buffers are not page-aligned/shareable the way ROFS's // read-around cache is, so copying is the correct first step; a borrow-and-pin // bridge like the ZFS ARC one can come later if it shows up hot.