test(qwp): portably regression-guard recovery's mmap-fault path (#64 M1)

The three MmapSegmentRecoveryFaultTest cases only fault on ZFS: on ext4/xfs a
within-EOF sparse hole zero-fills, so the scan stops via the CRC-mismatch /
bad-magic branch and every test stays green even with the recovery mmap-fault
guard fully reverted -- zero fail-on-revert protection on the primary CI
runners (PR #64 review, item M1). Proven by reverting both production files to
the pre-branch base: 3/3 still passed on ext4.

Add a portable seam and two tests that fault deterministically on any
filesystem. A read of a mapped page beyond the file's real EOF raises SIGBUS
everywhere (POSIX), which HotSpot translates to a catchable InternalError at an
Unsafe intrinsic site -- the same fault an unbacked ZFS page raises.
openExisting gains a FilesFacade overload (mirroring create) whose file length
flows through the facade; a test facade reports a length larger than the real
(truncated-down) file so the mapping extends past EOF.

- FilesFacade.length(String) + DefaultFilesFacade delegate; openExisting(String)
  now delegates to openExisting(FilesFacade, String), which also routes openRW
  and close through the facade.
- testScanFaultOnMapPastEofIsHandledAnyFilesystem and
  testHeaderFaultOnMapPastEofIsSkippableAnyFilesystem fault on ext4 and fail on
  revert (verified: raw InternalError escapes when the guard is neutralized,
  while the three sparse-hole tests stay green -- exactly the M1 gap).

The scan test accepts both handled outcomes: under the interpreter/C1 the fault
is caught in scanFrames (graceful partial recovery), but once C2 inlines
scanFrames into openExisting the async unsafe-access InternalError is delivered
to openExisting's outer catch instead, converting the file to a skippable
MmapSegmentException. Both are safe (no JVM abort, no raw error into recovery);
a revert lets a raw InternalError through and fails the test either way.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: puzpuzpuz

core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java

@@ -258,11 +258,31 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
      * last valid frame) do not log and report {@code 0}.
      */
     public static MmapSegment openExisting(String path) {
-        long fileSize = Files.length(path);
+        return openExisting(FilesFacade.INSTANCE, path);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openExisting(String)} that takes the file
+     * length (and open/close) through a {@link FilesFacade} instead of straight
+     * off {@link Files}. Production uses {@link FilesFacade#INSTANCE}; the seam
+     * exists so recovery's mmap-fault guard can be regression-tested on any
+     * filesystem.
+     * <p>
+     * The mapping is sized to {@code ff.length(path)} and every recovery read
+     * runs straight out of it, so a facade that reports a length <em>larger</em>
+     * than the real file makes the mapping extend past end-of-file. A read of a
+     * page beyond real EOF raises SIGBUS on <em>every</em> filesystem — the same
+     * fault an unbacked/sparse page raises on ZFS, but reproduced
+     * deterministically on ext4/xfs, where a within-EOF hole would instead
+     * zero-fill and never exercise the guard. See
+     * {@link FilesFacade#length(String)}.
+     */
+    public static MmapSegment openExisting(FilesFacade ff, String path) {
+        long fileSize = ff.length(path);
         if (fileSize < HEADER_SIZE) {
             throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize);
         }
-        int fd = Files.openRW(path);
+        int fd = ff.openRW(path);
         if (fd < 0) {
             throw new MmapSegmentException("openRW failed for " + path);
         }
@@ -309,7 +329,7 @@ public static MmapSegment openExisting(String path) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
             }
-            Files.close(fd);
+            ff.close(fd);
             // The header reads above (magic/version/baseSeq) run before
             // scanFrames and are otherwise unguarded. An unbacked page 0 --
             // an unflushed header on a truncate-based-allocate filesystem

core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java

@@ -91,6 +91,11 @@ public long length(int fd) {
         return Files.length(fd);
     }
 
+    @Override
+    public long length(String path) {
+        return Files.length(path);
+    }
+
     @Override
     public int lock(int fd) {
         return Files.lock(fd);

core/src/main/java/io/questdb/client/std/FilesFacade.java

@@ -87,6 +87,22 @@ public interface FilesFacade {
 
     long length(int fd);
 
+    /**
+     * Stat length of the file at {@code path}, in bytes. Default delegates to
+     * {@link Files#length(String)}.
+     *
+     * <p>Test injection point: {@code MmapSegment.openExisting} maps the file to
+     * this length and scans straight out of the mapping, so a wrapping facade
+     * that returns a value <em>larger</em> than the real file makes the mapping
+     * extend past end-of-file. A read of a page beyond real EOF raises SIGBUS on
+     * <em>every</em> filesystem (which HotSpot translates to a catchable
+     * {@code InternalError} at an {@code Unsafe} intrinsic site) — the same fault
+     * a genuinely unbacked/sparse page raises on ZFS, but reproduced
+     * deterministically on ext4/xfs too. That is what lets recovery's mmap-fault
+     * guard be regression-tested on any CI runner rather than only on ZFS.
+     */
+    long length(String path);
+
     int lock(int fd);
 
     int mkdir(String path, int mode);

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java

@@ -27,6 +27,7 @@
 import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException;
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Unsafe;
 import io.questdb.client.test.tools.TestUtils;
@@ -70,6 +71,18 @@
  * the JVM. A hole-zero-filling filesystem (ext4) instead reads the hole back as
  * zeroes, which fails the frame CRC; either way recovery must stop at the same
  * boundary and recover the same frames.
+ * <p>
+ * <b>Fail-on-revert on any filesystem.</b> The sparse-hole tests above only
+ * fault on ZFS: on ext4/xfs the within-EOF hole zero-fills, so the scan stops
+ * via the CRC-mismatch / bad-magic branch and they stay green even with the
+ * mmap-fault guard reverted -- no regression protection on the ext4/xfs CI
+ * runners. The two {@code MapPastEof} tests below close that gap portably.
+ * They truncate the file <em>down</em> (freeing the tail blocks) and hand
+ * {@code openExisting} a {@link FilesFacade} that reports the original, larger
+ * length, so the mapping extends past real end-of-file. A read of a page beyond
+ * real EOF raises SIGBUS on <em>every</em> filesystem -- the same catchable
+ * {@code InternalError} an unbacked ZFS page raises -- so they exercise the
+ * real fault path (and fail on revert) on ext4/xfs too, not only on ZFS.
  */
 public class MmapSegmentRecoveryFaultTest {
 
@@ -187,6 +200,96 @@ public void testUnbackedHeaderPageIsSkippableNotFatal() throws Exception {
         });
     }
 
+    /**
+     * Portable fail-on-revert guard for the recovery mmap-fault handling on the
+     * scan path. Unlike {@link #testRecoveryKeepsFramesBeforeUnbackedTail}
+     * (which only faults on ZFS), this maps the file past real EOF via the
+     * length-injecting facade, so the scan's read of the beyond-EOF page faults
+     * on ext4/xfs too. The fix must <em>recognize</em> that fault and keep
+     * recovery safe -- never a JVM abort, never a raw {@code InternalError}
+     * escaping into {@code SegmentRing}'s recovery loop. Revert the
+     * {@code scanFrames}/{@code openExisting} mmap-fault guard (or fold the CRC
+     * back through native {@code Crc32c}) and this errors or aborts the fork.
+     * <p>
+     * Two handled outcomes are accepted, because which one occurs depends on
+     * whether the recovery methods are JIT-compiled at fault time:
+     * <ul>
+     *   <li><b>Interpreter / C1:</b> {@code scanFrames}'s own
+     *       {@code catch (InternalError)} fires, so the frame below the tear is
+     *       recovered and a usable segment is returned.</li>
+     *   <li><b>C2:</b> once {@code scanFrames} is inlined into
+     *       {@code openExisting}, HotSpot delivers the async unsafe-access
+     *       {@code InternalError} to {@code openExisting}'s outer
+     *       {@code catch (Throwable)} instead of the inlined inner one, which
+     *       converts the file to a skippable {@link MmapSegmentException}.
+     *       Still fully handled -- {@code SegmentRing} skips just this
+     *       {@code .sfa} rather than aborting the slot.</li>
+     * </ul>
+     * (The C2 delivery imprecision is a property of HotSpot's async
+     * unsafe-access fault handling, not of this seam; the seam only makes it
+     * reproducible off ZFS.)
+     */
+    @Test
+    public void testScanFaultOnMapPastEofIsHandledAnyFilesystem() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            final String path = tmpDir + "/seg-mappasteof-scan.sfa";
+            final long page = Files.PAGE_SIZE;
+            // One frame that ends exactly on the first page boundary.
+            final int payloadLen = (int) (page - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE);
+            long boundary = writeSegment(path, 5L, new int[]{payloadLen});
+            assertEquals("frame must fill exactly one page", page, boundary);
+            // Free every block past the first page: the file is now exactly one
+            // (fully backed) page, with nothing beyond it on disk.
+            truncateTo(path, page);
+            // Report twice the real length so openExisting maps a second,
+            // beyond-EOF page; the scan faults reading it on any filesystem.
+            FilesFacade ff = new MapPastEofFacade(path, 2 * page);
+            try (MmapSegment seg = MmapSegment.openExisting(ff, path)) {
+                // Interpreter / C1: graceful partial recovery.
+                assertEquals("the frame below the beyond-EOF page must be recovered", 1L, seg.frameCount());
+                assertEquals("scan must stop at the beyond-EOF boundary", page, seg.publishedOffset());
+                assertEquals("a beyond-EOF page is not a torn write", 0L, seg.tornTailBytes());
+            } catch (MmapSegmentException skippedUnderC2) {
+                // C2: the inlined fault escaped to openExisting's outer catch and
+                // was converted to a per-file skip. Assert it is the recognized
+                // mmap fault (not some other data error) so a revert -- which
+                // lets a raw InternalError through instead -- still fails here.
+                assertTrue(skippedUnderC2.getMessage(),
+                        skippedUnderC2.getMessage().contains("unsafe memory access operation"));
+            }
+        });
+    }
+
+    /**
+     * Portable fail-on-revert guard for the {@code openExisting} header-block
+     * guard. The file is truncated to empty and the facade reports a full page,
+     * so the very first header read (magic) lands on a beyond-EOF page and
+     * faults on any filesystem. {@code openExisting} must convert that to a
+     * {@link MmapSegmentException} -- the per-file signal {@code SegmentRing}
+     * skips on -- not let the raw {@code InternalError} escape and abort recovery
+     * of every sibling. Revert the header-block conversion and this throws
+     * {@code InternalError} instead of {@code MmapSegmentException}.
+     */
+    @Test
+    public void testHeaderFaultOnMapPastEofIsSkippableAnyFilesystem() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            final String path = tmpDir + "/seg-mappasteof-header.sfa";
+            final long page = Files.PAGE_SIZE;
+            writeSegment(path, 9L, new int[]{64});
+            // Free every block: the file is now empty, so even page 0 (the
+            // header) is beyond EOF under the reported one-page mapping.
+            truncateTo(path, 0L);
+            FilesFacade ff = new MapPastEofFacade(path, page);
+            try {
+                MmapSegment.openExisting(ff, path).close();
+                fail("expected MmapSegmentException for a beyond-EOF header page");
+            } catch (MmapSegmentException expected) {
+                // ok -- SegmentRing's per-file catch skips just this file
+                // instead of aborting recovery of the whole slot.
+            }
+        });
+    }
+
     /**
      * Creates a segment at {@code path} and appends one frame per entry in
      * {@code payloadLens} (each filled with non-zero bytes so recovery can tell
@@ -231,4 +334,167 @@ private static void punchSparseTail(String path, long keepBytes) {
             Files.close(fd);
         }
     }
+
+    /**
+     * Shrinks the file to {@code keepBytes}, freeing every block past it, and
+     * leaves it there (no re-extend). Combined with a facade that reports a
+     * larger length, the freed region becomes a beyond-EOF part of the mapping
+     * that faults on read on any filesystem.
+     */
+    private static void truncateTo(String path, long keepBytes) {
+        int fd = Files.openRW(path);
+        assertTrue("openRW failed", fd >= 0);
+        try {
+            assertTrue("truncate failed", Files.truncate(fd, keepBytes));
+        } finally {
+            Files.close(fd);
+        }
+    }
+
+    /**
+     * A {@link FilesFacade} that reports an inflated stat length for one target
+     * path so {@code openExisting} maps that file past end-of-file (see
+     * {@link FilesFacade#length(String)}); every other call, including
+     * {@code length} for any other path, delegates to the production
+     * {@link FilesFacade#INSTANCE}.
+     */
+    private static final class MapPastEofFacade implements FilesFacade {
+        private final long reportedLength;
+        private final String targetPath;
+
+        MapPastEofFacade(String targetPath, long reportedLength) {
+            this.targetPath = targetPath;
+            this.reportedLength = reportedLength;
+        }
+
+        @Override
+        public boolean allocate(int fd, long size) {
+            return INSTANCE.allocate(fd, size);
+        }
+
+        @Override
+        public long allocNativePath(String path) {
+            return INSTANCE.allocNativePath(path);
+        }
+
+        @Override
+        public int close(int fd) {
+            return INSTANCE.close(fd);
+        }
+
+        @Override
+        public boolean exists(String path) {
+            return INSTANCE.exists(path);
+        }
+
+        @Override
+        public void findClose(long findPtr) {
+            INSTANCE.findClose(findPtr);
+        }
+
+        @Override
+        public long findFirst(String dir) {
+            return INSTANCE.findFirst(dir);
+        }
+
+        @Override
+        public long findName(long findPtr) {
+            return INSTANCE.findName(findPtr);
+        }
+
+        @Override
+        public int findNext(long findPtr) {
+            return INSTANCE.findNext(findPtr);
+        }
+
+        @Override
+        public int findType(long findPtr) {
+            return INSTANCE.findType(findPtr);
+        }
+
+        @Override
+        public void freeNativePath(long pathPtr) {
+            INSTANCE.freeNativePath(pathPtr);
+        }
+
+        @Override
+        public int fsync(int fd) {
+            return INSTANCE.fsync(fd);
+        }
+
+        @Override
+        public long length(int fd) {
+            return INSTANCE.length(fd);
+        }
+
+        @Override
+        public long length(long pathPtr) {
+            return INSTANCE.length(pathPtr);
+        }
+
+        @Override
+        public long length(String path) {
+            return targetPath.equals(path) ? reportedLength : INSTANCE.length(path);
+        }
+
+        @Override
+        public int lock(int fd) {
+            return INSTANCE.lock(fd);
+        }
+
+        @Override
+        public int mkdir(String path, int mode) {
+            return INSTANCE.mkdir(path, mode);
+        }
+
+        @Override
+        public int openCleanRW(String path) {
+            return INSTANCE.openCleanRW(path);
+        }
+
+        @Override
+        public int openCleanRW(long pathPtr) {
+            return INSTANCE.openCleanRW(pathPtr);
+        }
+
+        @Override
+        public int openRW(String path) {
+            return INSTANCE.openRW(path);
+        }
+
+        @Override
+        public int openRW(long pathPtr) {
+            return INSTANCE.openRW(pathPtr);
+        }
+
+        @Override
+        public long read(int fd, long addr, long len, long offset) {
+            return INSTANCE.read(fd, addr, len, offset);
+        }
+
+        @Override
+        public boolean remove(String path) {
+            return INSTANCE.remove(path);
+        }
+
+        @Override
+        public boolean remove(long pathPtr) {
+            return INSTANCE.remove(pathPtr);
+        }
+
+        @Override
+        public int rename(String oldPath, String newPath) {
+            return INSTANCE.rename(oldPath, newPath);
+        }
+
+        @Override
+        public boolean truncate(int fd, long size) {
+            return INSTANCE.truncate(fd, size);
+        }
+
+        @Override
+        public long write(int fd, long addr, long len, long offset) {
+            return INSTANCE.write(fd, addr, len, offset);
+        }
+    }
 }

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java

@@ -563,6 +563,11 @@ public long length(int fd) {
             return INSTANCE.length(fd);
         }
 
+        @Override
+        public long length(String path) {
+            return INSTANCE.length(path);
+        }
+
         @Override
         public int lock(int fd) {
             return INSTANCE.lock(fd);