Skip to content

Commit 104bc02

Browse files
puzpuzpuzclaude
puzpuzpuz
and
claude
committed
Survive an unbacked mapped page during segment recovery
MmapSegment.openExisting maps a recovered .sfa to its stat length and scanFrames/detectTornTail read the mapping directly. When a prior session left a sparse segment tail -- a truncate-based pre-allocation that never materialized the tail blocks, as happens on ZFS -- a read of an unbacked page raises the JVM's recoverable InternalError ("a fault occurred in an unsafe memory access operation"), a translated SIGBUS. That InternalError is not a MmapSegmentException, so SegmentRing.open Existing's per-file skip did not catch it: it propagated to the outer catch, aborted recovery of the whole slot, and surfaced (via a drainer or a slot-lock probe) as a spurious "unsafe memory access" failure on ZFS CI runners. scanFrames and detectTornTail now catch the mmap access fault and treat the unreadable region exactly like unwritten space or a torn tail: the boundary of recoverable data. Every frame verified below the fault is kept; the slot recovers instead of failing. isMmapAccessFault matches the JVM's stable message so a genuine VirtualMachineError (real OOM, StackOverflow) is never swallowed. MmapSegmentRecoveryFaultTest reproduces the fault deterministically on any filesystem: it maps a valid one-frame segment, then truncates the backing under the still-larger mapping so the tail page is beyond EOF (the one case POSIX mmap always faults on), and asserts the scan stops at that page and keeps the frame below it. The test errors with the exact InternalError before the fix and passes after. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 24fbbd5 commit 104bc02

2 files changed

Lines changed: 228 additions & 17 deletions

File tree

‎core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,27 @@ public long tornTailBytes() {
506506
return tornTailBytes;
507507
}
508508

509+
/**
510+
* True when {@code t} is the JVM's recoverable signal for a fault while
511+
* accessing a memory-mapped region -- a SIGBUS/SIGSEGV that HotSpot
512+
* translates into {@code InternalError("a fault occurred in an unsafe
513+
* memory access operation")} instead of aborting the process. It surfaces
514+
* when a mapped page is not backed by real file blocks: a sparse
515+
* {@code .sfa} tail on a filesystem whose pre-allocation leaves holes
516+
* (e.g. ZFS, where a truncate-based {@code allocate} does not materialize
517+
* blocks), or a file truncated under the mapping. Recovery treats this as
518+
* an I/O boundary -- the same way MappedByteBuffer readers do -- not a
519+
* fatal VM error. Matches on the JVM's stable message so a genuine
520+
* VirtualMachineError (real OOM, StackOverflow) is never swallowed.
521+
*/
522+
private static boolean isMmapAccessFault(Throwable t) {
523+
if (!(t instanceof InternalError)) {
524+
return false;
525+
}
526+
String msg = t.getMessage();
527+
return msg != null && msg.contains("a fault occurred in an unsafe memory access");
528+
}
529+
509530
/**
510531
* Forward scan that returns the offset just past the last frame whose
511532
* CRC verifies. A torn-tail frame (declared length runs past EOF, or
@@ -515,22 +536,43 @@ public long tornTailBytes() {
515536
*/
516537
private static long scanFrames(long addr, long fileSize) {
517538
long pos = HEADER_SIZE;
518-
while (pos + FRAME_HEADER_SIZE <= fileSize) {
519-
int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
520-
int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
521-
// Defensive: a corrupt length field could be enormous or negative,
522-
// both of which would otherwise overrun the mapping.
523-
if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
524-
return pos;
525-
}
526-
int crcCalc = Crc32c.update(Crc32c.INIT, addr + pos + 4, 4);
527-
if (payloadLen > 0) {
528-
crcCalc = Crc32c.update(crcCalc, addr + pos + FRAME_HEADER_SIZE, payloadLen);
539+
try {
540+
while (pos + FRAME_HEADER_SIZE <= fileSize) {
541+
int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
542+
int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
543+
// Defensive: a corrupt length field could be enormous or negative,
544+
// both of which would otherwise overrun the mapping.
545+
if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
546+
return pos;
547+
}
548+
int crcCalc = Crc32c.update(Crc32c.INIT, addr + pos + 4, 4);
549+
if (payloadLen > 0) {
550+
crcCalc = Crc32c.update(crcCalc, addr + pos + FRAME_HEADER_SIZE, payloadLen);
551+
}
552+
if (crcCalc != crcRead) {
553+
return pos;
554+
}
555+
pos += FRAME_HEADER_SIZE + payloadLen;
529556
}
530-
if (crcCalc != crcRead) {
531-
return pos;
557+
} catch (InternalError e) {
558+
// The read at `pos` hit a mapped page that is not backed by real
559+
// file blocks: the JVM translates the underlying SIGBUS into a
560+
// recoverable InternalError instead of aborting the process. This
561+
// happens when a prior session left a sparse segment tail (a
562+
// truncate-based pre-allocation that does not materialize blocks,
563+
// as on ZFS) or the file was truncated under the mapping. Every
564+
// frame below `pos` already verified; treat the unreadable region
565+
// exactly like unwritten space or a torn tail -- the boundary of
566+
// recoverable data -- rather than letting the error abort recovery
567+
// of the whole slot. Anything that is not the documented mmap
568+
// access fault is a genuine VM error, so rethrow it.
569+
if (!isMmapAccessFault(e)) {
570+
throw e;
532571
}
533-
pos += FRAME_HEADER_SIZE + payloadLen;
572+
LOG.warn("SF segment recovery: unreadable mapped page at offset {} (file size {}); "
573+
+ "treating it as the end of recoverable data. Expected when a prior "
574+
+ "session left a sparse segment tail; investigate disk health if it recurs.",
575+
pos, fileSize);
534576
}
535577
return pos;
536578
}
@@ -553,10 +595,21 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
553595
return 0L;
554596
}
555597
long probe = Math.min(FRAME_HEADER_SIZE, fileSize - lastGood);
556-
for (long i = 0; i < probe; i++) {
557-
if (Unsafe.getUnsafe().getByte(addr + lastGood + i) != 0) {
558-
return fileSize - lastGood;
598+
try {
599+
for (long i = 0; i < probe; i++) {
600+
if (Unsafe.getUnsafe().getByte(addr + lastGood + i) != 0) {
601+
return fileSize - lastGood;
602+
}
603+
}
604+
} catch (InternalError e) {
605+
// The bail-out region is an unbacked (sparse) mapped page -- see
606+
// scanFrames for the mechanism. An unbacked hole was never written,
607+
// so it is clean unwritten space, not a torn write. Rethrow any
608+
// error that is not the recoverable mmap access fault.
609+
if (!isMmapAccessFault(e)) {
610+
throw e;
559611
}
612+
return 0L;
560613
}
561614
return 0L;
562615
}

‎core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2026 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.client.test.cutlass.qwp.client.sf.cursor;
26+
27+
import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment;
28+
import io.questdb.client.std.Files;
29+
import io.questdb.client.std.MemoryTag;
30+
import io.questdb.client.std.Unsafe;
31+
import io.questdb.client.test.tools.TestUtils;
32+
import org.junit.After;
33+
import org.junit.Before;
34+
import org.junit.Test;
35+
36+
import java.lang.reflect.Method;
37+
import java.nio.file.Paths;
38+
39+
import static org.junit.Assert.assertEquals;
40+
import static org.junit.Assert.assertTrue;
41+
42+
/**
43+
* Regression guard for the recovery-time SIGBUS hazard in {@link MmapSegment}.
44+
* <p>
45+
* {@code openExisting} maps a recovered {@code .sfa} to its stat length and
46+
* {@code scanFrames} / {@code detectTornTail} read the mapping directly. When a
47+
* prior session left a sparse segment tail -- a truncate-based pre-allocation
48+
* that never materialized the tail blocks, as happens on ZFS -- a read of an
49+
* unbacked page raises the JVM's recoverable
50+
* {@code InternalError("a fault occurred in an unsafe memory access
51+
* operation")} (a translated SIGBUS). That error is NOT a
52+
* {@code MmapSegmentException}, so {@code SegmentRing.openExisting}'s per-file
53+
* skip did not catch it: it aborted recovery of the whole slot, which surfaced
54+
* (via a drainer/probe) as a spurious "unsafe memory access" failure on ZFS
55+
* CI runners.
56+
* <p>
57+
* The fault only reproduces on a real filesystem whose mmap reads of unwritten
58+
* regions fault instead of zero-filling (ZFS), so this test induces the very
59+
* same JVM-recoverable fault deterministically on any filesystem: it maps a
60+
* valid segment file, then truncates the backing file under the still-larger
61+
* mapping so the tail page is genuinely beyond EOF (the one case POSIX mmap
62+
* always faults on). The scan must then stop at that page and keep every frame
63+
* below it, exactly as it treats a torn tail.
64+
*/
65+
public class MmapSegmentRecoveryFaultTest {
66+
67+
private static final long SEGMENT_BYTES = 1L << 20;
68+
69+
private String tmpDir;
70+
71+
@Before
72+
public void setUp() {
73+
tmpDir = Paths.get(System.getProperty("java.io.tmpdir"),
74+
"qdb-mmap-recover-" + System.nanoTime()).toString();
75+
assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT));
76+
}
77+
78+
@After
79+
public void tearDown() {
80+
if (tmpDir == null) {
81+
return;
82+
}
83+
long find = Files.findFirst(tmpDir);
84+
if (find > 0) {
85+
try {
86+
int rc = 1;
87+
while (rc > 0) {
88+
String name = Files.utf8ToString(Files.findName(find));
89+
if (name != null && !".".equals(name) && !"..".equals(name)) {
90+
Files.remove(tmpDir + "/" + name);
91+
}
92+
rc = Files.findNext(find);
93+
}
94+
} finally {
95+
Files.findClose(find);
96+
}
97+
}
98+
Files.remove(tmpDir);
99+
}
100+
101+
@Test
102+
public void testRecoveryScanTreatsUnbackedTailAsBoundary() throws Exception {
103+
TestUtils.assertMemoryLeak(() -> {
104+
final String path = tmpDir + "/seg-unbacked-tail.sfa";
105+
// One frame sized so the segment's used region ends exactly on a
106+
// 4 KiB page boundary: HEADER_SIZE + FRAME_HEADER_SIZE + payload.
107+
// Truncating the backing to that boundary then leaves the NEXT
108+
// page entirely beyond EOF -- the deterministic mmap-fault case.
109+
final long pageBoundary = 4096L;
110+
final int payloadLen = (int) (pageBoundary - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE);
111+
long boundary;
112+
long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT);
113+
try {
114+
for (int i = 0; i < payloadLen; i++) {
115+
Unsafe.getUnsafe().putByte(buf + i, (byte) (i | 1)); // all non-zero
116+
}
117+
try (MmapSegment seg = MmapSegment.create(path, 7L, SEGMENT_BYTES)) {
118+
assertEquals(MmapSegment.HEADER_SIZE, seg.tryAppend(buf, payloadLen));
119+
boundary = seg.publishedOffset();
120+
}
121+
} finally {
122+
Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT);
123+
}
124+
assertEquals("frame must fill exactly one page", pageBoundary, boundary);
125+
126+
// Map the whole segment, then shrink the backing file under the
127+
// mapping so [boundary, SEGMENT_BYTES) is unbacked. Reads within
128+
// [0, boundary) stay valid; a read at/after `boundary` faults with
129+
// the same recoverable InternalError a sparse ZFS tail produces.
130+
int fd = Files.openRW(path);
131+
assertTrue("openRW failed", fd >= 0);
132+
long addr = Files.mmap(fd, SEGMENT_BYTES, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
133+
assertTrue("mmap failed", addr != Files.FAILED_MMAP_ADDRESS);
134+
try {
135+
assertTrue("truncate failed", Files.truncate(fd, boundary));
136+
137+
// scanFrames must keep the frame below the unbacked page and
138+
// return the boundary rather than propagating the fault.
139+
Method scanFrames = MmapSegment.class.getDeclaredMethod(
140+
"scanFrames", long.class, long.class);
141+
scanFrames.setAccessible(true);
142+
long lastGood = (Long) scanFrames.invoke(null, addr, SEGMENT_BYTES);
143+
assertEquals("scan must stop at the unbacked-page boundary", boundary, lastGood);
144+
145+
// detectTornTail probes the bail-out region -- itself unbacked
146+
// here -- and must report clean (0), not a fatal fault.
147+
Method detectTornTail = MmapSegment.class.getDeclaredMethod(
148+
"detectTornTail", long.class, long.class, long.class);
149+
detectTornTail.setAccessible(true);
150+
long torn = (Long) detectTornTail.invoke(null, addr, lastGood, SEGMENT_BYTES);
151+
assertEquals("unbacked tail is unwritten space, not a torn write", 0L, torn);
152+
} finally {
153+
Files.munmap(addr, SEGMENT_BYTES, MemoryTag.MMAP_DEFAULT);
154+
Files.close(fd);
155+
}
156+
});
157+
}
158+
}

0 commit comments

Comments
 (0)