diff --git a/core/src/main/java/io/questdb/client/QuestDBBuilder.java b/core/src/main/java/io/questdb/client/QuestDBBuilder.java
index 7a037698..0b73541b 100644
--- a/core/src/main/java/io/questdb/client/QuestDBBuilder.java
+++ b/core/src/main/java/io/questdb/client/QuestDBBuilder.java
@@ -73,6 +73,13 @@ public QuestDBBuilder acquireTimeoutMillis(long millis) {
* connections in each pool; further slots are allocated lazily up to
* {@code max} when load demands and reaped back to {@code min} when
* idle.
+ *
+ * Non-blocking on startup recovery: when store-and-forward is enabled,
+ * unacked data a previous run left in this pool's managed slots is
+ * recovered on a background housekeeper thread shortly after this method
+ * returns -- so {@code build()} does not block on a slow or
+ * reachable-but-not-acking server. The recovered data is durable on disk
+ * and is delivered once the server acks; until then it stays preserved.
*/
public QuestDB build() {
if (ingestConfig == null) {
diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java
index 8e9513b1..bd536e6b 100644
--- a/core/src/main/java/io/questdb/client/Sender.java
+++ b/core/src/main/java/io/questdb/client/Sender.java
@@ -1022,6 +1022,15 @@ final class LineSenderBuilder {
// runtime lands in a follow-up commit. For now we surface the
// count via logging so users can confirm orphans are being seen.
private boolean drainOrphans = false;
+ // Orphan-scan exclusion for the connection pool. The pool co-manages
+ // exactly - for i in [0, orphanDrainSlotCount) and
+ // recovers each of those on (re)creation, so pooled senders must never
+ // treat one another's live slots as drainable orphans. Anything else --
+ // a different base, a bare un-suffixed id, OR a same-base index at or
+ // above the count (a slot left behind by a larger pool before maxSize
+ // shrank) -- is still drained, so unacked data is never stranded.
+ private String orphanDrainBase;
+ private int orphanDrainSlotCount;
private long durableAckKeepaliveIntervalMillis = DURABLE_ACK_KEEPALIVE_NOT_SET;
// Optional user-supplied async error handler. When null, the sender
// uses DefaultSenderErrorHandler.INSTANCE (loud-not-silent log).
@@ -1472,7 +1481,15 @@ public Sender build() {
} else {
if (!Files.exists(sfDir)) {
int rc = Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT);
- if (rc != 0) {
+ // mkdir is non-zero on failure, but "already exists"
+ // is one such failure. Multiple SF senders sharing one
+ // sf_dir can be built concurrently (the pool calls
+ // build() outside its lock), so two threads can both
+ // pass the exists() check and race into mkdir; the
+ // loser gets EEXIST. Treat a benign creation race --
+ // the dir now exists -- as success and only fail when
+ // the directory is genuinely absent afterwards.
+ if (rc != 0 && !Files.exists(sfDir)) {
throw new LineSenderException(
"could not create sf_dir: " + sfDir + " rc=" + rc);
}
@@ -1548,7 +1565,7 @@ public Sender build() {
if (drainOrphans && sfDir != null) {
io.questdb.client.std.ObjList orphans =
io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner
- .scan(sfDir, senderId);
+ .scan(sfDir, senderId, orphanDrainBase, orphanDrainSlotCount);
if (orphans.size() > 0) {
org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class)
.info("dispatching drainers for {} orphan slot(s) under {} "
@@ -2471,6 +2488,71 @@ public LineSenderBuilder senderId(String id) {
return this;
}
+ /**
+ * The slot id ({@code sender_id}) currently configured on this
+ * builder, either parsed from the config string or left at its
+ * {@code "default"} default. Introspection hook for the connection
+ * pool, which derives a distinct per-slot id from this base so that
+ * multiple pooled senders sharing one {@code sf_dir} don't collide
+ * on the slot {@code flock}.
+ */
+ public String getConfiguredSenderId() {
+ return senderId;
+ }
+
+ /**
+ * The store-and-forward group root ({@code sf_dir}) currently
+ * configured on this builder, or {@code null} when SF is disabled.
+ * Introspection hook for the connection pool, which needs the group
+ * root to locate its own managed slot dirs {@code /-}
+ * when recovering unacked data a previous run left behind.
+ */
+ public String getConfiguredSfDir() {
+ return sfDir;
+ }
+
+ /**
+ * Excludes the connection pool's live slot set from
+ * {@link #drainOrphans(boolean)} scanning: a sibling slot under
+ * {@code sf_dir} named {@code -} with
+ * {@code 0 <= index < slotCount} is never treated as a drainable orphan.
+ *
+ * Internal introspection hook for the connection pool. The pool gives
+ * each pooled SF sender a distinct slot id {@code -} and
+ * recovers each slot's unacked data itself when it (re)creates that
+ * slot. Without this exclusion, one pooled sender's startup drainer
+ * could adopt a sibling pool slot's lock and dir, reintroducing the
+ * very "sf slot already in use" collision the per-slot ids were added
+ * to prevent.
+ *
+ * Unlike a blanket {@code -} prefix exclusion, the bound is the
+ * pool's {@code maxSize}: a same-base slot whose index is at or above
+ * {@code slotCount} (e.g. {@code -3} left behind by a larger pool
+ * before {@code maxSize} shrank from 4 to 2) is NOT excluded and is
+ * drained like any foreign leftover, so its unacked data is recovered
+ * instead of being silently stranded. Foreign leftovers (a different
+ * base, or a bare un-suffixed id) are also still drained.
+ *
+ * Pass a {@code null}/empty base or {@code slotCount <= 0} to disable
+ * the exclusion (the default).
+ */
+ public LineSenderBuilder orphanDrainExcludeManagedSlots(String base, int slotCount) {
+ this.orphanDrainBase = base;
+ this.orphanDrainSlotCount = slotCount;
+ return this;
+ }
+
+ /**
+ * True iff store-and-forward is enabled (an {@code sf_dir} was set).
+ * Introspection hook for the connection pool: SF senders own an
+ * exclusive on-disk slot, so each pooled sender needs its own slot
+ * id, whereas non-SF (memory-mode / HTTP / TCP) senders share no
+ * such resource and need no per-slot identity.
+ */
+ public boolean isStoreAndForwardEnabled() {
+ return sfDir != null;
+ }
+
/**
* Per-call deadline for {@code Sender.flush()} spinning on a full
* cursor segment ring waiting for ACKs to drain space. Default
diff --git a/core/src/main/java/io/questdb/client/cutlass/line/http/LineHttpSenderV3.java b/core/src/main/java/io/questdb/client/cutlass/line/http/LineHttpSenderV3.java
index e5fc31b4..eff6acd5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/line/http/LineHttpSenderV3.java
+++ b/core/src/main/java/io/questdb/client/cutlass/line/http/LineHttpSenderV3.java
@@ -129,7 +129,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) {
} catch (NumericException e) {
throw new LineSenderException("Failed to parse sent decimal value: " + value, e);
}
- var request = writeFieldName(name);
+ HttpClient.Request request = writeFieldName(name);
request.put(value).putAscii('d');
return this;
}
@@ -139,7 +139,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) {
if (value == null || value.isNull()) {
return this;
}
- var request = writeFieldName(name)
+ HttpClient.Request request = writeFieldName(name)
.putAscii('=')
.put(EntityTypes.DECIMAL)
.put((byte) value.getScale())
@@ -156,7 +156,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) {
if (value == null || value.isNull()) {
return this;
}
- var request = writeFieldName(name)
+ HttpClient.Request request = writeFieldName(name)
.putAscii('=')
.put(EntityTypes.DECIMAL)
.put((byte) value.getScale())
@@ -171,7 +171,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) {
if (value == null || value.isNull()) {
return this;
}
- var request = writeFieldName(name)
+ HttpClient.Request request = writeFieldName(name)
.putAscii('=')
.put(EntityTypes.DECIMAL)
.put((byte) value.getScale())
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java
index ccc6990b..c5c292d3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -24,6 +24,8 @@
package io.questdb.client.cutlass.qwp.client;
+import io.questdb.client.std.Compat;
+
import java.util.concurrent.locks.LockSupport;
/**
@@ -112,7 +114,6 @@ public T poll() {
* {@link InterruptedException} when the consumer thread was interrupted
* while waiting.
*/
- @SuppressWarnings("unchecked")
public T take() throws InterruptedException {
// Fast path: the producer beat us here and the value is already visible.
T value = poll();
@@ -123,7 +124,7 @@ public T take() throws InterruptedException {
// inside the window. onSpinWait lets the CPU slow its pipeline while
// waiting -- meaningful on hyperthreaded cores.
for (int i = 0; i < SPIN_ITERATIONS; i++) {
- Thread.onSpinWait();
+ Compat.onSpinWait();
if ((value = poll()) != null) {
return value;
}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index e34a1923..6735e7ac 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -67,6 +67,7 @@
import java.time.Instant;
import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -234,6 +235,12 @@ public class QwpWebSocketSender implements Sender {
private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF;
private boolean ownsCursorEngine;
private long pendingBytes;
+ // Set true by close() once the SF slot flock has been released (the normal
+ // teardown path). Stays false if close() bailed early with the I/O thread
+ // still running -- then cursorEngine.close() never ran and the flock is
+ // still held, so the owning pool MUST keep the slot reserved rather than
+ // hand the still-locked dir to the next borrow ("sf slot already in use").
+ private boolean slotLockReleased;
private int pendingRowCount;
private SenderProgressDispatcher progressDispatcher;
// Async-delivery sink for ack-watermark advances. Default no-op; a
@@ -279,7 +286,7 @@ private QwpWebSocketSender(
if (endpoints == null || endpoints.isEmpty()) {
throw new IllegalArgumentException("endpoints must be non-empty");
}
- this.endpoints = List.copyOf(endpoints);
+ this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints));
this.hostTracker = new QwpHostHealthTracker(this.endpoints.size());
this.authorizationHeader = authorizationHeader;
this.tlsConfig = tlsConfig;
@@ -1087,6 +1094,10 @@ public void close() {
cursorEngine = null;
ownsCursorEngine = false;
}
+ // Past the ioThreadStopped guard => cursorEngine.close() ran and
+ // released the SF flock in its finally (or this sender owned no
+ // engine holding one). Signal the pool it may reuse the slot.
+ slotLockReleased = true;
// Shutdown order: dispatcher last, after the I/O loop has stopped
// producing into it. close() drains pending entries with a short
@@ -1129,6 +1140,16 @@ public void close() {
}
}
+ /**
+ * True once {@link #close()} has released the store-and-forward slot
+ * flock. False means close() leaked the still-running I/O thread (and its
+ * resources), so the flock is still held; the owning pool must keep the
+ * slot index reserved instead of reusing the still-locked slot dir.
+ */
+ public boolean isSlotLockReleased() {
+ return slotLockReleased;
+ }
+
@Override
public Sender decimalColumn(CharSequence name, Decimal64 value) {
checkNotClosed();
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainerPool.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainerPool.java
index eab701ba..458a4b9a 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainerPool.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainerPool.java
@@ -24,6 +24,7 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
+import io.questdb.client.std.Compat;
import io.questdb.client.std.ObjList;
import io.questdb.client.std.QuietCloseable;
import org.slf4j.Logger;
@@ -122,7 +123,7 @@ public void close() {
// ensures every submit's executor.submit has already returned
// before we shut the executor down.
while (state.get() != CLOSED_BIT) {
- Thread.onSpinWait();
+ Compat.onSpinWait();
}
// Reject new tasks but let in-flight drainers finish their drain
// naturally. Without this grace window a drainer that's seconds
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index e23b2252..94322e9c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -24,6 +24,7 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
+import io.questdb.client.std.Compat;
import io.questdb.client.std.Files;
import io.questdb.client.std.ObjList;
import io.questdb.client.std.QuietCloseable;
@@ -412,7 +413,7 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos
// The spin tightens the gap between manager-installs-spare and
// producer-consumes-spare — usually a few µs on an idle manager thread.
while (System.nanoTime() < spinDeadlineNanos) {
- Thread.onSpinWait();
+ Compat.onSpinWait();
fsn = ring.appendOrFsn(payloadAddr, payloadLen);
if (fsn >= 0 || fsn == SegmentRing.PAYLOAD_TOO_LARGE) {
return fsn;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
index ba29779d..833960ec 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
@@ -25,6 +25,7 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
import io.questdb.client.std.Files;
+import io.questdb.client.std.IntList;
import io.questdb.client.std.ObjList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,13 +76,51 @@ private OrphanScanner() {
* "no orphans" answer in that case.
*/
public static ObjList scan(String sfDir, String excludeSlotName) {
+ // Thin delegate to the managed-aware scan: a null managedBase disables
+ // managed-slot exclusion, so this is exactly the unmanaged scan. Kept
+ // as a convenience overload for callers (and tests) with no pool-minted
+ // slot namespace to skip.
+ return scan(sfDir, excludeSlotName, null, 0);
+ }
+
+ /**
+ * As {@link #scan(String, String)}, but excludes only the exact
+ * set of slot dirs a connection pool can re-create and self-recover:
+ * {@code -} for {@code 0 <= i < managedSlotCount}.
+ *
+ * The exclusion is bounded to the canonical pool-minted slots rather than
+ * the whole {@code -} namespace, so unacked data is never
+ * stranded after a {@code maxSize} shrink across restarts: a slot like
+ * {@code -3} left over from a larger pool is neither re-created (out
+ * of the new {@code [0,maxSize)} index range) nor silently excluded. By
+ * bounding the exclusion to {@code [0,managedSlotCount)},
+ * any same-base slot with an index at or above {@code managedSlotCount} is
+ * treated like a foreign leftover and becomes a drainable orphan. Its data
+ * is recovered either by the pool's startup recovery (which adopts these
+ * out-of-range same-base slots regardless of {@code drain_orphans}; see
+ * {@link #listStrandedOutOfRangeManagedSlots}) or, when
+ * {@code drain_orphans=on}, by the per-sender drain path -- never silently
+ * stranded.
+ *
+ * Only canonical, pool-minted names are excluded: the suffix after
+ * {@code -} must be a canonical non-negative decimal
+ * ({@code 0,1,2,...} with no leading zeros, sign, or non-digits). Anything
+ * else under the same base ({@code -foo}, {@code -007}) is not a
+ * name the pool creates and is reported as a candidate.
+ *
+ * When {@code managedBase} is null/empty or {@code managedSlotCount <= 0}
+ * no exclusion is applied (every sibling with data is a candidate).
+ */
+ public static ObjList scan(String sfDir, String excludeSlotName, String managedBase, int managedSlotCount) {
ObjList orphans = new ObjList<>();
if (sfDir == null || !Files.exists(sfDir)) {
return orphans;
}
+ boolean hasManaged = managedBase != null && !managedBase.isEmpty() && managedSlotCount > 0;
+ String managedPrefix = hasManaged ? managedBase + "-" : null;
long find = Files.findFirst(sfDir);
if (find < 0) {
- LOG.warn("orphan scan could not enumerate {} — treating as no orphans, "
+ LOG.warn("orphan scan could not enumerate {} \u2014 treating as no orphans, "
+ "but this may indicate a permission or transient error", sfDir);
return orphans;
}
@@ -99,6 +138,9 @@ public static ObjList scan(String sfDir, String excludeSlotName) {
if (excludeSlotName != null && excludeSlotName.equals(name)) {
continue;
}
+ if (hasManaged && isManagedSlot(name, managedPrefix, managedSlotCount)) {
+ continue;
+ }
String slotPath = sfDir + "/" + name;
if (!isCandidateOrphan(slotPath)) {
continue;
@@ -111,6 +153,116 @@ public static ObjList scan(String sfDir, String excludeSlotName) {
return orphans;
}
+ /**
+ * Lists the canonical indices of this pool's OWN same-base slots that a
+ * previous run left behind out of the current index range, i.e.
+ * {@code -} for {@code i >= managedSlotCount} that still
+ * hold unacked data (no failure sentinel).
+ *
+ * These are not foreign orphans: they are the pool's own canonical,
+ * pool-minted slots from a run with a larger {@code maxSize}. Because the
+ * current run's index range is {@code [0, managedSlotCount)} they are never
+ * re-created and so never recovered by the pool's normal (re)creation path,
+ * yet their unacked data is durable on disk. Startup recovery uses this to
+ * adopt and drain them once, at construction -- so their data is delivered
+ * under the default config, without waiting for {@code drain_orphans=on}.
+ *
+ * Only exact, pool-minted names match: the suffix after
+ * {@code -} must be a canonical non-negative decimal
+ * ({@code 0,1,2,...} with no leading zeros, sign, or non-digits). Anything
+ * else under the same base is a foreign leftover, not a slot the pool
+ * created, and is left to the {@code drain_orphans} path.
+ *
+ * @return canonical indices ({@code >= managedSlotCount}) of same-base
+ * slots holding unacked data; empty when none, or when inputs are invalid
+ */
+ public static IntList listStrandedOutOfRangeManagedSlots(String sfDir, String managedBase, int managedSlotCount) {
+ IntList out = new IntList();
+ if (sfDir == null || managedBase == null || managedBase.isEmpty()
+ || managedSlotCount < 0 || !Files.exists(sfDir)) {
+ return out;
+ }
+ String managedPrefix = managedBase + "-";
+ long find = Files.findFirst(sfDir);
+ if (find < 0) {
+ LOG.warn("orphan scan could not enumerate {} \u2014 treating as no stranded slots, "
+ + "but this may indicate a permission or transient error", sfDir);
+ return out;
+ }
+ if (find == 0) {
+ return out;
+ }
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ rc = Files.findNext(find);
+ if (name == null || ".".equals(name) || "..".equals(name)) {
+ continue;
+ }
+ if (!name.startsWith(managedPrefix)) {
+ continue;
+ }
+ int idx = parseCanonicalIndex(name, managedPrefix.length());
+ if (idx < managedSlotCount) {
+ // Negative (non-canonical) or in-range: not our concern here.
+ continue;
+ }
+ if (!isCandidateOrphan(sfDir + "/" + name)) {
+ continue;
+ }
+ out.add(idx);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ return out;
+ }
+
+ /**
+ * True iff {@code name} is a slot the pool actively co-manages, i.e.
+ * {@code } where {@code i} is a canonical non-negative
+ * decimal in {@code [0, managedSlotCount)}. Visible for testing.
+ */
+ public static boolean isManagedSlot(String name, String managedPrefix, int managedSlotCount) {
+ if (name == null || managedPrefix == null || !name.startsWith(managedPrefix)) {
+ return false;
+ }
+ int idx = parseCanonicalIndex(name, managedPrefix.length());
+ return idx >= 0 && idx < managedSlotCount;
+ }
+
+ /**
+ * Parses the canonical non-negative decimal that makes up the rest of
+ * {@code name} from {@code from}. Returns {@code -1} for an empty suffix,
+ * a non-digit, a leading zero (e.g. {@code "007"}), or anything that would
+ * overflow {@code int}. Only the exact form the pool emits
+ * ({@code Integer.toString(index)}) is accepted, so foreign or malformed
+ * same-base names never get mistaken for a managed slot.
+ */
+ private static int parseCanonicalIndex(String name, int from) {
+ int len = name.length();
+ if (from >= len) {
+ return -1;
+ }
+ // Reject leading zeros unless the whole suffix is exactly "0".
+ if (name.charAt(from) == '0' && len - from > 1) {
+ return -1;
+ }
+ long acc = 0;
+ for (int i = from; i < len; i++) {
+ char c = name.charAt(i);
+ if (c < '0' || c > '9') {
+ return -1;
+ }
+ acc = acc * 10 + (c - '0');
+ if (acc > Integer.MAX_VALUE) {
+ return -1;
+ }
+ }
+ return (int) acc;
+ }
+
/**
* True iff {@code slotPath} looks like a slot dir with unacked data
* and no failure sentinel. Visible for testing.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 3bd8a2ed..0b8379de 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -24,6 +24,7 @@
package io.questdb.client.cutlass.qwp.client.sf.cursor;
+import io.questdb.client.std.Compat;
import io.questdb.client.std.Files;
import io.questdb.client.std.MemoryTag;
import io.questdb.client.std.QuietCloseable;
@@ -154,7 +155,7 @@ private static String readHolder(String pidPath) {
private static void writePid(String pidPath) {
long pid;
try {
- pid = ProcessHandle.current().pid();
+ pid = Compat.currentPid();
} catch (Throwable ignored) {
// Diagnostic-only — never block lock acquisition on it.
pid = -1L;
diff --git a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java
index ecdd6fc9..d5ff3db4 100644
--- a/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java
+++ b/core/src/main/java/io/questdb/client/impl/PoolHousekeeper.java
@@ -31,6 +31,17 @@
*/
final class PoolHousekeeper {
+ // How long stop() waits for the daemon to exit. Kept ABOVE
+ // SenderPool.RECOVERY_DRAIN_BUDGET_MILLIS so a startup-recovery drain still
+ // in flight when close() arrives finishes well within this join (C1 fix).
+ // The recovery build that precedes the drain is bounded separately --
+ // recoverers force initial_connect_mode=OFF, so the build makes at most one
+ // connect attempt rather than a SYNC reconnect-budget retry (M1). The lone
+ // case that can still overrun this join is an in-flight connect to a
+ // black-holed host (no application-level connect timeout in the transport);
+ // see the residual-window note on SenderPool.recoverOneSlotStep.
+ static final long STOP_TIMEOUT_MILLIS = 2_000;
+
private final long intervalMillis;
private final QueryClientPool queryPool;
private final SenderPool senderPool;
@@ -56,7 +67,7 @@ void stop() {
signalLock.notifyAll();
}
try {
- thread.join(2_000);
+ thread.join(STOP_TIMEOUT_MILLIS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -64,15 +75,40 @@ void stop() {
private void runLoop() {
while (!stop) {
+ // Per-slot startup SF recovery, driven on THIS (the reap-loop)
+ // thread, one stranded slot per iteration. Doing it here rather than
+ // in the SenderPool constructor keeps QuestDB.build() from blocking
+ // on a slow or reachable-but-not-acking server. Each step does at
+ // most one drain bounded by SenderPool.RECOVERY_DRAIN_BUDGET_MILLIS
+ // (< STOP_TIMEOUT_MILLIS) on a recoverer whose initial connect is
+ // forced OFF (at most one connect attempt, never a SYNC
+ // reconnect-budget retry -- M1), and we re-check stop every step, so
+ // a close() landing mid-recovery normally only waits out a single
+ // bounded drain and the join in stop() does not time out. The sole
+ // residual overrun is an in-flight connect to a black-holed host;
+ // see SenderPool.recoverOneSlotStep.
+ // While recovery still has work we skip the idle wait so the backlog
+ // drains promptly; once done we fall back to the normal interval.
+ // No-op once recovery completes or the pool is closing. Best-effort:
+ // a recovery failure (including an Error) must never kill this
+ // daemon, so swallow Throwable -- exactly as the reap guards below.
+ boolean recovering;
+ try {
+ recovering = senderPool.runStartupRecoveryStep();
+ } catch (Throwable ignored) {
+ recovering = false;
+ }
synchronized (signalLock) {
if (stop) {
return;
}
- try {
- signalLock.wait(intervalMillis);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
+ if (!recovering) {
+ try {
+ signalLock.wait(intervalMillis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+ }
}
}
if (stop) {
@@ -80,12 +116,21 @@ private void runLoop() {
}
try {
senderPool.reapIdle();
- } catch (RuntimeException ignored) {
- // Reaping must not propagate -- it's best-effort housekeeping.
+ } catch (Throwable ignored) {
+ // Defensive, intentionally unreachable in normal operation:
+ // SenderPool.reapIdle() already swallows per-delegate close()
+ // failures internally. The outer catch is a belt-and-braces
+ // guard. Reaping must not propagate -- it's best-effort
+ // housekeeping. Catch Throwable (not just RuntimeException) so
+ // an Error from a delegate teardown can never kill this daemon
+ // thread and stop all future reaping for the life of the handle.
}
try {
queryPool.reapIdle();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Same rationale as the senderPool guard above: best-effort,
+ // must never propagate, and Throwable (not RuntimeException) so
+ // an Error from query-client teardown cannot kill the daemon.
}
}
}
diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java
index 9e2dbbb6..61d89296 100644
--- a/core/src/main/java/io/questdb/client/impl/PooledSender.java
+++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java
@@ -50,13 +50,21 @@ public final class PooledSender implements Sender {
private final long createdAtMillis;
private final Sender delegate;
private final SenderPool pool;
+ // Index of the store-and-forward slot this wrapper owns within the pool,
+ // or -1 when SF is disabled. Stable for the wrapper's whole life; the
+ // pool returns it to the free set only when the wrapper is evicted from
+ // {@code all} (discardBroken / reapIdle). Used to derive a distinct
+ // {@code sender_id} per pooled sender so concurrent SF senders sharing
+ // one {@code sf_dir} never collide on the slot {@code flock}.
+ private final int slotIndex;
private volatile long idleSinceMillis;
private volatile boolean inUse;
private volatile boolean invalidated;
- PooledSender(Sender delegate, SenderPool pool) {
+ PooledSender(Sender delegate, SenderPool pool, int slotIndex) {
this.delegate = delegate;
this.pool = pool;
+ this.slotIndex = slotIndex;
this.createdAtMillis = System.currentTimeMillis();
this.idleSinceMillis = this.createdAtMillis;
}
@@ -148,17 +156,15 @@ public void close() {
if (!inUse) {
return;
}
- boolean broken = false;
+ // Track normal completion rather than catching a specific throwable
+ // type. flush() can exit abnormally with an Error (AssertionError
+ // under -ea, OutOfMemoryError, ...) as well as a RuntimeException;
+ // keying the recycle decision off normal completion treats every
+ // abnormal exit as unrecyclable, which is the fail-safe default.
+ boolean flushed = false;
try {
delegate.flush();
- } catch (RuntimeException e) {
- // Sender does not clear its buffer on flush failure (see
- // Sender Javadoc), and WebSocket transport latches the failure
- // for good. Either way, the wrapper is unsafe to recycle: the
- // next borrower would inherit the failed rows or a dead
- // connection.
- broken = true;
- throw e;
+ flushed = true;
} finally {
inUse = false;
// Clear the pin BEFORE returning the slot. If we cleared
@@ -167,10 +173,17 @@ public void close() {
// re-pin on this thread would return the (now in-use)
// wrapper -- the same race this clear is meant to close.
pool.clearPinIfCurrent(this);
- if (broken) {
- pool.discardBroken(this);
- } else {
+ if (flushed) {
pool.giveBack(this);
+ } else {
+ // flush() did not complete normally. Sender does not clear
+ // its buffer on flush failure (see Sender Javadoc), and
+ // WebSocket transport latches the failure for good. Either
+ // way the wrapper is unsafe to recycle: the next borrower
+ // would inherit the failed rows or a dead connection. The
+ // original throwable propagates naturally once this finally
+ // returns -- no explicit rethrow needed.
+ pool.discardBroken(this);
}
}
}
@@ -372,6 +385,10 @@ long createdAtMillis() {
return createdAtMillis;
}
+ int slotIndex() {
+ return slotIndex;
+ }
+
Sender delegate() {
return delegate;
}
diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
index 8191f162..a6365dfa 100644
--- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
+++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java
@@ -34,6 +34,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
/**
* Elastic pool of {@link QueryWorker}s. Each worker pairs one
@@ -46,12 +47,26 @@
* outside the pool lock; an {@code inFlightCreations} counter keeps the
* cap check honest under concurrent acquires.
*/
-public final class QueryClientPool {
+public final class QueryClientPool implements AutoCloseable {
private final long acquireTimeoutMillis;
private final ArrayList all;
private final ArrayDeque available;
private final String configurationString;
+ // Test seam. Production connects via QwpQueryClient.connect(); white-box
+ // tests in io.questdb.client.test.impl reach the package-private constructor
+ // by reflection to inject a hook that throws a non-RuntimeException
+ // Throwable (e.g. an -ea AssertionError) from the native connect path,
+ // exercising the Error-safe cleanup on the prewarm and acquire paths.
+ private final Consumer connectHook;
+ // Test seam. Production starts the worker's dispatch thread via
+ // QueryWorker.start(); white-box tests in io.questdb.client.test.impl reach
+ // the package-private constructor by reflection to inject a hook that throws
+ // a Throwable (modelling OutOfMemoryError "unable to create native thread")
+ // *after* createUnlocked() has returned a fully connected client, exercising
+ // the Error-safe client teardown on the prewarm and acquire paths -- the
+ // start()-throws path connectHook cannot reach.
+ private final Consumer startHook;
private final long idleTimeoutMillis;
private final ReentrantLock lock = new ReentrantLock();
private final long maxLifetimeMillis;
@@ -69,10 +84,49 @@ public QueryClientPool(
long acquireTimeoutMillis,
long idleTimeoutMillis,
long maxLifetimeMillis
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, null);
+ }
+
+ // Package-private constructor exposing the connectHook test seam: production
+ // passes null (-> the real QwpQueryClient.connect()). White-box tests in
+ // io.questdb.client.test.impl reach this by reflection to inject a hook that
+ // throws a non-RuntimeException Throwable from the native connect path.
+ QueryClientPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ Consumer connectHook
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, connectHook, null);
+ }
+
+ // Package-private constructor exposing both the connectHook and startHook
+ // test seams: production passes null for each (-> the real
+ // QwpQueryClient.connect() and QueryWorker.start()). White-box tests in
+ // io.questdb.client.test.impl reach this by reflection to inject a hook that
+ // throws a Throwable from either the native connect path (connectHook) or
+ // the worker thread-start path (startHook).
+ QueryClientPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ Consumer connectHook,
+ Consumer startHook
) {
if (minSize < 0 || maxSize < 1 || minSize > maxSize) {
throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize);
}
+ this.connectHook = connectHook != null ? connectHook : QwpQueryClient::connect;
+ this.startHook = startHook != null ? startHook : QueryWorker::start;
this.configurationString = configurationString;
this.minSize = minSize;
this.maxSize = maxSize;
@@ -83,19 +137,48 @@ public QueryClientPool(
this.available = new ArrayDeque<>(maxSize);
this.workerReleased = lock.newCondition();
int built = 0;
+ // Tracks a worker built by createUnlocked() but not yet added to `all`:
+ // it is fully connected (socket + native scratch + I/O thread) the
+ // instant createUnlocked() returns, yet the following start() can still
+ // throw (e.g. OutOfMemoryError creating the dispatch thread). Without
+ // this handle the cleanup loop below -- which only walks `all` -- would
+ // never close it, stranding exactly the I/O thread and native
+ // allocations this catch exists to reclaim.
+ QueryWorker pending = null;
try {
for (int i = 0; i < minSize; i++) {
- QueryWorker w = createUnlocked();
- w.start();
- all.add(w);
- available.add(w);
+ pending = createUnlocked();
+ this.startHook.accept(pending);
+ all.add(pending);
+ available.add(pending);
+ pending = null;
built++;
}
- } catch (RuntimeException e) {
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException: createUnlocked()/start()
+ // run a heavy native build path that can throw an Error -- e.g. an
+ // -ea AssertionError or OutOfMemoryError -- mid-prewarm. If we only
+ // caught RuntimeException the Error would propagate without running
+ // the cleanup below, stranding every already-built worker's I/O
+ // thread and native allocations.
for (int i = 0; i < built; i++) {
try {
all.get(i).shutdown();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort cleanup: an Error (e.g. -ea AssertionError)
+ // from one worker's shutdown must not strand the remaining
+ // pre-warmed workers nor mask the original failure below.
+ }
+ }
+ // Close the worker that was built but never made it into `all`
+ // (start() threw after createUnlocked() returned a live client).
+ // createUnlocked() already self-cleans when connect() throws, so
+ // pending is only non-null on the start()-throws path.
+ if (pending != null) {
+ try {
+ pending.shutdown();
+ } catch (Throwable ignored) {
+ // Best-effort: must not mask the original failure below.
}
}
throw e;
@@ -119,15 +202,37 @@ public QueryWorker acquire() {
if (all.size() + inFlightCreations < maxSize) {
inFlightCreations++;
lock.unlock();
- QueryWorker created;
+ QueryWorker created = null;
try {
created = createUnlocked();
- created.start();
- } catch (RuntimeException e) {
+ startHook.accept(created);
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException:
+ // createUnlocked()/start() run a heavy native build path
+ // that can throw an Error -- e.g. an -ea AssertionError
+ // or OutOfMemoryError. If we only caught RuntimeException
+ // the Error would propagate with inFlightCreations still
+ // incremented, permanently shrinking pool capacity until
+ // every acquire() times out. Restoring the reservation
+ // for any throwable is safe.
lock.lock();
inFlightCreations--;
workerReleased.signal();
lock.unlock();
+ // createUnlocked() returns a fully connected client
+ // (socket + native scratch + I/O thread), so if start()
+ // threw afterwards we must close it here -- nothing else
+ // references it. createUnlocked() already self-cleans
+ // when connect() throws, leaving created == null, so
+ // this only fires on the start()-throws path.
+ if (created != null) {
+ try {
+ created.shutdown();
+ } catch (Throwable ignored) {
+ // Best-effort: a teardown Error must not mask the
+ // original creation failure rethrown below.
+ }
+ }
throw new QueryException((byte) 0,
"failed to create query client: " + e.getMessage(), e);
}
@@ -136,7 +241,9 @@ public QueryWorker acquire() {
if (closed) {
try {
created.shutdown();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: an Error from teardown must not mask
+ // the closed-pool signal.
}
throw new QueryException((byte) 0, "QuestDB handle is closed");
}
@@ -180,7 +287,13 @@ public void close() {
// join the worker threads and close their clients. Done outside the lock
// so a slow join doesn't keep the pool latched.
for (int i = 0; i < snapshot.size(); i++) {
- snapshot.get(i).shutdown();
+ try {
+ snapshot.get(i).shutdown();
+ } catch (Throwable ignored) {
+ // Best-effort: a single worker's shutdown failure (including an
+ // Error such as an -ea AssertionError) must not abort the loop
+ // and strand the remaining workers unclosed.
+ }
}
}
@@ -218,7 +331,10 @@ void reapIdle() {
for (int i = 0, n = toShutdown.size(); i < n; i++) {
try {
toShutdown.get(i).shutdown();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: a single worker's shutdown failure (including
+ // an Error such as an -ea AssertionError) must not abort the
+ // reap loop and strand the remaining reaped workers.
}
}
}
@@ -239,19 +355,36 @@ void release(QueryWorker w) {
}
}
+ // Package-private white-box accessor for tests: reports the current
+ // in-flight creation count under the pool lock. A non-zero value after a
+ // failed acquire() means the slot reservation was never released -- the
+ // capacity-shrink bug this guards against.
+ int inFlightCreations() {
+ lock.lock();
+ try {
+ return inFlightCreations;
+ } finally {
+ lock.unlock();
+ }
+ }
+
private QueryWorker createUnlocked() {
QwpQueryClient client = QwpQueryClient.fromConfig(configurationString);
try {
- client.connect();
- } catch (RuntimeException e) {
- // connect() may throw after QwpQueryClient.fromConfig() has already
+ connectHook.accept(client);
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException: connect() runs a heavy
+ // native path that can throw an Error (e.g. an -ea AssertionError or
+ // OutOfMemoryError) after QwpQueryClient.fromConfig() has already
// allocated native scratch (the QwpBindValues NativeBufferWriter is
// field-initialised). Close the half-built client so its allocations
- // are released, otherwise every connect failure during pool growth
- // leaks NATIVE_DEFAULT bytes.
+ // are released, otherwise an Error during pool growth leaks the
+ // NATIVE_DEFAULT bytes that only this cleanup would reclaim.
try {
client.close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: an Error from closing the half-built client must
+ // not mask the original connect failure being rethrown below.
}
throw e;
}
diff --git a/core/src/main/java/io/questdb/client/impl/QueryWorker.java b/core/src/main/java/io/questdb/client/impl/QueryWorker.java
index 4b251431..f4f641c8 100644
--- a/core/src/main/java/io/questdb/client/impl/QueryWorker.java
+++ b/core/src/main/java/io/questdb/client/impl/QueryWorker.java
@@ -105,21 +105,33 @@ void shutdown() {
} finally {
signalLock.unlock();
}
- // If a query is in flight on this worker, ask the client to abort so
- // execute() returns promptly and the thread can exit before join times
- // out. cancel() is documented as thread-safe and is a no-op when idle.
try {
- client.cancel();
- } catch (RuntimeException ignored) {
- }
- try {
- thread.join(SHUTDOWN_JOIN_MILLIS);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- try {
- client.close();
- } catch (RuntimeException ignored) {
+ // If a query is in flight on this worker, ask the client to abort so
+ // execute() returns promptly and the thread can exit before join
+ // times out. cancel() is documented as thread-safe and is a no-op
+ // when idle.
+ try {
+ client.cancel();
+ } catch (Throwable ignored) {
+ // Best-effort. Catch Throwable, not just RuntimeException: an
+ // Error (e.g. an -ea AssertionError) thrown here must not skip
+ // the join() below or the close() in finally, which would leak
+ // the worker thread and the client's native socket/buffers.
+ }
+ try {
+ thread.join(SHUTDOWN_JOIN_MILLIS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ } finally {
+ // close() must run even if cancel()/join() threw, otherwise the
+ // client's native buffer pool and socket leak for the lifetime of
+ // the process. Catch Throwable so shutdown() itself never propagates
+ // a teardown Error to its callers.
+ try {
+ client.close();
+ } catch (Throwable ignored) {
+ }
}
}
diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
index cc974ac1..5bba8d46 100644
--- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
+++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java
@@ -29,6 +29,10 @@
import io.questdb.client.Query;
import io.questdb.client.Sender;
import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler;
+import io.questdb.client.cutlass.qwp.client.QwpQueryClient;
+
+import java.util.function.Consumer;
+import java.util.function.IntFunction;
/**
* Implementation of {@link QuestDB}. Owns the elastic {@link SenderPool}
@@ -55,6 +59,31 @@ public QuestDBImpl(
long idleTimeoutMillis,
long maxLifetimeMillis,
long housekeeperIntervalMillis
+ ) {
+ this(ingestConfig, queryConfig, senderMin, senderMax, queryMin, queryMax,
+ acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis,
+ housekeeperIntervalMillis, null, null);
+ }
+
+ // Package-private constructor exposing the senderFactory and connectHook test
+ // seams: production passes null for both (-> the real native build/connect
+ // paths). White-box tests in io.questdb.client.test.impl reach this by
+ // reflection (the main module is declared `open`) to make SenderPool prewarm
+ // an observable delegate while QueryClientPool construction throws an Error,
+ // exercising the cleanup catch below.
+ QuestDBImpl(
+ String ingestConfig,
+ String queryConfig,
+ int senderMin,
+ int senderMax,
+ int queryMin,
+ int queryMax,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ long housekeeperIntervalMillis,
+ IntFunction senderFactory,
+ Consumer connectHook
) {
SenderPool builtSenderPool = null;
QueryClientPool builtQueryPool = null;
@@ -62,22 +91,38 @@ public QuestDBImpl(
try {
builtSenderPool = new SenderPool(
ingestConfig, senderMin, senderMax, acquireTimeoutMillis,
- idleTimeoutMillis, maxLifetimeMillis);
+ idleTimeoutMillis, maxLifetimeMillis, senderFactory,
+ // Defer SF startup recovery to the PoolHousekeeper thread so
+ // build() never blocks on a slow / reachable-but-not-acking
+ // server; the housekeeper drives it via runStartupRecoveryStep().
+ true);
builtQueryPool = new QueryClientPool(
queryConfig, queryMin, queryMax, acquireTimeoutMillis,
- idleTimeoutMillis, maxLifetimeMillis);
+ idleTimeoutMillis, maxLifetimeMillis, connectHook);
builtHousekeeper = new PoolHousekeeper(builtSenderPool, builtQueryPool, housekeeperIntervalMillis);
builtHousekeeper.start();
- } catch (RuntimeException e) {
- if (builtHousekeeper != null) {
- builtHousekeeper.stop();
- }
- if (builtQueryPool != null) {
- builtQueryPool.close();
- }
- if (builtSenderPool != null) {
- builtSenderPool.close();
- }
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException: this orchestrator is the
+ // direct caller of the SenderPool and QueryClientPool constructors,
+ // both of which run heavy native build/connect paths that can throw an
+ // Error under -ea (AssertionError, OutOfMemoryError). The pools widened
+ // their own prewarm catches to Throwable for exactly this reason; if we
+ // only caught RuntimeException here, an Error from QueryClientPool
+ // construction (or the housekeeper start) would propagate without
+ // closing the already-built SenderPool, stranding its prewarmed
+ // delegates' flocks, mmap'd rings, and I/O threads -- the precise leak
+ // class this teardown-hardening work exists to kill. The cleanup below
+ // is best-effort and rethrows the original failure.
+ //
+ // Each cleanup step is independently guarded: a Throwable (e.g. an OOM
+ // from QueryClientPool.close()'s `new ArrayList<>(all)`, or an Error
+ // from the housekeeper join) must never skip the remaining closes. The
+ // SenderPool -- owner of the flock/mmap/I/O-thread resources -- is
+ // closed last, so an unguarded earlier failure would strand exactly
+ // the resources this catch exists to reclaim.
+ closeQuietly(builtHousekeeper);
+ closeQuietly(builtQueryPool);
+ closeQuietly(builtSenderPool);
throw e;
}
this.senderPool = builtSenderPool;
@@ -97,9 +142,44 @@ public void close() {
return;
}
closed = true;
- housekeeper.stop();
- queryPool.close();
- senderPool.close();
+ // Cancel any in-flight startup SF recovery BEFORE stopping the
+ // housekeeper: markClosing() raises the pool's shutdown signal so a
+ // recovery step driven on the housekeeper thread bails between slots and
+ // the housekeeper join below cannot time out waiting on a fresh slot's
+ // recovery drain (C1 fix #1). This only raises the flag; full pool
+ // teardown still happens in senderPool.close() further down.
+ senderPool.markClosing();
+ // Independently guarded so a Throwable from one teardown step cannot skip
+ // the rest. senderPool is closed last and owns the flock/mmap/I/O-thread
+ // resources, so it must run even if housekeeper.stop()/queryPool.close()
+ // throws an Error or OOM.
+ closeQuietly(housekeeper);
+ closeQuietly(queryPool);
+ closeQuietly(senderPool);
+ }
+
+ private static void closeQuietly(PoolHousekeeper housekeeper) {
+ if (housekeeper == null) {
+ return;
+ }
+ try {
+ housekeeper.stop();
+ } catch (Throwable ignored) {
+ // Best-effort teardown: never let a stop() failure skip the
+ // subsequent pool closes.
+ }
+ }
+
+ private static void closeQuietly(AutoCloseable closeable) {
+ if (closeable == null) {
+ return;
+ }
+ try {
+ closeable.close();
+ } catch (Throwable ignored) {
+ // Best-effort teardown: never let one close() failure skip the
+ // remaining closes.
+ }
}
@Override
diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java
index 61b6ac69..8c9fda7a 100644
--- a/core/src/main/java/io/questdb/client/impl/SenderPool.java
+++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java
@@ -26,6 +26,12 @@
import io.questdb.client.Sender;
import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.IntList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -33,6 +39,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.IntFunction;
/**
* Elastic pool of {@link Sender} instances, each wrapped in a
@@ -49,24 +56,129 @@
* Connection creation happens outside the lock so a slow connect (TLS
* handshake, DNS) does not block other borrowers or the housekeeper. The
* pool tracks in-flight creations via {@code inFlightCreations} so the cap
- * check ({@code allSize + inFlightCreations < maxSize}) stays correct under
- * concurrent borrows.
+ * check ({@code allSize + inFlightCreations + closingSlots + leakedSlots <
+ * maxSize}) stays correct under concurrent borrows.
+ *
+ * Store-and-forward slots. When the configuration enables SF
+ * ({@code sf_dir} set), every sender owns an exclusive on-disk slot
+ * {@code /} guarded by a {@code flock}. A pool reuses one
+ * immutable config string for every sender, so without intervention all
+ * senders would inherit the same {@code sender_id}, point at the same slot,
+ * and every sender after the first would die with "sf slot already in use".
+ * The pool therefore hands each slot a distinct id {@code -},
+ * where {@code } is the configured {@code sender_id} (default
+ * {@code "default"}) and {@code } is a stable pool slot index in
+ * {@code [0, maxSize)}. Indices are reused deterministically (lowest free
+ * first), so across a restart the same slot dirs are re-adopted and any
+ * unacked data they hold is recovered on creation. A slot is only returned
+ * to the free set once its delegate has released the {@code flock}, tracked
+ * via {@code closingSlots} so a concurrent borrow can never reclaim a slot
+ * dir whose lock is still held.
*/
public final class SenderPool implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(SenderPool.class);
+ // Per-slot wall-clock cap on a single startup-recovery drain. Kept BELOW the
+ // PoolHousekeeper stop/join budget (PoolHousekeeper.STOP_TIMEOUT_MILLIS) so a
+ // recovery drain still in flight when close() arrives cannot outlive the
+ // housekeeper join -- the residual-budget bound that, together with the
+ // early markClosing() signal, keeps close() prompt (C1 fix).
+ //
+ // This caps only the DRAIN. The recovery build that precedes it is bounded
+ // separately: recovery delegates force initial_connect_mode=OFF (see
+ // defaultRecoverySender) so the build does at most ONE connect attempt
+ // rather than a SYNC reconnect-budget retry loop (M1). One in-flight
+ // connect against a black-holed host still blocks on the OS connect timeout
+ // -- the residual window documented on recoverOneSlotStep -- because the
+ // transport has no application-level connect timeout to clamp it.
+ private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000;
private final long acquireTimeoutMillis;
private final ArrayList all;
private final ArrayDeque available;
private final String configurationString;
private final long idleTimeoutMillis;
+ // Test seam. Production builds delegates via defaultSender(); white-box
+ // tests in io.questdb.client.test.impl reach the package-private
+ // constructor by reflection to inject a factory that throws a non-
+ // RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm,
+ // exercising the Error-safe delegate cleanup loop.
+ private final IntFunction senderFactory;
+ // Factory for startup-recovery delegates. Distinct from senderFactory so a
+ // recoverer can force a non-blocking initial connect (initial_connect_mode=
+ // OFF) regardless of user config: a recovery build runs on the
+ // PoolHousekeeper thread and must NOT inherit SYNC (auto-enabled by any
+ // reconnect_* knob), which would retry the connect for the whole reconnect
+ // budget inside build() -- far past PoolHousekeeper.STOP_TIMEOUT_MILLIS, so
+ // a close() landing during that build would make housekeeper.stop()'s join
+ // time out and leave the recoverer holding the slot flock after close()
+ // returned (M1). Mirrors senderFactory's test seam: an injected factory
+ // (non-null) drives BOTH paths so white-box recovery tests keep control.
+ private final IntFunction recoverySenderFactory;
private final ReentrantLock lock = new ReentrantLock();
private final long maxLifetimeMillis;
private final int maxSize;
private final int minSize;
+ // SF slot base id (configured sender_id, default "default") when SF is
+ // enabled; null otherwise. Each pooled sender's slot id is
+ // {@code slotBaseId + "-" + slotIndex}.
+ private final String slotBaseId;
+ // SF group root (sf_dir) when SF is enabled; null otherwise. Used to
+ // locate this pool's own managed slot dirs /-
+ // for startup recovery of unacked data left by a previous run.
+ private final String sfDir;
+ // Reservation bitmap for SF slot indices [0, maxSize). Guarded by lock.
+ // null when SF is disabled (no per-slot identity needed).
+ private final boolean[] slotInUse;
private final Condition slotReleased;
+ // True iff the configuration enables store-and-forward (sf_dir set).
+ private final boolean storeAndForward;
private final ThreadLocal threadAffine = new ThreadLocal<>();
+ // Slots removed from `all` whose delegate is still releasing its flock.
+ // They keep reserving capacity (and their slotInUse mark) until the
+ // flock drops, so the cap check and the slot allocator stay consistent
+ // and no concurrent borrow can reclaim a slot dir that is still locked.
+ // Guarded by lock. Only ever ticks for SF slots.
+ private int closingSlots;
+ // Shutdown signal: "the pool is shutting down". markClosing() raises it early
+ // (without tearing down delegates) so an in-flight startup-recovery step
+ // stops promptly between slots; close() also raises it. Read on the hot
+ // paths (borrow/giveBack/discardBroken/reapIdle/recovery).
private volatile boolean closed;
+ // True once close() has begun the one-and-only delegate teardown. Distinct
+ // from `closed` so markClosing() can raise the shutdown signal early
+ // (cancelling recovery) WITHOUT making a later close() short-circuit the
+ // teardown. Guarded by lock.
+ private boolean closeStarted;
private int inFlightCreations;
+ // Slots whose delegate close() returned with the SF flock still held
+ // (the I/O thread refused to stop). Permanently consumed: the index is
+ // never freed and never reused, so no borrow ever hands out a still-
+ // locked slot dir. Counted in the cap check so the lost capacity is
+ // accounted for. Guarded by lock; only ever ticks for SF slots.
+ private int leakedSlots;
+ // SF slots currently held by the in-range startup-recovery pass
+ // (recoverOneSlotStep): each is reserved under `lock` for the
+ // duration of its drain and counted in the borrow() cap check so a
+ // concurrent borrow can neither over-allocate past maxSize nor target a
+ // dir being recovered. Only ever non-zero on the deferred (housekeeper-
+ // driven) recovery path, where recovery overlaps borrow()/return; on the
+ // inline construction path the pool is still single-threaded. Guarded by
+ // lock; only ever ticks for SF slots.
+ private int recoveringSlots;
+ // Resumable startup-recovery scan cursor. Advanced only by the single
+ // recovery driver -- the inline constructor loop (single-threaded,
+ // unpublished) or the PoolHousekeeper thread (the sole deferred driver) --
+ // so the cursor itself needs no lock; the per-slot reservation it performs
+ // (slotInUse/recoveringSlots) is still taken under `lock` because borrow()
+ // races it. recoveryInRangeNext is the next in-range index in [0, maxSize)
+ // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily
+ // built pass-2 work list (same-base slots at index >= maxSize) and its
+ // cursor; recoveryComplete latches true when the whole scan finishes or is
+ // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent.
+ private int recoveryInRangeNext;
+ private IntList recoveryOutOfRange;
+ private int recoveryOutOfRangeNext;
+ private boolean recoveryComplete;
public SenderPool(
String configurationString,
@@ -75,10 +187,55 @@ public SenderPool(
long acquireTimeoutMillis,
long idleTimeoutMillis,
long maxLifetimeMillis
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, null);
+ }
+
+ // Package-private constructor exposing the senderFactory test seam:
+ // production passes null (-> the real defaultSender()). White-box tests in
+ // io.questdb.client.test.impl reach this by reflection to inject a factory
+ // that throws a non-RuntimeException Throwable mid-prewarm. Recovery runs
+ // inline here (deferStartupRecovery=false); the pooled QuestDB handle uses
+ // the 8-arg overload to defer it to the housekeeper thread.
+ SenderPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ IntFunction senderFactory
+ ) {
+ this(configurationString, minSize, maxSize, acquireTimeoutMillis,
+ idleTimeoutMillis, maxLifetimeMillis, senderFactory, false);
+ }
+
+ // Full constructor. deferStartupRecovery=true skips the inline,
+ // construction-time SF recovery (recoverOneSlotStep) so
+ // QuestDB.build() never blocks on a slow or reachable-but-not-acking
+ // server; the owner (QuestDBImpl) then drives recovery one slot per tick on
+ // the PoolHousekeeper thread via runStartupRecoveryStep(). The in-range
+ // recovery pass is concurrency-safe against borrow()/return on that
+ // deferred path -- see recoverOneSlotStep().
+ SenderPool(
+ String configurationString,
+ int minSize,
+ int maxSize,
+ long acquireTimeoutMillis,
+ long idleTimeoutMillis,
+ long maxLifetimeMillis,
+ IntFunction senderFactory,
+ boolean deferStartupRecovery
) {
if (minSize < 0 || maxSize < 1 || minSize > maxSize) {
throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize);
}
+ this.senderFactory = senderFactory != null ? senderFactory : this::defaultSender;
+ // An injected factory (tests) drives recovery too, preserving the
+ // white-box recovery seam; production recovery forces OFF-mode connects
+ // via defaultRecoverySender (see field comment / createRecoverer).
+ this.recoverySenderFactory = senderFactory != null ? senderFactory : this::defaultRecoverySender;
this.configurationString = configurationString;
this.minSize = minSize;
this.maxSize = maxSize;
@@ -88,24 +245,383 @@ public SenderPool(
this.all = new ArrayList<>(maxSize);
this.available = new ArrayDeque<>(maxSize);
this.slotReleased = lock.newCondition();
- // Pre-warm minSize connections.
+ // Probe the config once, up front: this validates it eagerly (so a
+ // bad config fails at construction even when minSize == 0) and tells
+ // us whether SF is on and, if so, the base slot id to derive
+ // per-sender ids from.
+ Sender.LineSenderBuilder probe = Sender.builder(configurationString);
+ this.storeAndForward = probe.isStoreAndForwardEnabled();
+ this.slotBaseId = this.storeAndForward ? probe.getConfiguredSenderId() : null;
+ this.sfDir = this.storeAndForward ? probe.getConfiguredSfDir() : null;
+ this.slotInUse = this.storeAndForward ? new boolean[maxSize] : null;
+ // Pre-warm minSize connections. Pre-warm runs single-threaded in the
+ // constructor, so slots 0..minSize-1 are reserved directly.
int built = 0;
try {
for (int i = 0; i < minSize; i++) {
- PooledSender ps = createUnlocked();
+ if (storeAndForward) {
+ slotInUse[i] = true;
+ }
+ PooledSender ps = createUnlocked(storeAndForward ? i : -1);
all.add(ps);
available.add(ps);
built++;
}
- } catch (RuntimeException e) {
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException: createUnlocked() runs a
+ // heavy native build path (mmap, flock, WebSocket connect) that can
+ // throw an Error -- e.g. an -ea AssertionError or OutOfMemoryError --
+ // mid-prewarm. If we only caught RuntimeException the Error would
+ // propagate without running the cleanup below, leaking every
+ // already-built delegate's flock + mmap'd ring + I/O thread and
+ // resurrecting "sf slot already in use" on the next attempt.
for (int i = 0; i < built; i++) {
try {
all.get(i).delegate().close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort cleanup: a delegate close() can throw an
+ // Error (e.g. an -ea AssertionError) as well as a
+ // RuntimeException. Swallow either so we still close the
+ // remaining pre-warmed slots and rethrow the original
+ // construction failure below.
}
}
throw e;
}
+ // Prewarm succeeded. Recover any unacked data a previous run left in
+ // this pool's own managed slots that prewarm did not already re-adopt.
+ // The pooled QuestDB handle defers this to the housekeeper thread
+ // (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow
+ // or reachable-but-not-acking server; direct constructions run it inline
+ // here, while still single-threaded and unpublished.
+ if (!deferStartupRecovery) {
+ runStartupRecoveryToCompletion();
+ }
+ }
+
+ /**
+ * Drives startup SF recovery to completion in a single call, bounded by one
+ * shared {@code acquireTimeoutMillis} wall-clock budget (and each individual
+ * drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the inline
+ * construction path -- single-threaded and unpublished -- and by manual /
+ * test drives. No-op when SF is off, the pool is shutting down, or recovery
+ * has already finished. Idempotent.
+ */
+ void runStartupRecoveryToCompletion() {
+ if (!storeAndForward) {
+ return;
+ }
+ // One shared wall-clock budget for the WHOLE scan, not per slot: without
+ // it a reachable-but-not-acking server would pay a full drain timeout on
+ // every stranded slot. One acquire timeout is the ceiling already
+ // accepted for a single borrow, so we reuse it as the total budget; once
+ // spent, the remaining slots wait for a later attempt (data stays
+ // durable on disk).
+ final long deadlineNanos =
+ System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis);
+ while (!closed && !recoveryComplete) {
+ long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime());
+ if (remainingMillis <= 0) {
+ LOG.warn("startup SF recovery: {}ms budget exhausted; "
+ + "skipping remaining slots", acquireTimeoutMillis);
+ recoveryComplete = true;
+ return;
+ }
+ if (!recoverOneSlotStep(Math.min(remainingMillis, RECOVERY_DRAIN_BUDGET_MILLIS))) {
+ return;
+ }
+ }
+ }
+
+ /**
+ * Recovers at most ONE stranded managed slot and reports whether more remain.
+ * Driven by the {@link PoolHousekeeper} one slot per step, back-to-back, on
+ * the reap-loop thread: each step performs at most one drain bounded by
+ * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the housekeeper
+ * stop/join budget -- on a delegate whose initial connect is forced OFF
+ * ({@link #defaultRecoverySender}) so the build makes at most one connect
+ * attempt. The housekeeper re-checks {@code stop} between steps while
+ * {@link QuestDBImpl#close()} raises the {@code closed} shutdown signal (via
+ * {@link #markClosing()}) BEFORE stopping it, so a {@code close()} landing
+ * mid-recovery normally only has to wait out a single bounded drain. The one
+ * exception is an in-flight connect to a black-holed host, which blocks on
+ * the OS connect timeout -- see the residual-window note on
+ * {@link #recoverOneSlotStep}.
+ * No-op (returns {@code false}) when SF is off, the pool is shutting down, or
+ * recovery has already finished.
+ *
+ * @return {@code true} if recovery has more work (call again), {@code false}
+ * when recovery is complete or the pool is shutting down
+ */
+ boolean runStartupRecoveryStep() {
+ if (!storeAndForward || closed || recoveryComplete) {
+ return false;
+ }
+ return recoverOneSlotStep(RECOVERY_DRAIN_BUDGET_MILLIS);
+ }
+
+ /**
+ * Best-effort recovery of unacked data left in this pool's own managed SF
+ * slots by a previous run -- both the in-range slots {@code [0, maxSize)}
+ * (pass 1) and the same-base slots a larger previous run left OUT of range
+ * ({@code -i} with {@code i >= maxSize}, pass 2). Performs at most ONE
+ * actual drain per call, advancing the resumable scan cursor, and returns
+ * whether more stranded slots remain.
+ *
+ * Every pooled SF sender's orphan drainer deliberately excludes the whole
+ * {@code [0, maxSize)} managed range (via
+ * {@code orphanDrainExcludeManagedSlots}) so a sibling never adopts a slot
+ * dir/lock the pool intends to (re)create -- that exclusion is what keeps the
+ * per-slot ids from resurfacing "sf slot already in use". The trade-off is
+ * that an in-range slot left holding unacked data is otherwise recovered ONLY
+ * when the pool happens to (re)create that index; under steady low load the
+ * pool may never grow to a high index, stranding that slot's data (durable on
+ * disk, but undelivered) until a later restart or load spike. An out-of-range
+ * slot is worse off still: the pool never re-creates its index, and the
+ * per-sender drainer only adopts it when {@code drain_orphans=on}. This scan
+ * closes both gaps regardless of {@code drain_orphans}.
+ *
+ * The in-range pass reserves each slot index under {@code lock} for the
+ * duration of its recovery AND counts it in the borrow() capacity check (via
+ * {@code recoveringSlots}), so a concurrent borrow on the deferred path can
+ * neither target the dir being recovered nor over-allocate past
+ * {@code maxSize}. Prewarmed/borrowed slots (already live, holding their
+ * flock) are skipped, as are empty slots (a cheap directory probe); only a
+ * slot that actually holds stranded data spends the step's single drain. The
+ * out-of-range pass needs no reservation: those indices have no
+ * {@code slotInUse} entry and are never allocated by borrow().
+ *
+ * Best-effort throughout: a build/close Error or a slow drain is logged and
+ * never propagates, since the data stays durable on disk for a later attempt;
+ * the first build failure or drain timeout latches {@code recoveryComplete}
+ * (the failure will very likely repeat for every remaining slot).
+ *
+ * Boundedness / residual window. Recovery is driven on the
+ * PoolHousekeeper thread, and {@code close()} relies on a step finishing
+ * within {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build +
+ * drain + close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}
+ * and the build forces {@code initial_connect_mode=OFF} (see
+ * {@link #defaultRecoverySender}), so it makes at most ONE connect attempt
+ * instead of a SYNC reconnect-budget retry loop. That removes the
+ * minutes-long block a {@code reconnect_*}-tuned config used to cause (M1).
+ * One residual window remains and is NOT closed here: a single in-flight
+ * connect to a black-holed/firewalled host blocks on the OS connect timeout
+ * (the transport exposes no application-level connect timeout to clamp it).
+ * If {@code close()} lands during that one connect the housekeeper join can
+ * still time out and the detached build releases the slot flock shortly
+ * after {@code close()} returns. No data is lost (the slot stays durable on
+ * disk); the exposure is a brief "sf slot already in use" window on an
+ * immediate reopen, bounded by a single OS connect timeout.
+ *
+ * @return {@code true} if a drain was performed and more slots may remain;
+ * {@code false} once the scan is complete or the pool is shutting down
+ */
+ private boolean recoverOneSlotStep(long stepBudgetMillis) {
+ if (sfDir == null || !Files.exists(sfDir)) {
+ recoveryComplete = true;
+ return false;
+ }
+ final boolean[] flockHeld = new boolean[1];
+
+ // Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots
+ // cheaply; spend the step on the first slot that actually holds data.
+ while (recoveryInRangeNext < maxSize) {
+ if (closed) {
+ return false;
+ }
+ int i = recoveryInRangeNext;
+ // Reserve this index unless prewarm (or a concurrent borrow, on the
+ // deferred path) already holds it live. Count the reservation in
+ // recoveringSlots so the borrow() cap check cannot over-allocate
+ // while this slot is held for recovery.
+ boolean reserved;
+ lock.lock();
+ try {
+ reserved = slotInUse[i];
+ if (!reserved) {
+ slotInUse[i] = true;
+ recoveringSlots++;
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (reserved) {
+ recoveryInRangeNext++;
+ continue;
+ }
+ String slotPath = sfDir + "/" + slotBaseId + "-" + i;
+ if (!OrphanScanner.isCandidateOrphan(slotPath)) {
+ // No stranded data: release the reservation and keep scanning;
+ // an empty slot must not cost a whole step.
+ lock.lock();
+ try {
+ recoveringSlots--;
+ slotInUse[i] = false;
+ slotReleased.signal();
+ } finally {
+ lock.unlock();
+ }
+ recoveryInRangeNext++;
+ continue;
+ }
+ // A real candidate -> spend the step on it. Advance the cursor first
+ // so a resume never reprocesses this index.
+ recoveryInRangeNext++;
+ boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld);
+ lock.lock();
+ try {
+ // Release the recovery reservation accounting; from here either
+ // leakedSlots (retire) or the freed index carries the cap math.
+ recoveringSlots--;
+ if (flockHeld[0]) {
+ // close() bailed early with the I/O thread still running and
+ // the flock still held. Retire the slot permanently (mirror
+ // discardBroken/reapIdle): keep slotInUse[i] set and count it
+ // in leakedSlots so the borrow() cap math accounts for the
+ // lost capacity and no later borrow ever reuses the
+ // still-locked dir.
+ leakedSlots++;
+ LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with "
+ + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, "
+ + "now {} of {} usable [leakedSlots={}]",
+ i, maxSize - leakedSlots, maxSize, leakedSlots);
+ } else {
+ slotInUse[i] = false;
+ // On the deferred path a borrow may be waiting on capacity
+ // this recovery held; the freed index can now admit a
+ // creation.
+ slotReleased.signal();
+ }
+ } finally {
+ lock.unlock();
+ }
+ if (stopScan) {
+ // A build failure or drain timeout that will very likely repeat
+ // for every remaining slot -- abort the scan; the data stays
+ // durable on disk for a later attempt. Do not start pass 2.
+ recoveryComplete = true;
+ return false;
+ }
+ return true;
+ }
+
+ // Pass 1 done. Build the pass-2 work list once: same-base slots a
+ // previous run left OUT of the current index range (-i with
+ // i >= maxSize, from a run with a larger maxSize). The pool never
+ // re-creates these indices, and the per-sender drainer only adopts them
+ // when drain_orphans=on, so without this pass their unacked data would
+ // sit durable-but-undelivered under the default config. They are outside
+ // [0, maxSize), have no slotInUse entry, and never affect the borrow()
+ // cap math, so no reservation is needed.
+ if (recoveryOutOfRange == null) {
+ recoveryOutOfRange = OrphanScanner.listStrandedOutOfRangeManagedSlots(
+ sfDir, slotBaseId, maxSize);
+ recoveryOutOfRangeNext = 0;
+ }
+ while (recoveryOutOfRangeNext < recoveryOutOfRange.size()) {
+ if (closed) {
+ return false;
+ }
+ int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++);
+ String slotPath = sfDir + "/" + slotBaseId + "-" + idx;
+ if (!OrphanScanner.isCandidateOrphan(slotPath)) {
+ continue;
+ }
+ boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld);
+ if (flockHeld[0]) {
+ // Out of the pool's [0, maxSize) capacity range: there is no
+ // slotInUse entry to retire and no future borrow targets this
+ // dir, so a still-held flock only leaks this recoverer's I/O
+ // thread (a best-effort teardown loss, logged). Crucially we do
+ // NOT touch leakedSlots -- that would wrongly shrink the
+ // in-range pool capacity.
+ LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held "
+ + "(I/O thread refused to stop); its data is durable on disk for a later attempt",
+ slotPath);
+ }
+ if (stopScan) {
+ recoveryComplete = true;
+ return false;
+ }
+ return true;
+ }
+
+ recoveryComplete = true;
+ return false;
+ }
+
+ /**
+ * Drains one candidate orphan slot dir within {@code remainingMillis},
+ * best-effort and never throwing. Builds a recoverer on {@code slotIndex}
+ * (whose {@link #defaultSender} derives the dir {@code -slotIndex}),
+ * drains its unacked data, and closes the delegate. Shared by both recovery
+ * passes -- the in-range pass and the out-of-range pass -- which differ only
+ * in their slot bookkeeping, handled by the caller via {@code flockHeld}.
+ *
+ * @param flockHeld single-element out-param set to {@code true} iff a
+ * recoverer was built and its {@code close()} returned with
+ * the flock still held (the I/O thread refused to stop)
+ * @return {@code true} if a build/drain failure occurred that will very
+ * likely repeat for every remaining slot, so the caller should stop scanning
+ */
+ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath,
+ long remainingMillis, boolean[] flockHeld) {
+ flockHeld[0] = false;
+ // Hoisted so the flock check after the try can consult it:
+ // createRecoverer() takes the slot flock on -slotIndex, and
+ // delegate().close() can early-return with the I/O thread still running
+ // (flock still held).
+ PooledSender recoverer = null;
+ boolean stopScan = false;
+ try {
+ if (!OrphanScanner.isCandidateOrphan(slotPath)) {
+ return false;
+ }
+ try {
+ // Recovery delegate: forced OFF-mode initial connect (see
+ // createRecoverer / defaultRecoverySender), so this build does
+ // at most ONE connect attempt -- it never inherits the SYNC
+ // reconnect-budget retry loop that would block this
+ // (PoolHousekeeper) thread for minutes (M1).
+ recoverer = createRecoverer(slotIndex);
+ } catch (Throwable buildErr) {
+ // A build/connect failure (e.g. server unreachable) will very
+ // likely repeat for every remaining slot, so stop here rather
+ // than pay a connect timeout per slot.
+ LOG.warn("startup SF recovery: could not open slot {} ({}); "
+ + "skipping remaining slots", slotPath, buildErr.toString());
+ return true;
+ }
+ try {
+ // Cap the drain at the remaining shared budget and short-circuit
+ // on a timeout: a server that fails to ack within the budget
+ // will very likely do the same for every remaining slot -- the
+ // same reasoning as the build-failure case above.
+ if (!recoverer.drain(remainingMillis)) {
+ LOG.warn("startup SF recovery: drain did not ack slot {} "
+ + "within {}ms; skipping remaining slots",
+ slotPath, remainingMillis);
+ stopScan = true;
+ }
+ } catch (Throwable drainErr) {
+ LOG.warn("startup SF recovery: drain failed for slot {} ({})",
+ slotPath, drainErr.toString());
+ } finally {
+ try {
+ recoverer.delegate().close();
+ } catch (Throwable ignored) {
+ // Best-effort close: a teardown Error must not abort
+ // recovery of the remaining slots.
+ }
+ }
+ } catch (Throwable scanErr) {
+ LOG.warn("startup SF recovery: scan failed for slot {} ({})",
+ slotPath, scanErr.toString());
+ }
+ if (recoverer != null) {
+ flockHeld[0] = !flockReleased(recoverer);
+ }
+ return stopScan;
}
public PooledSender borrow() {
@@ -124,15 +640,31 @@ public PooledSender borrow() {
s.markInUse();
return s;
}
- if (all.size() + inFlightCreations < maxSize) {
+ if (all.size() + inFlightCreations + closingSlots + leakedSlots + recoveringSlots < maxSize) {
inFlightCreations++;
+ // Reserve a slot index under the lock so concurrent
+ // creations never target the same SF slot dir. -1 when
+ // SF is off (no per-slot identity needed).
+ int slotIndex = storeAndForward ? allocateSlotIndex() : -1;
lock.unlock();
PooledSender created;
try {
- created = createUnlocked();
- } catch (RuntimeException e) {
+ created = createUnlocked(slotIndex);
+ } catch (Throwable e) {
+ // Catch Throwable, not just RuntimeException:
+ // createUnlocked() runs a heavy native build path
+ // (mmap, flock, WebSocket connect) that can throw an
+ // Error -- e.g. an -ea AssertionError or
+ // OutOfMemoryError. If we only caught RuntimeException
+ // the Error would propagate with inFlightCreations
+ // still incremented and the SF slot index still
+ // reserved (slotInUse[idx] stuck true), permanently
+ // lowering pool capacity. The cleanup below is
+ // idempotent, so undoing the reservation for any
+ // throwable is safe.
lock.lock();
inFlightCreations--;
+ freeSlotIndex(slotIndex);
slotReleased.signal();
lock.unlock();
throw e;
@@ -143,9 +675,12 @@ public PooledSender borrow() {
// Pool was closed mid-creation -- destroy the new connection
// rather than leaking it. Other waiters have been signaled
// by close() already.
+ freeSlotIndex(slotIndex);
try {
created.delegate().close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: an Error (e.g. -ea AssertionError)
+ // from teardown must not mask the closed-pool signal.
}
throw new LineSenderException("QuestDB handle is closed");
}
@@ -171,14 +706,30 @@ public PooledSender borrow() {
}
}
+ /**
+ * Raises the shutdown signal early -- without tearing down live delegates --
+ * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper}
+ * thread stops promptly between slots. {@link QuestDBImpl#close()} calls this
+ * BEFORE stopping the housekeeper, so the housekeeper join cannot time out
+ * waiting on a fresh slot's recovery drain. Full delegate teardown still
+ * happens in {@link #close()} (guarded by {@code closeStarted}, so this early
+ * signal never short-circuits it). Idempotent; safe to call repeatedly.
+ */
+ void markClosing() {
+ closed = true;
+ }
+
@Override
public void close() {
PooledSender[] snapshot;
lock.lock();
try {
- if (closed) {
+ if (closeStarted) {
return;
}
+ closeStarted = true;
+ // Raise the shutdown signal too (a direct, non-pooled caller may
+ // close() without a prior markClosing()); harmless if already set.
closed = true;
// Mark every pooled wrapper invalidated so pinToCurrentThread()
// on other threads -- which never takes this lock -- can detect
@@ -205,7 +756,9 @@ public void close() {
for (int i = 0; i < snapshot.length; i++) {
try {
snapshot[i].delegate().close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: an Error from one delegate's teardown must not
+ // abort the loop and strand the remaining delegates unclosed.
}
}
}
@@ -241,21 +794,53 @@ void clearPinIfCurrent(PooledSender s) {
*/
void discardBroken(PooledSender s) {
s.markInvalidated();
+ boolean reserved = false;
lock.lock();
try {
if (closed) {
return;
}
- all.remove(s);
- // Wake one waiter -- the cap check in borrow() uses all.size(),
- // so a freed slot may now allow a creation attempt.
+ boolean removed = all.remove(s);
+ // For an SF slot, keep its index reserved (move the reservation
+ // from `all` to `closingSlots`) until the delegate below releases
+ // the flock. Capacity stays accounted for, so a concurrent borrow
+ // cannot reclaim this slot dir while its lock is still held.
+ if (removed && s.slotIndex() >= 0) {
+ closingSlots++;
+ reserved = true;
+ }
+ // Wake one waiter -- the cap check in borrow() may now admit a
+ // creation attempt (on a *different* slot).
slotReleased.signal();
} finally {
lock.unlock();
}
+ // Close the delegate outside the lock (releases the SF flock). Always
+ // attempt it so native resources are freed even on the defensive path
+ // where the wrapper had already left `all`.
try {
s.delegate().close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort teardown: a delegate close() can throw an Error
+ // (e.g. an -ea AssertionError) as well as a RuntimeException.
+ // Either way the slot accounting in the finally below MUST run,
+ // otherwise an SF slot stays reserved forever (slotInUse stuck
+ // true, closingSlots over-counted) and the pool leaks capacity
+ // until borrow() can only ever time out.
+ } finally {
+ if (reserved) {
+ lock.lock();
+ try {
+ // Free the index only when the flock was released; a slot
+ // left locked is retired permanently. Signal a waiter only
+ // on the free path, where a new creation can now be admitted.
+ if (reclaimSlot(s, "")) {
+ slotReleased.signal();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
}
}
@@ -314,6 +899,11 @@ public void reapIdle() {
if (idleExpired || overAge) {
it.remove();
all.remove(s);
+ // Keep the SF slot reserved until its flock is released
+ // below (see discardBroken for the rationale).
+ if (s.slotIndex() >= 0) {
+ closingSlots++;
+ }
if (toClose == null) {
toClose = new ArrayList<>();
}
@@ -327,7 +917,30 @@ public void reapIdle() {
for (int i = 0, n = toClose.size(); i < n; i++) {
try {
toClose.get(i).delegate().close();
- } catch (RuntimeException ignored) {
+ } catch (Throwable ignored) {
+ // Best-effort: a single delegate close() failure (including
+ // an Error such as an -ea AssertionError) must not abort the
+ // loop -- that would leave sibling flocks unreleased -- nor
+ // skip the slot-accounting release block below, which would
+ // strand every reaped index (slotInUse stuck true,
+ // closingSlots over-counted) and leak pool capacity.
+ }
+ }
+ // Return reserved SF slot indices to the free set -- but only for
+ // slots whose delegate confirmed the flock was released. A slot
+ // left locked (I/O thread refused to stop) is retired permanently.
+ if (storeAndForward) {
+ lock.lock();
+ try {
+ for (int i = 0, n = toClose.size(); i < n; i++) {
+ PooledSender s = toClose.get(i);
+ if (s.slotIndex() >= 0) {
+ reclaimSlot(s, " during idle reaping");
+ }
+ }
+ slotReleased.signalAll();
+ } finally {
+ lock.unlock();
}
}
}
@@ -353,6 +966,23 @@ public int totalSize() {
}
}
+ /**
+ * Snapshot of the number of SF slots permanently retired because a
+ * delegate {@code close()} returned with the slot flock still held (the
+ * I/O thread refused to stop). Each leaked slot permanently lowers the
+ * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A
+ * non-zero, growing value explains a pool that has started timing out
+ * every {@code borrow()}. For metrics and tests.
+ */
+ public int leakedSlotCount() {
+ lock.lock();
+ try {
+ return leakedSlots;
+ } finally {
+ lock.unlock();
+ }
+ }
+
public void releaseCurrentThread() {
PooledSender pinned = threadAffine.get();
if (pinned == null) {
@@ -366,8 +996,173 @@ public void releaseCurrentThread() {
pinned.close();
}
- private PooledSender createUnlocked() {
- Sender raw = Sender.fromConfig(configurationString);
- return new PooledSender(raw, this);
+ private PooledSender createUnlocked(int slotIndex) {
+ return new PooledSender(senderFactory.apply(slotIndex), this, slotIndex);
+ }
+
+ /**
+ * Builds a {@link PooledSender} for startup recovery of one stranded slot.
+ * Routes through {@link #recoverySenderFactory}, which in production forces
+ * a non-blocking initial connect ({@link #defaultRecoverySender}) so a
+ * single recovery step stays bounded -- see that method and
+ * {@link #drainCandidateSlotForRecovery}.
+ */
+ private PooledSender createRecoverer(int slotIndex) {
+ return new PooledSender(recoverySenderFactory.apply(slotIndex), this, slotIndex);
+ }
+
+ private Sender defaultSender(int slotIndex) {
+ return buildManagedSlotSender(slotIndex, false);
+ }
+
+ /**
+ * Same managed-slot delegate as {@link #defaultSender}, but with the
+ * initial connect forced to {@link Sender.InitialConnectMode#OFF}. Used
+ * only for startup recovery, which runs on the PoolHousekeeper thread: OFF
+ * makes the build do at most ONE connect attempt instead of retrying for
+ * the whole reconnect budget (SYNC, auto-enabled by any reconnect_* knob),
+ * keeping a recovery step bounded below
+ * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. See M1 / the residual-window
+ * note on {@link #recoverOneSlotStep}.
+ *
+ * Also forces {@code drain_orphans=off} (see
+ * {@link #buildManagedSlotSender}): a recovery delegate must never spin up a
+ * BackgroundDrainerPool, whose {@code close()} could block ~3s and overrun
+ * the step / {@code STOP_TIMEOUT_MILLIS} budget while still holding the slot
+ * flock.
+ */
+ private Sender defaultRecoverySender(int slotIndex) {
+ return buildManagedSlotSender(slotIndex, true);
+ }
+
+ private Sender buildManagedSlotSender(int slotIndex, boolean forRecovery) {
+ if (!storeAndForward) {
+ return Sender.fromConfig(configurationString);
+ }
+ // Give this pooled sender its own slot dir /-
+ // so concurrent SF senders sharing one sf_dir never collide on
+ // the slot flock. senderId() is only legal on WebSocket transport,
+ // which is exactly when storeAndForward is true.
+ //
+ // Also fence off the pool's own live slot set [0, maxSize) from
+ // orphan draining: the pool co-manages every - slot it
+ // can re-create and recovers each slot's unacked data when it
+ // (re)creates it, so a sibling's startup drainer must never adopt
+ // another live pool slot's dir/lock (that would resurrect "sf slot
+ // already in use"). The bound is maxSize, NOT the whole "-"
+ // prefix: a same-base slot at an index >= maxSize (left behind when
+ // a previous run used a larger maxSize) is out of the pool's index
+ // range forever, so it is left drainable here. Its unacked data is
+ // delivered by the pool's own startup recovery
+ // (recoverOneSlotStep, pass 2), which adopts these
+ // out-of-range same-base slots at construction REGARDLESS of
+ // drain_orphans -- so the default config never strands it. The
+ // per-sender drainer is an additional path that only runs when
+ // drain_orphans=on; foreign leftovers under other names are drained
+ // only by that path.
+ Sender.LineSenderBuilder builder = Sender.builder(configurationString)
+ .senderId(slotBaseId + "-" + slotIndex)
+ .orphanDrainExcludeManagedSlots(slotBaseId, maxSize);
+ if (forRecovery) {
+ // Force OFF so the recovery build never blocks on the reconnect
+ // budget (see defaultRecoverySender). An explicit mode wins over
+ // the SYNC auto-promotion the user's reconnect_* knobs would
+ // otherwise trigger.
+ builder.initialConnectMode(Sender.InitialConnectMode.OFF);
+ // Force drain_orphans OFF on recovery delegates regardless of the
+ // shared config string. A recovery delegate's sole job is to drain
+ // its OWN slot (the one recoverOneSlotStep is processing); it must
+ // never start a BackgroundDrainerPool for sibling/foreign orphans.
+ // If it did, the delegate's close() -- called from
+ // drainCandidateSlotForRecovery() on the PoolHousekeeper thread,
+ // BEFORE its cursorEngine.close() releases the slot flock -- would
+ // block in BackgroundDrainerPool.close() for up to
+ // GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS (3s) against a
+ // reachable-but-not-acking server. That overruns a recovery step's
+ // budget (RECOVERY_DRAIN_BUDGET_MILLIS) and PoolHousekeeper
+ // .STOP_TIMEOUT_MILLIS, so a close() landing mid-step times out its
+ // join and returns while the recoverer still holds the slot flock
+ // -- resurrecting the "sf slot already in use" window this pool's
+ // per-slot ids exist to eliminate. Sibling in-range slots are
+ // covered by recoverOneSlotStep's own passes; foreign/out-of-range
+ // orphans are covered by the LIVE pooled senders' drainers (which
+ // keep drain_orphans=on and whose close() senderPool.close() awaits
+ // synchronously, so they release their flock before close()
+ // returns).
+ builder.drainOrphans(false);
+ }
+ return builder.build();
+ }
+
+ /**
+ * Reserves and returns the lowest free SF slot index. The borrow() cap
+ * check ({@code all.size() + inFlightCreations + closingSlots + leakedSlots
+ * < maxSize}) guarantees a free index exists whenever a creation is
+ * admitted, so this never fails in practice; the guard throws defensively rather than
+ * silently colliding two senders on one slot dir. Caller must hold
+ * {@code lock}.
+ */
+ private int allocateSlotIndex() {
+ for (int i = 0; i < slotInUse.length; i++) {
+ if (!slotInUse[i]) {
+ slotInUse[i] = true;
+ return i;
+ }
+ }
+ throw new IllegalStateException(
+ "no free SF slot index -- pool capacity invariant violated");
+ }
+
+ /**
+ * Returns an SF slot index to the free set. No-op for non-SF pools and
+ * for the {@code -1} sentinel. Caller must hold {@code lock}.
+ */
+ private void freeSlotIndex(int idx) {
+ if (idx >= 0 && slotInUse != null) {
+ slotInUse[idx] = false;
+ }
+ }
+
+ /**
+ * Whether the delegate's {@code close()} released the SF slot flock. A
+ * non-QWP delegate never holds an SF flock, so it is always treated as
+ * released. A {@link QwpWebSocketSender} reports it via
+ * {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close()
+ * bailed early with the I/O thread still running and the flock still held.
+ */
+ private static boolean flockReleased(PooledSender s) {
+ Sender d = s.delegate();
+ return !(d instanceof QwpWebSocketSender) || ((QwpWebSocketSender) d).isSlotLockReleased();
+ }
+
+ /**
+ * Reclaims one SF slot after its delegate's {@code close()} has been
+ * attempted. When the flock was released the index returns to the free
+ * set; when {@code close()} returned with the flock still held (the I/O
+ * thread refused to stop) the slot is retired permanently --
+ * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap
+ * math accounts for the lost capacity and no later borrow ever reuses the
+ * still-locked dir. Either way {@code closingSlots} is decremented.
+ *
+ * Caller must hold {@code lock} and is responsible for signalling waiters
+ * (only the free path admits a new creation). Shared by
+ * {@link #discardBroken} and {@link #reapIdle}.
+ *
+ * @param s sender whose slot is being reclaimed ({@code slotIndex() >= 0})
+ * @param context phrase woven into the retire WARN to name the reclaim
+ * path (e.g. {@code ""} or {@code " during idle reaping"})
+ * @return {@code true} if the index was freed, {@code false} if retired
+ */
+ private boolean reclaimSlot(PooledSender s, String context) {
+ closingSlots--;
+ if (flockReleased(s)) {
+ freeSlotIndex(s.slotIndex());
+ return true;
+ }
+ leakedSlots++;
+ LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " +
+ "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]",
+ s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots);
+ return false;
}
}
diff --git a/core/src/main/java/io/questdb/client/std/Decimal64.java b/core/src/main/java/io/questdb/client/std/Decimal64.java
index 6d37f567..0147b995 100644
--- a/core/src/main/java/io/questdb/client/std/Decimal64.java
+++ b/core/src/main/java/io/questdb/client/std/Decimal64.java
@@ -812,7 +812,7 @@ public String toString() {
if (isNull()) {
return "";
}
- var sink = new StringSink();
+ StringSink sink = new StringSink();
toSink(sink);
return sink.toString();
}
diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java
index cfc71140..c7b8acc9 100644
--- a/core/src/main/java/io/questdb/client/std/Numbers.java
+++ b/core/src/main/java/io/questdb/client/std/Numbers.java
@@ -27,7 +27,6 @@
import io.questdb.client.std.fastdouble.FastDoubleParser;
import io.questdb.client.std.str.CharSink;
import io.questdb.client.std.str.Utf8Sequence;
-import jdk.internal.math.FDBigInteger;
import java.util.Arrays;
@@ -658,12 +657,12 @@ private static void appendDouble0(
lowDigitDifference = (b << 1) - tens;
}
} else {
- FDBigInteger sVal = FDBigInteger.valueOfPow52(S5, S2);
+ FdBig sVal = FdBig.valueOfPow52(S5, S2);
final int shiftBias = sVal.getNormalizationBias();
sVal = sVal.leftShift(shiftBias);
- FDBigInteger bVal = FDBigInteger.valueOfMulPow52(fractionBits, B5, B2 + shiftBias);
- FDBigInteger mVal = FDBigInteger.valueOfPow52(B5 + 1, M2 + shiftBias + 1);
- FDBigInteger tensVal = FDBigInteger.valueOfPow52(S5 + 1, S2 + shiftBias + 1);
+ FdBig bVal = FdBig.valueOfMulPow52(fractionBits, B5, B2 + shiftBias);
+ FdBig mVal = FdBig.valueOfPow52(B5 + 1, M2 + shiftBias + 1);
+ FdBig tensVal = FdBig.valueOfPow52(S5 + 1, S2 + shiftBias + 1);
digitIndex = 0;
q = bVal.quoRemIteration(sVal);
low = bVal.cmp(mVal) < 0;
@@ -1455,11 +1454,6 @@ private interface LongHexAppender {
void append(CharSink sink, long value);
}
- static {
- Module currentModule = Numbers.class.getModule();
- Unsafe.addExports(Unsafe.JAVA_BASE_MODULE, currentModule, "jdk.internal.math");
- }
-
static {
pow10 = new long[20];
pow10[0] = 1;
diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java
index 2df7f2d8..b2f13476 100644
--- a/core/src/main/java/io/questdb/client/std/Unsafe.java
+++ b/core/src/main/java/io/questdb/client/std/Unsafe.java
@@ -28,7 +28,6 @@
import java.lang.reflect.AccessibleObject;
import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.util.concurrent.atomic.LongAdder;
public final class Unsafe {
@@ -36,7 +35,6 @@ public final class Unsafe {
// These are off-heap allocated atomic counters for memory usage tracking.
public static final long BYTE_OFFSET;
- public static final Module JAVA_BASE_MODULE = System.class.getModule();
private static final LongAdder[] COUNTERS = new LongAdder[MemoryTag.SIZE];
private static final long FREE_COUNT_ADDR;
private static final long MALLOC_COUNT_ADDR;
@@ -46,19 +44,10 @@ public final class Unsafe {
private static final long REALLOC_COUNT_ADDR;
private static final long RSS_MEM_USED_ADDR;
private static final sun.misc.Unsafe UNSAFE;
- private static final Method implAddExports;
private Unsafe() {
}
- public static void addExports(Module from, Module to, String packageName) {
- try {
- implAddExports.invoke(from, packageName, to);
- } catch (ReflectiveOperationException e) {
- e.printStackTrace(System.out);
- }
- }
-
public static int byteArrayGetInt(byte[] array, int index) {
assert index > -1 && index < array.length - 3;
return Unsafe.getUnsafe().getInt(array, BYTE_OFFSET + index);
@@ -266,11 +255,9 @@ private static boolean isJava8Or11() {
BYTE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(byte[].class);
OVERRIDE = AccessibleObject_override_fieldOffset();
- implAddExports = Module.class.getDeclaredMethod("implAddExports", String.class, Module.class);
} catch (ReflectiveOperationException e) {
throw new ExceptionInInitializerError(e);
}
- makeAccessible(implAddExports);
// A single allocation for all the off-heap native memory counters.
// Might help with locality, given they're often incremented together.
diff --git a/core/src/main/java11/io/questdb/client/std/Compat.java b/core/src/main/java11/io/questdb/client/std/Compat.java
new file mode 100644
index 00000000..d0c6b1a6
--- /dev/null
+++ b/core/src/main/java11/io/questdb/client/std/Compat.java
@@ -0,0 +1,74 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.std;
+
+import java.lang.reflect.Method;
+
+/**
+ * JDK-version-specific helpers. This is the Java 9+ variant; the parallel copy
+ * under {@code src/main/java8} provides Java 8 implementations of the same API.
+ */
+public final class Compat {
+
+ private Compat() {
+ }
+
+ /**
+ * Returns the current process id, or {@code -1} if it cannot be determined.
+ * Diagnostic use only — callers must never depend on a real value.
+ */
+ public static long currentPid() {
+ try {
+ return ProcessHandle.current().pid();
+ } catch (Throwable ignored) {
+ return -1L;
+ }
+ }
+
+ /**
+ * Spin-loop pause hint. Delegates to {@code Thread.onSpinWait()} on Java 9+;
+ * the Java 8 variant is a no-op (no such hint exists there).
+ */
+ public static void onSpinWait() {
+ Thread.onSpinWait();
+ }
+
+ /**
+ * Opens {@code java.base/jdk.internal.math} to this module so that
+ * {@code FDBigInteger} is reachable at runtime, mirroring the
+ * {@code --add-exports} flag used at compile time.
+ */
+ static void exportFdBigInteger() {
+ try {
+ Module base = System.class.getModule();
+ Module current = Compat.class.getModule();
+ Method implAddExports = Module.class.getDeclaredMethod("implAddExports", String.class, Module.class);
+ Unsafe.makeAccessible(implAddExports);
+ implAddExports.invoke(base, "jdk.internal.math", current);
+ } catch (ReflectiveOperationException e) {
+ e.printStackTrace(System.out);
+ }
+ }
+}
diff --git a/core/src/main/java11/io/questdb/client/std/FdBig.java b/core/src/main/java11/io/questdb/client/std/FdBig.java
new file mode 100644
index 00000000..0f019ab2
--- /dev/null
+++ b/core/src/main/java11/io/questdb/client/std/FdBig.java
@@ -0,0 +1,82 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.std;
+
+import jdk.internal.math.FDBigInteger;
+
+/**
+ * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the
+ * {@link Numbers} double-to-string slow path. The bignum class lives in
+ * {@code jdk.internal.math} on Java 9+; the parallel copy of this file under
+ * {@code src/main/java8} targets {@code sun.misc.FDBigInteger}. Keeping the
+ * type behind this wrapper lets {@code Numbers} carry a single, JDK-agnostic
+ * copy of the algorithm.
+ */
+final class FdBig {
+ static {
+ // Mirror the compile-time --add-exports: grant this module runtime
+ // access to jdk.internal.math before any FDBigInteger reference is
+ // resolved. No-op on Java 8 (sun.misc is open).
+ Compat.exportFdBigInteger();
+ }
+
+ private final FDBigInteger value;
+
+ private FdBig(FDBigInteger value) {
+ this.value = value;
+ }
+
+ static FdBig valueOfMulPow52(long value, int p5, int p2) {
+ return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2));
+ }
+
+ static FdBig valueOfPow52(int p5, int p2) {
+ return new FdBig(FDBigInteger.valueOfPow52(p5, p2));
+ }
+
+ int addAndCmp(FdBig x, FdBig y) {
+ return value.addAndCmp(x.value, y.value);
+ }
+
+ int cmp(FdBig other) {
+ return value.cmp(other.value);
+ }
+
+ int getNormalizationBias() {
+ return value.getNormalizationBias();
+ }
+
+ FdBig leftShift(int shift) {
+ return new FdBig(value.leftShift(shift));
+ }
+
+ FdBig multBy10() {
+ return new FdBig(value.multBy10());
+ }
+
+ int quoRemIteration(FdBig s) {
+ return value.quoRemIteration(s.value);
+ }
+}
diff --git a/core/src/main/java8/io/questdb/client/std/Compat.java b/core/src/main/java8/io/questdb/client/std/Compat.java
new file mode 100644
index 00000000..3d9073e7
--- /dev/null
+++ b/core/src/main/java8/io/questdb/client/std/Compat.java
@@ -0,0 +1,71 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.std;
+
+import java.lang.management.ManagementFactory;
+
+/**
+ * JDK-version-specific helpers. This is the Java 8 variant; the parallel copy
+ * under {@code src/main/java11} provides Java 9+ implementations of the same
+ * API.
+ */
+public final class Compat {
+
+ private Compat() {
+ }
+
+ /**
+ * Returns the current process id, or {@code -1} if it cannot be determined.
+ * Diagnostic use only — callers must never depend on a real value. Java 8
+ * has no {@code ProcessHandle}, so the pid is parsed from the
+ * {@code pid@host} runtime name.
+ */
+ public static long currentPid() {
+ try {
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ int at = name.indexOf('@');
+ if (at > 0) {
+ return Long.parseLong(name.substring(0, at));
+ }
+ } catch (Throwable ignored) {
+ // fall through
+ }
+ return -1L;
+ }
+
+ /**
+ * Spin-loop pause hint. No-op on Java 8 — {@code Thread.onSpinWait()} does
+ * not exist there. The parallel Java 9+ variant delegates to it.
+ */
+ public static void onSpinWait() {
+ }
+
+ /**
+ * No-op on Java 8: {@code sun.misc} is open, so {@code FDBigInteger} needs
+ * no module export. Present for symmetry with the Java 9+ variant.
+ */
+ static void exportFdBigInteger() {
+ }
+}
diff --git a/core/src/main/java8/io/questdb/client/std/FdBig.java b/core/src/main/java8/io/questdb/client/std/FdBig.java
new file mode 100644
index 00000000..7ca6600e
--- /dev/null
+++ b/core/src/main/java8/io/questdb/client/std/FdBig.java
@@ -0,0 +1,81 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.std;
+
+import sun.misc.FDBigInteger;
+
+/**
+ * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the
+ * {@link Numbers} double-to-string slow path. This is the Java 8 variant: the
+ * bignum class lives in {@code sun.misc}. The parallel copy under
+ * {@code src/main/java11} targets {@code jdk.internal.math.FDBigInteger}. The
+ * public surface is identical across both JDKs, so {@code Numbers} carries a
+ * single, JDK-agnostic copy of the algorithm.
+ */
+final class FdBig {
+ static {
+ // No-op on Java 8: sun.misc is open, no module export is needed. Kept
+ // for symmetry with the Java 9+ variant.
+ Compat.exportFdBigInteger();
+ }
+
+ private final FDBigInteger value;
+
+ private FdBig(FDBigInteger value) {
+ this.value = value;
+ }
+
+ static FdBig valueOfMulPow52(long value, int p5, int p2) {
+ return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2));
+ }
+
+ static FdBig valueOfPow52(int p5, int p2) {
+ return new FdBig(FDBigInteger.valueOfPow52(p5, p2));
+ }
+
+ int addAndCmp(FdBig x, FdBig y) {
+ return value.addAndCmp(x.value, y.value);
+ }
+
+ int cmp(FdBig other) {
+ return value.cmp(other.value);
+ }
+
+ int getNormalizationBias() {
+ return value.getNormalizationBias();
+ }
+
+ FdBig leftShift(int shift) {
+ return new FdBig(value.leftShift(shift));
+ }
+
+ FdBig multBy10() {
+ return new FdBig(value.multBy10());
+ }
+
+ int quoRemIteration(FdBig s) {
+ return value.quoRemIteration(s.value);
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
index 25a9d275..83cc9a0c 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/line/tcp/v4/QwpAllocationTestClient.java
@@ -187,7 +187,7 @@ public static void main(String[] args) {
System.out.println("Skipping table drop (--no-drop)");
}
runTest(protocol, host, port, totalRows, batchSize, flushBytes, flushIntervalMs,
- inFlightWindow, maxDatagramSize, warmupRows, reportInterval, targetThroughput);
+ maxDatagramSize, warmupRows, reportInterval, targetThroughput);
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
e.printStackTrace(System.err);
@@ -202,7 +202,6 @@ private static Sender createSender(
int batchSize,
int flushBytes,
long flushIntervalMs,
- int inFlightWindow,
int maxDatagramSize
) {
switch (protocol) {
@@ -330,13 +329,13 @@ private static void recreateTable(String host) throws Exception {
private static void runTest(String protocol, String host, int port, int totalRows,
int batchSize, int flushBytes, long flushIntervalMs,
- int inFlightWindow, int maxDatagramSize,
+ int maxDatagramSize,
int warmupRows, int reportInterval,
int targetThroughput) throws IOException {
System.out.println("Connecting to " + host + ":" + port + "...");
try (Sender sender = createSender(protocol, host, port, batchSize, flushBytes, flushIntervalMs,
- inFlightWindow, maxDatagramSize)) {
+ maxDatagramSize)) {
System.out.println("Connected! Protocol: " + protocol);
System.out.println();
@@ -378,7 +377,7 @@ private static void runTest(String protocol, String host, int port, int totalRow
if (nanosPerRow > 0 && (i + 1) % paceCheckInterval == 0) {
long expectedElapsedNanos = (long) ((i + 1) * nanosPerRow);
while (System.nanoTime() - startTime < expectedElapsedNanos) {
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
index a874063a..fe3bb059 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
@@ -30,6 +30,7 @@
import io.questdb.client.SenderErrorHandler;
import io.questdb.client.cutlass.line.LineSenderException;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -61,7 +62,7 @@ public class CloseSafetyNetTest {
public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build();
@Test(timeout = 30_000)
- public void testCloseRethrowsUnsurfacedTerminalWithoutCustomHandler() throws Exception {
+ public void testCloseRethrowsUnsurfacedTerminalWithoutCustomHandler() {
// No server, no handler, tight reconnect budget: the I/O thread
// latches a never-connected budget-exhaustion terminal that nothing
// has surfaced to the user. close() must throw it.
@@ -115,7 +116,7 @@ private static void awaitLatchedTerminal(QwpWebSocketSender sender) {
if (System.nanoTime() > deadlineNanos) {
throw new AssertionError("I/O thread did not latch a terminal within 10s");
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
@@ -142,7 +143,7 @@ SenderError get() {
}
@Override
- public void onError(SenderError err) {
+ public void onError(@NotNull SenderError err) {
if (ref.compareAndSet(null, err)) {
latch.countDown();
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseTerminalConflationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseTerminalConflationTest.java
index 1a60c2e3..2dd22d59 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseTerminalConflationTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseTerminalConflationTest.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -31,6 +31,7 @@
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.Test;
@@ -144,7 +145,7 @@ private static void awaitLatchedTerminal(QwpWebSocketSender sender) {
if (System.nanoTime() > deadlineNanos) {
throw new AssertionError("I/O thread did not latch a terminal within 10s");
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
@@ -206,7 +207,7 @@ SenderError get() {
}
@Override
- public void onError(SenderError err) {
+ public void onError(@NotNull SenderError err) {
if (ref.compareAndSet(null, err)) {
latch.countDown();
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java
index ffbc16b5..e163b22e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java
@@ -230,7 +230,7 @@ public void testEdgeCase_veryLongSymbol() throws Exception {
GlobalSymbolDictionary globalDict = new GlobalSymbolDictionary();
// Create a very long symbol (1000 chars)
- String longSymbol = "X".repeat(1000);
+ String longSymbol = io.questdb.client.test.tools.TestUtils.repeat("X", 1000);
int id = globalDict.getOrAddSymbol(longSymbol);
Assert.assertEquals(0, id);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
index 4e28b14b..0733de8f 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java
@@ -28,6 +28,7 @@
import io.questdb.client.SenderError;
import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.Test;
@@ -330,7 +331,7 @@ private static void awaitAtLeastOneConnectAttempt(QwpWebSocketSender wss) {
throw new AssertionError(
"I/O thread did not log a connect attempt within 5s");
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
@@ -427,7 +428,7 @@ SenderError get() {
}
@Override
- public void onError(SenderError err) {
+ public void onError(@NotNull SenderError err) {
if (ref.compareAndSet(null, err)) {
latch.countDown();
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java
index 19ddd2fc..9d422ee9 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpBindEncoderTest.java
@@ -862,7 +862,7 @@ public void testEncodeVarcharUnicode() throws Exception {
public void testEncoderGrowsBufferBeyondDefault() throws Exception {
assertMemoryLeak(() -> {
try (QwpBindValues binds = new QwpBindValues()) {
- String big = "x".repeat(20_000);
+ String big = io.questdb.client.test.tools.TestUtils.repeat("x", 20_000);
binds.setVarchar(0, big);
Assert.assertEquals(1, binds.count());
// type(1) + flag(1) + offset0(4) + len(4) + 20000 bytes = 20010
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
index 067a8a97..f16bc267 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java
@@ -44,6 +44,7 @@
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -218,7 +219,7 @@ public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Except
}
assertRowsEqual(
- List.of(decodedRow(
+ Collections.singletonList(decodedRow(
"arrays",
"la", longArrayValue(shape(2), 9, 10),
"da", doubleArrayValue(shape(2), 9.5, 10.5)
@@ -232,7 +233,7 @@ public void testArrayWrapperStagingSnapshotsMutationAndCancelRow() throws Except
public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
assertMemoryLeak(() -> {
String large = repeat('x', 5000);
- List oversizedRow = List.of(
+ List oversizedRow = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("a", 2)
.stringColumn("s", large)
@@ -272,7 +273,7 @@ public void testAtMicrosOversizeFailureRollsBackWithoutLeakingTimestampState() t
public void testAtNanosOversizeFailureRollsBackWithoutLeakingTimestampState() throws Exception {
assertMemoryLeak(() -> {
String large = repeat('x', 5000);
- List oversizedRow = List.of(
+ List oversizedRow = Collections.singletonList(
row("tn", sender -> sender.table("tn")
.longColumn("a", 2)
.stringColumn("s", large)
@@ -335,7 +336,7 @@ public void testAtNowAllowsTimestampOnlyRowWhenTableHasColumns() throws Exceptio
public void testAtNowOversizeFailureRollsBackWithoutExplicitCancel() throws Exception {
assertMemoryLeak(() -> {
String large = repeat('x', 5000);
- List oversizedRow = List.of(
+ List oversizedRow = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("a", 2)
.stringColumn("s", large)
@@ -389,7 +390,7 @@ public void testAtNowRejectsEmptyRowOnFirstRow() throws Exception {
sender.flush();
}
- assertRowsEqual(List.of(
+ assertRowsEqual(Collections.singletonList(
decodedRow("t", "x", 1L)
), decodeRows(nf.packets));
});
@@ -875,7 +876,7 @@ public void testCloseDropsInProgressRowButFlushesCommittedRows() throws Exceptio
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
+ assertRowsEqual(Collections.singletonList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
});
}
@@ -918,7 +919,7 @@ public void testEstimateMatchesActualEncodedSize() throws Exception {
@Test
public void testFirstRowAllowsMultipleNewColumnsAndEncodesRow() throws Exception {
assertMemoryLeak(() -> {
- List rows = List.of(
+ List rows = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("a", 1)
.doubleColumn("b", 2.0)
@@ -973,7 +974,7 @@ public void testFlushWhileRowInProgressThrowsAndPreservesRow() throws Exception
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
+ assertRowsEqual(Collections.singletonList(decodedRow("t", "x", 1L)), decodeRows(nf.packets));
});
}
@@ -993,7 +994,7 @@ public void testIrregularArrayRejectedDuringStagingAndSenderRemainsUsable() thro
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(decodedRow("ok", "x", 1L)), decodeRows(nf.packets));
+ assertRowsEqual(Collections.singletonList(decodedRow("ok", "x", 1L)), decodeRows(nf.packets));
});
}
@@ -1012,7 +1013,7 @@ public void testIrregularArrayRejectedDuringStagingDoesNotLeakColumnIntoSameTabl
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets));
+ assertRowsEqual(Collections.singletonList(decodedRow("arrays", "x", 1L)), decodeRows(nf.packets));
});
}
@@ -1034,7 +1035,7 @@ public void testLongArrayWrapperStagingSnapshotsMutation() throws Exception {
}
assertRowsEqual(
- List.of(decodedRow(
+ Collections.singletonList(decodedRow(
"arrays",
"la", longArrayValue(shape(2, 2), 1, 2, 3, 4)
)),
@@ -1158,7 +1159,7 @@ public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exceptio
doubleValues[i] = i + 0.25;
}
- List rows = List.of(
+ List rows = Collections.singletonList(
row("arrays", sender -> sender.table("arrays")
.longColumn("x", 1)
.longArray("la", longValues)
@@ -1189,7 +1190,7 @@ public void testOversizedArrayRowRejectedUsesActualEncodedSize() throws Exceptio
public void testOversizedRowAfterMidRowSchemaChangeCancelDoesNotLeakSchema() throws Exception {
assertMemoryLeak(() -> {
String large = repeat('x', 5000);
- List oversizedRow = List.of(
+ List oversizedRow = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("a", 2)
.stringColumn("s", large)
@@ -1230,7 +1231,7 @@ public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() thr
assertMemoryLeak(() -> {
String small = repeat('s', 32);
String large = repeat('x', 5000);
- List largeRow = List.of(
+ List largeRow = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("x", 2)
.stringColumn("s", large)
@@ -1257,7 +1258,7 @@ public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() thr
Assert.assertEquals(1, nf.sendCount);
assertPacketsWithinLimit(new RunResult(nf.packets, nf.lengths, nf.sendCount), maxDatagramSize);
assertRowsEqual(
- List.of(decodedRow("t", "x", 1L, "s", small)),
+ Collections.singletonList(decodedRow("t", "x", 1L, "s", small)),
decodeRows(nf.packets)
);
});
@@ -1267,7 +1268,7 @@ public void testOversizedSingleRowRejectedAfterReplayUsesActualEncodedSize() thr
public void testOversizedSingleRowRejectedBeforeReplayUsesActualEncodedSize() throws Exception {
assertMemoryLeak(() -> {
String large = repeat('x', 5000);
- List rows = List.of(
+ List rows = Collections.singletonList(
row("t", sender -> sender.table("t")
.longColumn("x", 1)
.stringColumn("s", large)
@@ -1561,7 +1562,7 @@ public void testSimpleLongRowUsesScatterSendPath() throws Exception {
Assert.assertEquals(1, nf.scatterSendCount);
Assert.assertEquals(0, nf.rawSendCount);
Assert.assertTrue("expected multiple segments for header/schema/data", nf.segmentCounts.get(0) > 1);
- assertRowsEqual(List.of(decodedRow("t", "x", 42L)), decodeRows(nf.packets));
+ assertRowsEqual(Collections.singletonList(decodedRow("t", "x", 42L)), decodeRows(nf.packets));
});
}
@@ -1693,7 +1694,7 @@ public void testTableRejectsEmptyName() throws Exception {
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(
+ assertRowsEqual(Collections.singletonList(
decodedRow("valid", "x", 1L)
), decodeRows(nf.packets));
});
@@ -1742,7 +1743,7 @@ public void testTableRejectsNullName() throws Exception {
}
Assert.assertEquals(1, nf.sendCount);
- assertRowsEqual(List.of(
+ assertRowsEqual(Collections.singletonList(
decodedRow("valid", "x", 1L)
), decodeRows(nf.packets));
});
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
index 271501b5..b7318a87 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java
@@ -326,7 +326,7 @@ public void testEncodeLongString() throws Exception {
try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder();
QwpTableBuffer buffer = new QwpTableBuffer("test_table")) {
- String sb = "a".repeat(10_000);
+ String sb = io.questdb.client.test.tools.TestUtils.repeat("a", 10_000);
QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("data", TYPE_VARCHAR, true);
col.addString(sb);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
index 32d2725d..e868dafc 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
@@ -172,7 +172,7 @@ private static int countPopulatedSegmentFiles(String dir) {
}
private static String repeat(String c, int n) {
- return String.valueOf(c).repeat(Math.max(0, n));
+ return io.questdb.client.test.tools.TestUtils.repeat(String.valueOf(c), Math.max(0, n));
}
private static void rmDirRec(String dir) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java
new file mode 100644
index 00000000..85522e9d
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java
@@ -0,0 +1,246 @@
+/*******************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.LockSupport;
+
+/**
+ * Pins the {@link QwpWebSocketSender#isSlotLockReleased()} contract that the
+ * {@link io.questdb.client.impl.SenderPool} leaked-slot accounting depends on:
+ *
+ * - Happy path — a clean {@code close()} that winds the I/O thread
+ * down reports {@code isSlotLockReleased() == true}.
+ * - Leak path — a {@code close()} that early-returns because the I/O
+ * thread refused to stop ({@code ioThreadStopped == false}) reports
+ * {@code isSlotLockReleased() == false}.
+ *
+ * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held,
+ * retire the slot permanently". Before this test that contract was driven only
+ * by tests that forged the {@code slotLockReleased} field directly, so
+ * the real production write path (and the direction of the getter) was never
+ * exercised: an inverted getter or a misplaced {@code slotLockReleased = true}
+ * would have gone unnoticed. These tests drive the real {@code close()} paths.
+ */
+public class SlotLockReleasedContractTest {
+
+ /**
+ * Happy path: a live sender has not released its slot lock; a clean
+ * {@code close()} (I/O thread stops normally) flips it to released.
+ */
+ @Test
+ public void testSlotLockReleasedAfterCleanClose() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String cfg = "ws::addr=localhost:" + port + ";close_flush_timeout_millis=2000;";
+ QwpWebSocketSender wss = (QwpWebSocketSender) Sender.fromConfig(cfg);
+ // A live, never-closed sender holds its slot: must report
+ // NOT-released (guards against the flag defaulting/initialising
+ // to true).
+ Assert.assertFalse(
+ "a live sender must not report its slot lock released",
+ wss.isSlotLockReleased());
+
+ wss.table("t").longColumn("v", 1L).atNow();
+ wss.flush();
+ wss.close();
+
+ // Clean close => I/O thread stopped => slot lock released.
+ Assert.assertTrue(
+ "a clean close() (I/O thread stopped) must release the slot lock",
+ wss.isSlotLockReleased());
+ }
+ });
+ }
+
+ /**
+ * Leak path: when {@code close()} cannot wind the I/O loop down it bails
+ * out via the {@code !ioThreadStopped} early-return and must leave the slot
+ * lock reported as NOT released.
+ *
+ * We can't make a real {@link CursorWebSocketSendLoop#close()} throw
+ * (the loop is {@code final}, its latch is {@code final}, and every throw
+ * inside is caught), so we forge the exact production precondition the catch
+ * at {@code QwpWebSocketSender.close()} reacts to: a loop whose
+ * {@code close()} throws while the I/O thread is still alive. After cleanly
+ * stopping the real I/O thread (so it doesn't leak), we null the loop's
+ * shutdown latch and point its {@code ioThread} at a live thread, so
+ * {@code shutdownLatch.await()} NPEs — exactly the "Error closing cursor
+ * send loop" path that sets {@code ioThreadStopped = false}.
+ */
+ @Test
+ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ try (TestWebSocketServer server = new TestWebSocketServer(new AckAllHandler())) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // close_flush_timeout_millis=0 => close() skips the drain step;
+ // we write no rows, so the drain block has nothing to do.
+ String cfg = "ws::addr=localhost:" + port + ";close_flush_timeout_millis=0;";
+ QwpWebSocketSender wss = (QwpWebSocketSender) Sender.fromConfig(cfg);
+ Assert.assertFalse("precondition: live sender, lock not released",
+ wss.isSlotLockReleased());
+
+ CursorWebSocketSendLoop loop = readField(wss, "cursorSendLoop", CursorWebSocketSendLoop.class);
+ Assert.assertNotNull("connected sender must have an I/O loop", loop);
+
+ try {
+ // 1) Cleanly stop the REAL I/O thread (so it does not leak);
+ // this also counts its shutdown latch down to zero.
+ Thread realIo = readField(loop, "ioThread", Thread.class);
+ setField(loop, "running", Boolean.FALSE);
+ if (realIo != null) {
+ LockSupport.unpark(realIo);
+ realIo.join(TimeUnit.SECONDS.toMillis(5));
+ Assert.assertFalse("real I/O thread must have stopped", realIo.isAlive());
+ }
+
+ // 2) Forge "the I/O thread refuses to stop": with the latch
+ // nulled and ioThread pointing at a live thread,
+ // loop.close()'s shutdownLatch.await() throws NPE -- the
+ // uncaught throwable production catches as
+ // ioThreadStopped=false.
+ setField(loop, "shutdownLatch", null);
+ setField(loop, "ioThread", Thread.currentThread());
+
+ // 3) Drive the real close() early-return. It rethrows the
+ // swallowed teardown error after the guard.
+ try {
+ wss.close();
+ Assert.fail("close() must surface the loop-teardown failure");
+ } catch (Throwable expected) {
+ // expected: rethrowTerminal() surfaces the captured NPE.
+ }
+
+ // 4) Contract under test: the I/O thread refused to stop, so
+ // the slot lock must NOT be reported released.
+ Assert.assertFalse(
+ "isSlotLockReleased() must be false when close() early-returns "
+ + "with the I/O thread still running",
+ wss.isSlotLockReleased());
+ } finally {
+ // close()'s early-return deliberately leaks the resources the
+ // still-running I/O thread might touch (to avoid SIGSEGV).
+ // Free them here -- the same set the post-guard close() tail
+ // would have freed -- so assertMemoryLeak passes.
+ freeFieldQuietly(wss, "buffer0");
+ freeFieldQuietly(wss, "buffer1");
+ freeFieldQuietly(wss, "client");
+ if (Boolean.TRUE.equals(readField(wss, "ownsCursorEngine", Boolean.class))) {
+ freeFieldQuietly(wss, "cursorEngine");
+ }
+ freeFieldQuietly(wss, "errorDispatcher");
+ freeFieldQuietly(wss, "progressDispatcher");
+ freeFieldQuietly(wss, "connectionDispatcher");
+ }
+ }
+ });
+ }
+
+ // ------------------------------------------------------------------ utils
+
+ private static void freeFieldQuietly(Object target, String name) {
+ try {
+ Object v = readField(target, name, Object.class);
+ if (v != null) {
+ v.getClass().getMethod("close").invoke(v);
+ }
+ } catch (Throwable ignored) {
+ // Best-effort cleanup of a leaked resource: a double-close (or an
+ // -ea AssertionError) must not mask the assertions above.
+ }
+ }
+
+ private static T readField(Object target, String name, Class type) throws Exception {
+ Class cls = target.getClass();
+ while (cls != null) {
+ try {
+ Field f = cls.getDeclaredField(name);
+ f.setAccessible(true);
+ return type.cast(f.get(target));
+ } catch (NoSuchFieldException e) {
+ cls = cls.getSuperclass();
+ }
+ }
+ throw new NoSuchFieldException(name);
+ }
+
+ private static void setField(Object target, String name, Object value) throws Exception {
+ Class cls = target.getClass();
+ while (cls != null) {
+ try {
+ Field f = cls.getDeclaredField(name);
+ f.setAccessible(true);
+ f.set(target, value);
+ return;
+ } catch (NoSuchFieldException e) {
+ cls = cls.getSuperclass();
+ }
+ }
+ throw new NoSuchFieldException(name);
+ }
+
+ /** ACKs every binary frame with a running sequence so flush/close drain cleanly. */
+ private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler {
+ private final AtomicLong nextSeq = new AtomicLong();
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ try {
+ client.sendBinary(buildAck(nextSeq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static byte[] buildAck(long seq) {
+ byte[] buf = new byte[1 + 8 + 2];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00);
+ bb.putLong(seq);
+ bb.putShort((short) 0);
+ return buf;
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WriteFailoverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WriteFailoverTest.java
index bb33eb1c..8ab444e8 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WriteFailoverTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WriteFailoverTest.java
@@ -337,7 +337,7 @@ public void testUpgradeException421CarriesRoleHeader() throws Exception {
// sits on the cause chain via initCause().
TestWebSocketServer replica = new TestWebSocketServer(new AckHandler());
int port = replica.getPort();
- try (replica) {
+ try (TestWebSocketServer replicaResource = replica) {
replica.setRejectWithRole("REPLICA");
replica.start();
Assert.assertTrue(replica.awaitStart(5, TimeUnit.SECONDS));
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopReconnectLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopReconnectLeakTest.java
index 759ea873..806682fe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopReconnectLeakTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopReconnectLeakTest.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -99,7 +99,7 @@ public void testCloseClosesLivePostReconnectClient() throws Exception {
+ wss.getTotalReconnectsSucceeded()
+ " successful reconnects");
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
// Reach into the loop to capture the live client BEFORE we
@@ -200,7 +200,7 @@ private void awaitAckProcessed(long baseline) {
"client never reported processing the ACK within 5s "
+ "(baseline=" + baseline + ", current=" + s.getTotalAcks() + ")");
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
index bf9f084a..162474fe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java
@@ -26,6 +26,7 @@
import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
import io.questdb.client.std.Files;
+import io.questdb.client.std.IntList;
import io.questdb.client.std.ObjList;
import io.questdb.client.test.tools.TestUtils;
import org.junit.After;
@@ -148,6 +149,162 @@ public void testMultipleOrphansReturned() throws Exception {
});
}
+ // ----------------------------------------------------------------------
+ // Bounded (managed-slot) exclusion: scan(sfDir, exclude, base, count).
+ // This is how a connection pool fences off its own live siblings while
+ // still draining same-base leftovers, and the fix for the
+ // "shrinking maxSize strands unacked SF data" bug.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testBoundedScanExcludesInRangeManagedSlots() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // Pool at maxSize=2 co-manages default-0 and default-1; a startup
+ // scan must not list a live sibling as an orphan. A foreign slot
+ // is still reported.
+ for (String name : new String[]{"default-0", "default-1"}) {
+ String slot = sfDir + "/" + name;
+ assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+ touchFile(slot + "/sf-0001.sfa");
+ }
+ String foreign = sfDir + "/legacy";
+ assertEquals(0, Files.mkdir(foreign, Files.DIR_MODE_DEFAULT));
+ touchFile(foreign + "/sf-0001.sfa");
+
+ // caller is default-0; managed set is [0, 2)
+ ObjList orphans = OrphanScanner.scan(sfDir, "default-0", "default", 2);
+ assertEquals("only the foreign slot is a candidate", 1, orphans.size());
+ assertEquals(foreign, orphans.get(0));
+ });
+ }
+
+ @Test
+ public void testBoundedScanDrainsOutOfRangeSameBaseSlotsAfterShrink() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // The bug: a previous run used maxSize=4 (default-0..3 hold unacked
+ // data); this run restarts at maxSize=2. default-0/1 are re-created
+ // and self-recovered (excluded), but default-2/3 are OUT of the new
+ // [0,2) index range forever -- they must be drainable, not stranded.
+ for (String name : new String[]{"default-0", "default-1", "default-2", "default-3"}) {
+ String slot = sfDir + "/" + name;
+ assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+ touchFile(slot + "/sf-0001.sfa");
+ }
+ // caller is default-0; managed set is [0, 2)
+ ObjList orphans = OrphanScanner.scan(sfDir, "default-0", "default", 2);
+ assertEquals("default-2 and default-3 must be drainable orphans", 2, orphans.size());
+ boolean has2 = false, has3 = false;
+ for (int i = 0; i < orphans.size(); i++) {
+ String p = orphans.get(i);
+ if (p.equals(sfDir + "/default-2")) has2 = true;
+ if (p.equals(sfDir + "/default-3")) has3 = true;
+ }
+ assertTrue("default-2 stranded", has2);
+ assertTrue("default-3 stranded", has3);
+ });
+ }
+
+ @Test
+ public void testListStrandedOutOfRangeManagedSlots() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // A previous run used maxSize=2; default-0/1 are in range, default-2/3
+ // are out of range and hold unacked data. A non-canonical same-base
+ // name (default-04) and a foreign base (other-9) must NOT be listed:
+ // the pool only ever minted canonical default-, so only those are
+ // its own stranded out-of-range slots.
+ for (String name : new String[]{"default-0", "default-1", "default-2", "default-3",
+ "default-04", "other-9"}) {
+ String slot = sfDir + "/" + name;
+ assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+ touchFile(slot + "/sf-0001.sfa");
+ }
+ // An out-of-range slot already flagged .failed must be skipped.
+ String failed = sfDir + "/default-5";
+ assertEquals(0, Files.mkdir(failed, Files.DIR_MODE_DEFAULT));
+ touchFile(failed + "/sf-0001.sfa");
+ OrphanScanner.markFailed(failed, "test-induced");
+ // An out-of-range slot with no segment file must be skipped.
+ String empty = sfDir + "/default-6";
+ assertEquals(0, Files.mkdir(empty, Files.DIR_MODE_DEFAULT));
+
+ IntList stranded = OrphanScanner.listStrandedOutOfRangeManagedSlots(sfDir, "default", 2);
+ assertEquals("only canonical same-base i>=2 with unacked data", 2, stranded.size());
+ boolean has2 = false, has3 = false;
+ for (int i = 0; i < stranded.size(); i++) {
+ if (stranded.getQuick(i) == 2) has2 = true;
+ if (stranded.getQuick(i) == 3) has3 = true;
+ }
+ assertTrue("default-2 must be listed", has2);
+ assertTrue("default-3 must be listed", has3);
+
+ // count <= 0 / null / empty base disables the bound.
+ assertEquals(0, OrphanScanner.listStrandedOutOfRangeManagedSlots(sfDir, null, 2).size());
+ assertEquals(0, OrphanScanner.listStrandedOutOfRangeManagedSlots(sfDir, "", 2).size());
+ });
+ }
+
+ @Test
+ public void testBoundedScanDrainsNonCanonicalSameBaseNames() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // The pool only ever mints canonical Integer.toString suffixes
+ // ("0","1",...). A same-base dir with a leading-zero or non-numeric
+ // suffix is not a managed slot, so it is drained like any foreign
+ // leftover -- even if its numeric value would fall inside [0,count).
+ for (String name : new String[]{"default-00", "default-01", "default-foo", "default-"}) {
+ String slot = sfDir + "/" + name;
+ assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT));
+ touchFile(slot + "/sf-0001.sfa");
+ }
+ // also a genuinely-managed slot that must stay excluded
+ String managed = sfDir + "/default-1";
+ assertEquals(0, Files.mkdir(managed, Files.DIR_MODE_DEFAULT));
+ touchFile(managed + "/sf-0001.sfa");
+
+ ObjList orphans = OrphanScanner.scan(sfDir, "default-0", "default", 4);
+ assertEquals("all non-canonical same-base names are drainable", 4, orphans.size());
+ for (int i = 0; i < orphans.size(); i++) {
+ assertFalse("managed slot must not appear",
+ orphans.get(i).equals(managed));
+ }
+ });
+ }
+
+ @Test
+ public void testBoundedScanDisabledWhenCountNonPositive() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // count <= 0 or null/empty base disables the exclusion: every
+ // sibling with data (except the explicit excludeSlotName) is a
+ // candidate.
+ String sibling = sfDir + "/default-1";
+ assertEquals(0, Files.mkdir(sibling, Files.DIR_MODE_DEFAULT));
+ touchFile(sibling + "/sf-0001.sfa");
+
+ assertEquals(1, OrphanScanner.scan(sfDir, "default-0", "default", 0).size());
+ assertEquals(1, OrphanScanner.scan(sfDir, "default-0", null, 4).size());
+ assertEquals(1, OrphanScanner.scan(sfDir, "default-0", "", 4).size());
+ });
+ }
+
+ @Test
+ public void testIsManagedSlotPredicate() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // In-range canonical indices are managed.
+ assertTrue(OrphanScanner.isManagedSlot("default-0", "default-", 2));
+ assertTrue(OrphanScanner.isManagedSlot("default-1", "default-", 2));
+ // At-or-above the count is NOT managed (drainable).
+ assertFalse(OrphanScanner.isManagedSlot("default-2", "default-", 2));
+ assertFalse(OrphanScanner.isManagedSlot("default-10", "default-", 2));
+ // Non-canonical / foreign suffixes are NOT managed.
+ assertFalse(OrphanScanner.isManagedSlot("default-00", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("default-01", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("default-foo", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("default-", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("default--1", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("other-0", "default-", 4));
+ assertFalse(OrphanScanner.isManagedSlot("default", "default-", 4));
+ });
+ }
+
@Test
public void testIsCandidateOrphanDirect() throws Exception {
TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerRecoveryCapTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerRecoveryCapTest.java
index c8168d3c..eed6bc87 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerRecoveryCapTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerRecoveryCapTest.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -86,7 +86,7 @@ public void testManagerHonorsCapAgainstRecoveredSegmentsOnRegister() throws Exce
Assert.assertNotNull("recovery should produce a ring", ring);
SegmentManager manager = new SegmentManager(SEGMENT_SIZE, 1_000_000L /* 1ms */, cap);
- try (manager) {
+ try (SegmentManager ignored = manager) {
manager.start();
manager.register(ring, slotDir);
// Give the manager several ticks. With the bug, it provisions
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java
index 3ee4301b..0c102fce 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -118,7 +118,7 @@ public void testProducerCanRotateAcrossManySegmentsWithoutBackpressure() throws
throw new AssertionError(
"stuck waiting for spare at i=" + i + ", needsSpare=" + ring.needsHotSpare());
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
assertEquals(i, fsn);
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java
index 431dc25e..644b2709 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java
@@ -189,7 +189,7 @@ private static void awaitParked(Thread t) {
if (System.nanoTime() > deadline) {
throw new AssertionError("worker did not park within 5 s; state=" + s);
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java
index 85efd40c..8cc05407 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java
@@ -213,7 +213,7 @@ private static void awaitParked(Thread t) {
if (System.nanoTime() > deadline) {
throw new AssertionError("worker did not park within 5 s; state=" + s);
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
@@ -223,7 +223,7 @@ private static void awaitSpare(SegmentRing ring, String where) {
if (System.nanoTime() > deadline) {
throw new AssertionError("spare never installed " + where);
}
- Thread.onSpinWait();
+ io.questdb.client.std.Compat.onSpinWait();
}
}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java
index 172d118f..9f533fc7 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java
@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*+*****************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
@@ -129,7 +129,7 @@ public void testRecoveryUnlinksEmptyOrphansAlongsideValidSegments() throws Excep
Assert.assertTrue("setup: orphan .sfa should exist", Files.exists(orphanPath));
SegmentRing recovered = SegmentRing.openExisting(tmpDir, SEGMENT_SIZE);
- try (recovered) {
+ try (SegmentRing ignored = recovered) {
Assert.assertNotNull("recovery dropped the valid segment", recovered);
Assert.assertTrue(
"recovery should keep the valid segment on disk",
diff --git a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java
new file mode 100644
index 00000000..3994a1d2
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java
@@ -0,0 +1,261 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.impl;
+
+import io.questdb.client.QueryException;
+import io.questdb.client.cutlass.qwp.client.QwpQueryClient;
+import io.questdb.client.impl.QueryClientPool;
+import io.questdb.client.impl.QueryWorker;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.Unsafe;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+
+// Error-safety of the three QueryClientPool creation paths the teardown-hardening
+// fix widened from catch (RuntimeException) to catch (Throwable). The native
+// build/connect path runs under -ea and can throw an Error (AssertionError,
+// OutOfMemoryError); the old catches let that Error skip cleanup.
+//
+// QwpQueryClient is a concrete class with no fake seam, so these tests inject an
+// Error at the real connect step via the package-private connectHook constructor
+// (reached by reflection -- the main module is declared `open`). fromConfig()
+// still runs for real, committing the NATIVE_DEFAULT scratch the cleanup must
+// reclaim, so the memory assertions are meaningful.
+public class QueryClientPoolErrorSafetyTest {
+
+ // ws config that fromConfig() parses without opening a socket; the injected
+ // connectHook replaces connect(), so the port is never dialled.
+ private static final String CFG = "ws::addr=127.0.0.1:9000;";
+
+ // Site: acquire() inner catch around client.connect(). createUnlocked() must
+ // close the half-built client when connect throws an Error, otherwise the
+ // field-initialised QwpBindValues scratch (NATIVE_DEFAULT) leaks.
+ // RED: catch (RuntimeException) -> Error skips client.close() -> leak.
+ // GREEN: catch (Throwable) -> client.close() runs -> no leak.
+ @Test(timeout = 30_000)
+ public void acquireDoesNotLeakNativeScratchOnErrorFromConnect() throws Exception {
+ QueryClientPool pool = newPool(CFG, 0, 1, 250, alwaysThrow());
+ try {
+ long baseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ try {
+ pool.acquire();
+ Assert.fail("expected acquire() to propagate the injected Error");
+ } catch (Throwable expected) {
+ // wrapped or raw -- the leak check is the discriminator
+ }
+ long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ Assert.assertEquals(
+ "acquire() leaked NATIVE_DEFAULT scratch on an Error from connect()",
+ baseline, after);
+ } finally {
+ pool.close();
+ }
+ }
+
+ // Site: acquire() outer catch around createUnlocked()/start(). An Error must
+ // still run inFlightCreations--, otherwise the reserved slot is leaked and
+ // (maxSize == 1) the pool is wedged forever.
+ // RED: catch (RuntimeException) -> inFlightCreations stuck at 1.
+ // GREEN: catch (Throwable) -> inFlightCreations restored to 0.
+ @Test(timeout = 30_000)
+ public void acquireRestoresInFlightCreationsOnErrorFromConnect() throws Exception {
+ QueryClientPool pool = newPool(CFG, 0, 1, 250, alwaysThrow());
+ try {
+ try {
+ pool.acquire();
+ Assert.fail("expected acquire() to propagate the injected Error");
+ } catch (Throwable expected) {
+ // expected
+ }
+
+ Assert.assertEquals(
+ "acquire() leaked an in-flight creation slot on an Error from connect()",
+ 0, inFlightCreations(pool));
+
+ // Corollary: capacity is usable again -- the next acquire() must
+ // reach the creation path (and fail there) rather than time out.
+ try {
+ pool.acquire();
+ Assert.fail("expected second acquire() to re-attempt creation");
+ } catch (QueryException e) {
+ Assert.assertFalse(
+ "pool wedged: second acquire() timed out -> capacity permanently lost ("
+ + e.getMessage() + ")",
+ e.getMessage() != null && e.getMessage().contains("timed out"));
+ } catch (Throwable injectedAgain) {
+ // also fine: the Error surfaced again from the re-attempt
+ }
+ } finally {
+ pool.close();
+ }
+ }
+
+ // Site: constructor prewarm outer catch. An Error mid-prewarm must run the
+ // cleanup loop that shuts down already-built workers, otherwise the first
+ // worker's client (NATIVE_DEFAULT) and I/O thread leak.
+ // RED: catch (RuntimeException) -> first worker's client never closed.
+ // GREEN: catch (Throwable) -> cleanup loop closes it -> no leak.
+ @Test(timeout = 30_000)
+ public void preWarmDoesNotLeakNativeScratchOnErrorFromConnect() throws Exception {
+ long baseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ // First connect() succeeds (no-op, leaves the client unconnected but
+ // built); the second throws an Error mid-prewarm.
+ AtomicInteger calls = new AtomicInteger();
+ Consumer hook = client -> {
+ if (calls.incrementAndGet() >= 2) {
+ throw new AssertionError("injected native connect failure");
+ }
+ };
+ try {
+ newPool(CFG, 2, 2, 250, hook);
+ Assert.fail("expected prewarm to propagate the injected Error");
+ } catch (Throwable expected) {
+ // expected -- construction aborts
+ }
+ long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ Assert.assertEquals(
+ "prewarm leaked NATIVE_DEFAULT scratch of an already-built worker on an Error",
+ baseline, after);
+ }
+
+ // Site: acquire() outer catch around createUnlocked()/start(). When start()
+ // throws *after* createUnlocked() returned a fully connected client, that
+ // client (NATIVE_DEFAULT scratch + socket + I/O thread) must be torn down,
+ // otherwise it is stranded -- nothing else references it. This is the
+ // start()-throws path connectHook cannot reach: connect() runs (no-op here,
+ // committing the scratch) and the failure is injected at thread start.
+ // RED: catch does inFlightCreations-- but never created.shutdown() -> leak.
+ // GREEN: catch calls created.shutdown() -> client.close() -> no leak.
+ @Test(timeout = 30_000)
+ public void acquireDoesNotLeakNativeScratchOnErrorFromStart() throws Exception {
+ QueryClientPool pool = newPool(CFG, 0, 1, 250, noConnect(), alwaysThrowStart());
+ try {
+ long baseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ try {
+ pool.acquire();
+ Assert.fail("expected acquire() to propagate the injected start Error");
+ } catch (Throwable expected) {
+ // wrapped or raw -- the leak check is the discriminator
+ }
+ long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ Assert.assertEquals(
+ "acquire() leaked NATIVE_DEFAULT scratch on an Error from start()",
+ baseline, after);
+ // The reservation must also be restored so the pool is not wedged.
+ Assert.assertEquals(
+ "acquire() leaked an in-flight creation slot on an Error from start()",
+ 0, inFlightCreations(pool));
+ } finally {
+ pool.close();
+ }
+ }
+
+ // Site: constructor prewarm. When start() throws after createUnlocked()
+ // returned a fully connected client, the just-built worker is not yet in
+ // `all`, so the cleanup loop (which only walks `all`) would skip it. Its
+ // client (NATIVE_DEFAULT) must still be torn down.
+ // RED: cleanup loop walks `all` only -> the worker that failed at start()
+ // is never closed -> leak.
+ // GREEN: the pending-worker teardown closes it -> no leak.
+ @Test(timeout = 30_000)
+ public void preWarmDoesNotLeakNativeScratchOnErrorFromStart() throws Exception {
+ long baseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ // First worker is admitted to `all` (start is a no-op here -- the test
+ // never runs queries, and an unstarted thread keeps the later
+ // shutdown()'s join() instant); the second throws at start() after its
+ // client is fully built. That second worker is the stranded one: it
+ // never made it into `all`, so only the new pending-worker teardown
+ // closes it. The assertion catches a leak of EITHER worker's scratch.
+ AtomicInteger calls = new AtomicInteger();
+ Consumer startHook = w -> {
+ if (calls.incrementAndGet() >= 2) {
+ throw new AssertionError("injected thread-start failure");
+ }
+ // first worker: leave the dispatch thread unstarted (see above)
+ };
+ try {
+ newPool(CFG, 2, 2, 250, noConnect(), startHook);
+ Assert.fail("expected prewarm to propagate the injected start Error");
+ } catch (Throwable expected) {
+ // expected -- construction aborts
+ }
+ long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT);
+ Assert.assertEquals(
+ "prewarm leaked NATIVE_DEFAULT scratch of a start()-failed worker",
+ baseline, after);
+ }
+
+ private static Consumer alwaysThrow() {
+ return client -> {
+ throw new AssertionError("injected native connect failure");
+ };
+ }
+
+ // connectHook that connects nothing: fromConfig() has already committed the
+ // NATIVE_DEFAULT scratch, so the client is half-built (scratch, no socket)
+ // -- exactly the state createUnlocked() returns before start() runs.
+ private static Consumer noConnect() {
+ return client -> {
+ };
+ }
+
+ private static Consumer alwaysThrowStart() {
+ return worker -> {
+ throw new AssertionError("injected thread-start failure");
+ };
+ }
+
+ private static int inFlightCreations(QueryClientPool pool) throws Exception {
+ Method m = QueryClientPool.class.getDeclaredMethod("inFlightCreations");
+ m.setAccessible(true);
+ return (int) m.invoke(pool);
+ }
+
+ private static QueryClientPool newPool(
+ String cfg, int min, int max, long acquireMs, Consumer connectHook
+ ) throws Exception {
+ Constructor c = QueryClientPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class, Consumer.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, connectHook);
+ }
+
+ private static QueryClientPool newPool(
+ String cfg, int min, int max, long acquireMs,
+ Consumer connectHook, Consumer startHook
+ ) throws Exception {
+ Constructor c = QueryClientPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class,
+ Consumer.class, Consumer.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE,
+ connectHook, startHook);
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplErrorSafetyTest.java
new file mode 100644
index 00000000..93b10301
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplErrorSafetyTest.java
@@ -0,0 +1,154 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.impl;
+
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.qwp.client.QwpQueryClient;
+import io.questdb.client.impl.QuestDBImpl;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Proxy;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.function.IntFunction;
+
+// Error-safety of the QuestDBImpl constructor that ties SenderPool and
+// QueryClientPool together. The constructor builds the SenderPool first, then
+// the QueryClientPool; both run heavy native build/connect paths that, under -ea,
+// can throw an Error (AssertionError, OutOfMemoryError). If the orchestrator's
+// cleanup catch is narrower than the pools' own (catch (RuntimeException) vs
+// catch (Throwable)), an Error from QueryClientPool construction propagates
+// without closing the already-built SenderPool, stranding its prewarmed
+// delegates (flock + mmap'd ring + I/O thread).
+//
+// Sender is an interface, faked with a Proxy whose close() flips a flag, injected
+// via the SenderPool senderFactory seam. The connect Error is injected via the
+// QueryClientPool connectHook seam. Both are passed through the package-private
+// QuestDBImpl seam constructor (reached by reflection -- the main module is
+// declared `open`); production callers pass null for both.
+public class QuestDBImplErrorSafetyTest {
+
+ // Non-SF http config: the SenderPool factory replaces the build, but the
+ // constructor's eager config probe must still parse it.
+ private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;";
+ // ws config that fromConfig() parses without opening a socket; the injected
+ // connectHook replaces connect(), so the port is never dialled.
+ private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;";
+
+ // RED: catch (RuntimeException) -> the Error from QueryClientPool construction
+ // skips the cleanup block -> the prewarmed SenderPool is never closed ->
+ // its fake delegate's close() is never called.
+ // GREEN: catch (Throwable) -> cleanup closes the SenderPool -> the fake
+ // delegate's close() runs.
+ @Test(timeout = 30_000)
+ public void ctorClosesBuiltSenderPoolWhenQueryPoolConstructionThrowsError() throws Exception {
+ AtomicBoolean senderClosed = new AtomicBoolean(false);
+ // senderMin = 1 -> SenderPool prewarms one observable delegate.
+ IntFunction senderFactory = slotIndex -> fakeSender(senderClosed);
+ // queryMin = 1 -> QueryClientPool prewarm reaches connect(), which throws.
+ Consumer connectHook = client -> {
+ throw new AssertionError("injected native connect failure");
+ };
+
+ try {
+ newQuestDB(senderFactory, connectHook);
+ Assert.fail("expected QuestDBImpl construction to propagate the injected Error");
+ } catch (Throwable expected) {
+ // expected -- construction aborts
+ }
+
+ Assert.assertTrue(
+ "QuestDBImpl ctor leaked the already-built SenderPool on an Error from "
+ + "QueryClientPool construction: the prewarmed delegate's close() was never called",
+ senderClosed.get());
+ }
+
+ private static Sender fakeSender(AtomicBoolean closedFlag) {
+ return (Sender) Proxy.newProxyInstance(
+ Sender.class.getClassLoader(),
+ new Class[]{Sender.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "close":
+ closedFlag.set(true);
+ return null;
+ case "toString":
+ return "FakeSender";
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "equals":
+ return proxy == args[0];
+ default:
+ Class rt = method.getReturnType();
+ if (rt == boolean.class) return false;
+ if (rt == byte.class) return (byte) 0;
+ if (rt == short.class) return (short) 0;
+ if (rt == int.class) return 0;
+ if (rt == long.class) return 0L;
+ if (rt == float.class) return 0f;
+ if (rt == double.class) return 0d;
+ if (rt == char.class) return (char) 0;
+ if (rt == void.class) return null;
+ if (rt.isInstance(proxy)) return proxy;
+ return null;
+ }
+ });
+ }
+
+ private static QuestDBImpl newQuestDB(
+ IntFunction senderFactory, Consumer connectHook
+ ) throws Exception {
+ Constructor c = QuestDBImpl.class.getDeclaredConstructor(
+ String.class, String.class, int.class, int.class, int.class, int.class,
+ long.class, long.class, long.class, long.class,
+ IntFunction.class, Consumer.class);
+ c.setAccessible(true);
+ try {
+ return c.newInstance(
+ SENDER_CFG, QUERY_CFG,
+ /*senderMin*/ 1, /*senderMax*/ 1,
+ /*queryMin*/ 1, /*queryMax*/ 1,
+ /*acquireTimeoutMillis*/ 250L,
+ /*idleTimeoutMillis*/ Long.MAX_VALUE,
+ /*maxLifetimeMillis*/ Long.MAX_VALUE,
+ /*housekeeperIntervalMillis*/ Long.MAX_VALUE,
+ senderFactory, connectHook);
+ } catch (InvocationTargetException e) {
+ // Unwrap so the caller sees the real construction failure (Error or
+ // RuntimeException), matching a direct constructor invocation.
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof Error) {
+ throw (Error) cause;
+ }
+ throw e;
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java
new file mode 100644
index 00000000..b7b56e7a
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java
@@ -0,0 +1,255 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.impl;
+
+import io.questdb.client.Sender;
+import io.questdb.client.impl.SenderPool;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Proxy;
+import java.nio.file.Paths;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.IntFunction;
+
+// Error-safety of the SenderPool constructor prewarm catch that the
+// teardown-hardening fix widened from catch (RuntimeException) to
+// catch (Throwable). createUnlocked() runs a heavy native build path (mmap,
+// flock, WebSocket connect) that can throw an Error under -ea; if the prewarm
+// catch does not fire, the cleanup loop never runs and every already-built
+// delegate leaks its flock + mmap'd ring + I/O thread -- resurrecting
+// "sf slot already in use" on the next attempt.
+//
+// Sender is an interface, so the build path is faked with a Proxy whose close()
+// flips a flag. The fake is injected via the package-private senderFactory
+// constructor (reached by reflection -- the main module is declared `open`).
+public class SenderPoolErrorSafetyTest {
+
+ // Non-SF http config: fromConfig is never reached (the factory replaces the
+ // build), but the constructor's eager config probe must still parse it.
+ private static final String CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;";
+
+ // RED: catch (RuntimeException) -> the Error from the 2nd build skips the
+ // cleanup loop -> the 1st delegate is never closed.
+ // GREEN: catch (Throwable) -> the cleanup loop closes the 1st delegate.
+ @Test(timeout = 30_000)
+ public void preWarmClosesBuiltDelegatesWhenBuildThrowsError() throws Exception {
+ AtomicBoolean firstClosed = new AtomicBoolean(false);
+ AtomicInteger calls = new AtomicInteger();
+ IntFunction factory = slotIndex -> {
+ if (calls.incrementAndGet() >= 2) {
+ throw new AssertionError("injected native build failure");
+ }
+ return fakeSender(firstClosed);
+ };
+
+ try {
+ newPool(CFG, 2, 2, 250, factory);
+ Assert.fail("expected prewarm to propagate the injected Error");
+ } catch (Throwable expected) {
+ // expected -- construction aborts
+ }
+
+ Assert.assertTrue(
+ "prewarm leaked an already-built delegate: its close() was never called on an Error",
+ firstClosed.get());
+ }
+
+ // Companion to the catch (RuntimeException) -> track-normal-completion fix in
+ // PooledSender.close(). flush() can exit with an Error (AssertionError under
+ // -ea, OutOfMemoryError, ...) as well as a RuntimeException; the wrapper is
+ // unsafe to recycle either way because Sender does not clear its buffer on a
+ // failed flush and WebSocket transport latches the failure.
+ //
+ // RED (catch (RuntimeException)): the AssertionError from flush() is not
+ // caught, broken stays false, the finally runs giveBack() -> the broken
+ // wrapper is recycled -> the next borrow() hands back the SAME instance.
+ // GREEN (track normal completion): flush() throwing leaves flushed=false ->
+ // discardBroken() -> the next borrow() builds a fresh wrapper.
+ @Test(timeout = 30_000)
+ public void flushErrorDiscardsBrokenSenderInsteadOfRecycling() throws Exception {
+ IntFunction factory = slotIndex -> flushThrowingSender();
+
+ try (SenderPool pool = newPool(CFG, 1, 1, 1_000, factory)) {
+ Sender first = pool.borrow();
+ try {
+ first.close();
+ Assert.fail("close() must propagate the Error thrown by flush()");
+ } catch (AssertionError expected) {
+ // expected: the original throwable propagates naturally
+ }
+
+ Sender second = pool.borrow();
+ try {
+ Assert.assertNotSame(
+ "a sender whose flush() exited with an Error must be discarded, not recycled",
+ first, second);
+ } finally {
+ // second's flush() also throws on close(); swallow on teardown.
+ try {
+ second.close();
+ } catch (AssertionError ignored) {
+ // expected
+ }
+ }
+ }
+ }
+
+ // Like fakeSender(), but flush() throws an Error to drive the
+ // PooledSender.close() abnormal-exit branch.
+ private static Sender flushThrowingSender() {
+ return (Sender) Proxy.newProxyInstance(
+ Sender.class.getClassLoader(),
+ new Class[]{Sender.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "flush":
+ throw new AssertionError("injected flush failure");
+ case "close":
+ return null;
+ case "toString":
+ return "FlushThrowingSender";
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "equals":
+ return proxy == args[0];
+ default:
+ Class rt = method.getReturnType();
+ if (rt == boolean.class) return false;
+ if (rt == byte.class) return (byte) 0;
+ if (rt == short.class) return (short) 0;
+ if (rt == int.class) return 0;
+ if (rt == long.class) return 0L;
+ if (rt == float.class) return 0f;
+ if (rt == double.class) return 0d;
+ if (rt == char.class) return (char) 0;
+ if (rt == void.class) return null;
+ // fluent ArraySender methods return Sender
+ if (rt.isInstance(proxy)) return proxy;
+ return null;
+ }
+ });
+ }
+
+ // Companion to the SF slot-index reservation in borrow(): when
+ // createUnlocked() throws on a borrow-triggered grow, the reserved slot
+ // index MUST be returned via freeSlotIndex(). Otherwise slotInUse[idx] is
+ // stuck true, pool capacity is permanently lowered, and the next borrow()
+ // either trips the "no free SF slot index" invariant in allocateSlotIndex()
+ // or eventually only times out -- the exact failure mode this PR fixes.
+ //
+ // The other SenderPool error-injection test fails in the constructor
+ // pre-warm loop with a non-SF config (slotIndex == -1), so neither the
+ // borrow-path freeSlotIndex nor the SF (slotIndex >= 0) case is otherwise
+ // covered.
+ //
+ // RED (freeSlotIndex(slotIndex) removed from the borrow catch): the 2nd
+ // borrow() throws IllegalStateException out of allocateSlotIndex().
+ // GREEN (slot index returned): the 2nd borrow() reuses the slot and
+ // succeeds, proving capacity survived the failed grow.
+ @Test(timeout = 30_000)
+ public void borrowReleasesSfSlotIndexWhenCreationFails() throws Exception {
+ // Unique, non-existent sf_dir: minSize=0 means no pre-warm, so the dir
+ // is never created and the constructor's startup SF recovery is a no-op.
+ // The factory replaces createUnlocked(), so localhost:1 is never dialed.
+ String sfDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-sf-borrowfail-" + System.nanoTime()).toString();
+ String sfCfg = "ws::addr=localhost:1;sf_dir=" + sfDir + ";";
+
+ AtomicInteger calls = new AtomicInteger();
+ IntFunction factory = slotIndex -> {
+ // First borrow-triggered build fails (the slot index reserved for
+ // it must be released); later builds succeed.
+ if (calls.getAndIncrement() == 0) {
+ throw new AssertionError("injected native build failure on first grow");
+ }
+ return fakeSender(new AtomicBoolean());
+ };
+
+ try (SenderPool pool = newPool(sfCfg, 0, 1, 2_000, factory)) {
+ try {
+ pool.borrow();
+ Assert.fail("borrow() must propagate the Error from the failed build");
+ } catch (AssertionError expected) {
+ // expected: the original throwable propagates out of borrow()
+ }
+
+ // The single SF slot index must have been returned to the free set.
+ // If it leaked, this borrow() trips the capacity invariant (or, in
+ // the timeout-only variant, exhausts the acquire budget).
+ Sender second = pool.borrow();
+ try {
+ Assert.assertNotNull(
+ "after a failed grow the SF slot index must be reusable", second);
+ } finally {
+ second.close();
+ }
+ }
+ }
+
+ private static Sender fakeSender(AtomicBoolean closedFlag) {
+ return (Sender) Proxy.newProxyInstance(
+ Sender.class.getClassLoader(),
+ new Class[]{Sender.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "close":
+ closedFlag.set(true);
+ return null;
+ case "toString":
+ return "FakeSender";
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "equals":
+ return proxy == args[0];
+ default:
+ Class rt = method.getReturnType();
+ if (rt == boolean.class) return false;
+ if (rt == byte.class) return (byte) 0;
+ if (rt == short.class) return (short) 0;
+ if (rt == int.class) return 0;
+ if (rt == long.class) return 0L;
+ if (rt == float.class) return 0f;
+ if (rt == double.class) return 0d;
+ if (rt == char.class) return (char) 0;
+ if (rt == void.class) return null;
+ // fluent ArraySender methods return Sender
+ if (rt.isInstance(proxy)) return proxy;
+ return null;
+ }
+ });
+ }
+
+ private static SenderPool newPool(
+ String cfg, int min, int max, long acquireMs, IntFunction senderFactory
+ ) throws Exception {
+ Constructor c = SenderPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class, IntFunction.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, senderFactory);
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
new file mode 100644
index 00000000..e4b2b49a
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
@@ -0,0 +1,2063 @@
+/*+*****************************************************************************
+ * ___ _ ____ ____
+ * / _ \ _ _ ___ ___| |_| _ \| __ )
+ * | | | | | | |/ _ \/ __| __| | | | _ \
+ * | |_| | |_| | __/\__ \ |_| |_| | |_) |
+ * \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ * Copyright (c) 2014-2019 Appsicle
+ * Copyright (c) 2019-2026 QuestDB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.impl;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+import io.questdb.client.Sender;
+import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner;
+import io.questdb.client.impl.PooledSender;
+import io.questdb.client.impl.SenderPool;
+import io.questdb.client.std.Files;
+import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
+import io.questdb.client.test.tools.TestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.IntFunction;
+
+/**
+ * Exhaustive tests for {@link SenderPool} interaction with store-and-forward
+ * (SF) slots.
+ *
+ * The pool reuses one immutable config string for every sender it builds, so
+ * before the slot-id fix every SF sender inherited the same {@code sender_id},
+ * pointed at the same {@code /} slot, and the second sender
+ * to start died with "sf slot already in use by another process". These tests
+ * pin down the fix: each pooled SF sender gets a distinct, stable slot id
+ * {@code -}; indices are reused deterministically; a slot is only
+ * returned to the free set once its delegate releases the {@code flock}; and
+ * the cross-writer guard that the slot lock exists for is still enforced
+ * between independent pools.
+ */
+public class SenderPoolSfTest {
+
+ private String sfDir;
+
+ @Before
+ public void setUp() {
+ sfDir = Paths.get(System.getProperty("java.io.tmpdir"),
+ "qdb-sf-pool-" + System.nanoTime()).toString();
+ }
+
+ @After
+ public void tearDown() {
+ rmDir(sfDir);
+ }
+
+ // ----------------------------------------------------------------------
+ // Core fix: the original claim repro -- two concurrent SF senders.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testTwoConcurrentSfSendersGetDistinctSlots() throws Exception {
+ // The exact scenario from the bug report: a maxSize=2 SF pool must
+ // hand out two live senders. Pre-fix, the second borrow() blew up on
+ // the slot flock.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ try {
+ Assert.assertNotSame("two borrows must be distinct wrappers", a, b);
+ Assert.assertTrue("slot default-0 must exist", Files.exists(slot("default-0")));
+ Assert.assertTrue("slot default-1 must exist", Files.exists(slot("default-1")));
+ Assert.assertEquals("exactly two slot dirs", 2, countSlotDirs());
+ } finally {
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testGrowToMaxAllSfSendersCoexist() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 4, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ PooledSender c = pool.borrow();
+ PooledSender d = pool.borrow();
+ try {
+ Assert.assertEquals(4, pool.totalSize());
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue("slot default-" + i + " must exist",
+ Files.exists(slot("default-" + i)));
+ }
+ Assert.assertEquals(4, countSlotDirs());
+ // At max -- the 5th borrow must time out, not collide.
+ try {
+ pool.borrow();
+ Assert.fail("5th borrow must time out at max=4");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out"));
+ }
+ } finally {
+ d.close();
+ c.close();
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testConfiguredSenderIdUsedAsSlotBase() throws Exception {
+ // A sender_id in the config string becomes the base prefix; the pool
+ // appends - per slot. This is the knob that lets two pools (or
+ // two processes) share one sf_dir without colliding.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sender_id=myapp;";
+ try (SenderPool pool = new SenderPool(config, 2, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ Assert.assertTrue(Files.exists(slot("myapp-0")));
+ Assert.assertTrue(Files.exists(slot("myapp-1")));
+ Assert.assertFalse("no default-* slots when sender_id is set",
+ Files.exists(slot("default-0")));
+ Assert.assertEquals(2, countSlotDirs());
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // Slot lifecycle: reuse, reap-and-reuse, deterministic index recycling.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testReturnedSenderReusesSameSlot() throws Exception {
+ // Borrow, return, borrow again: the wrapper (and thus its slot) is
+ // recycled, NOT grown into a new slot dir.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender first = pool.borrow();
+ first.close();
+ PooledSender second = pool.borrow();
+ try {
+ Assert.assertSame("returned slot must be recycled", first, second);
+ Assert.assertEquals("no new slot dir on recycle", 1, countSlotDirs());
+ Assert.assertTrue(Files.exists(slot("default-0")));
+ } finally {
+ second.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testReapIdleFreesSlotAndIndexIsReused() throws Exception {
+ // Grow to max, return all, reap the over-min idle slots, then borrow
+ // again. The reaped slot indices must be returned to the free set and
+ // re-used -- no new index beyond the original high-water mark, and no
+ // "no free SF slot index" / "sf slot already in use" failure.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 3, 1_000, 80, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ PooledSender c = pool.borrow();
+ Assert.assertEquals(3, pool.totalSize());
+ a.close();
+ b.close();
+ c.close();
+
+ Thread.sleep(150);
+ pool.reapIdle();
+ Assert.assertEquals("reap shrinks to min", 1, pool.totalSize());
+
+ // High-water mark of slot dirs created so far is 3.
+ int dirsAfterReap = countSlotDirs();
+ Assert.assertTrue("slot dirs persist on disk after reap (>=1)", dirsAfterReap >= 1);
+
+ // Borrow back up to max. Must reuse the freed indices: no
+ // new slot dir beyond default-0..2 is ever created.
+ PooledSender x = pool.borrow();
+ PooledSender y = pool.borrow();
+ PooledSender z = pool.borrow();
+ try {
+ Assert.assertEquals(3, pool.totalSize());
+ Assert.assertEquals("indices recycled -- no 4th slot dir",
+ 3, countSlotDirs());
+ Assert.assertFalse("default-3 must never be created",
+ Files.exists(slot("default-3")));
+ } finally {
+ x.close();
+ y.close();
+ z.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRepeatedSaturationNeverExhaustsSlotIndices() throws Exception {
+ // Regression guard for the cap/slot-allocator invariant: hammer the
+ // pool through many full saturate/return/reap cycles. If freeing and
+ // the closingSlots accounting ever drifted, allocateSlotIndex() would
+ // throw "no free SF slot index" or a borrow would collide on a flock.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 3, 2_000, 1, Long.MAX_VALUE)) {
+ for (int cycle = 0; cycle < 20; cycle++) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ PooledSender c = pool.borrow();
+ Assert.assertEquals(3, pool.totalSize());
+ a.close();
+ b.close();
+ c.close();
+ pool.reapIdle();
+ }
+ // Never grew past max -- indices stayed within [0,3).
+ Assert.assertEquals(3, countSlotDirs());
+ Assert.assertFalse(Files.exists(slot("default-3")));
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // End-to-end ingest through pooled SF senders.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testEndToEndIngestThroughPooledSenders() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 3, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ try {
+ a.table("t1").longColumn("v", 1L).atNow();
+ a.flush();
+ b.table("t2").longColumn("v", 2L).atNow();
+ b.flush();
+ Assert.assertTrue("server must receive frames from both pooled senders",
+ awaitAtLeast(handler.frames, 2, 5_000));
+ } finally {
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // Cross-writer guard is preserved between independent pools / processes.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testSecondPoolSameSfDirSameBaseFailsFast() throws Exception {
+ // The slot flock is the multi-writer footgun guard. Two pools sharing
+ // one sf_dir with the same base would both try slot -0; the
+ // second must fail fast rather than interleave FSNs on disk. The pool
+ // fix must NOT weaken this contract.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool1 = new SenderPool(config, 1, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ Assert.assertTrue(Files.exists(slot("default-0")));
+ try {
+ SenderPool pool2 = new SenderPool(config, 1, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE);
+ pool2.close();
+ Assert.fail("second pool on same sf_dir+base must fail on the slot lock");
+ } catch (IllegalStateException e) {
+ Assert.assertTrue("message must name the slot-in-use contract, was: " + e.getMessage(),
+ e.getMessage().contains("sf slot already in use"));
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testTwoPoolsDistinctBaseShareSfDir() throws Exception {
+ // Distinct sender_id base per pool -> distinct slot dirs -> both pools
+ // coexist on one sf_dir and both can ingest.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String configA = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sender_id=appA;";
+ String configB = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sender_id=appB;";
+ try (SenderPool poolA = new SenderPool(configA, 1, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE);
+ SenderPool poolB = new SenderPool(configB, 1, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = poolA.borrow();
+ PooledSender b = poolB.borrow();
+ try {
+ Assert.assertTrue(Files.exists(slot("appA-0")));
+ Assert.assertTrue(Files.exists(slot("appB-0")));
+ a.table("t").longColumn("v", 1L).atNow();
+ a.flush();
+ b.table("t").longColumn("v", 2L).atNow();
+ b.flush();
+ Assert.assertTrue(awaitAtLeast(handler.frames, 2, 5_000));
+ } finally {
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testCloseReleasesAllSlots() throws Exception {
+ // After the pool closes, every slot flock must be released so the
+ // dirs can be re-acquired -- by a fresh pool or a standalone sender.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ SenderPool pool = new SenderPool(config, 2, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE);
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ // Leave them borrowed: close() must still release their flocks.
+ pool.close();
+
+ // A fresh pool over the same dirs must re-acquire slot 0 and 1.
+ try (SenderPool reopened = new SenderPool(config, 2, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ Assert.assertEquals(2, reopened.totalSize());
+ Assert.assertTrue(Files.exists(slot("default-0")));
+ Assert.assertTrue(Files.exists(slot("default-1")));
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock() throws Exception {
+ // Latent-fragility guard (M2): the pool returns a slot index to the
+ // free set ONLY after the delegate's close() has released the SF
+ // flock. If close() returns with the flock still held (it bailed out
+ // early with the I/O thread still running), the index must stay
+ // reserved forever -- otherwise the pool would hand the still-locked
+ // dir to the next borrow and resurrect "sf slot already in use".
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ Assert.assertTrue(Files.exists(slot("default-0")));
+
+ // Tear the delegate down for real first (so the test leaks
+ // no native resources), then forge the exact symptom:
+ // close() returned WITHOUT clearing slotLockReleased.
+ // close() is idempotent, so the discardBroken re-close
+ // below is a no-op and leaves the forged flag in place.
+ Sender delegate = getDelegate(a);
+ delegate.close();
+ setBooleanField(delegate, "slotLockReleased", false);
+
+ // Route the wrapper through the pool's broken-eviction path.
+ invokeDiscardBroken(pool, a);
+
+ // The leaked index must NOT be returned to the free set,
+ // and capacity must be accounted as permanently consumed.
+ Assert.assertEquals("one slot must be retired as leaked",
+ 1, getIntField(pool, "leakedSlots"));
+ boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse");
+ Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]);
+
+ // The next borrow must take a fresh index -- never reuse the
+ // still-locked default-0 dir.
+ PooledSender b = pool.borrow();
+ try {
+ Assert.assertTrue("new borrow must use a fresh slot dir",
+ Files.exists(slot("default-1")));
+ Assert.assertEquals(2, countSlotDirs());
+
+ // Capacity is permanently reduced by the leaked slot:
+ // max=2, one leaked + one live => the next borrow times
+ // out rather than colliding on the locked dir.
+ try {
+ pool.borrow();
+ Assert.fail("capacity must be reduced by the leaked slot");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out"));
+ }
+ } finally {
+ b.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testLeakedSlotIsObservable() throws Exception {
+ // M1 (observability): when a delegate's close() returns with the SF
+ // flock still held, the pool retires the slot forever and silently
+ // shrinks capacity. SenderPool has no logger today, so a pool that
+ // bleeds capacity this way degrades to "every borrow() times out"
+ // with nothing in the logs to explain why. Pin the contract: the
+ // leakedSlots++ path MUST emit a WARN (or louder) that names the
+ // retired slot. This test is RED until that log line is added.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Capture everything SenderPool logs (logback is the SLF4J
+ // binding on the test classpath).
+ Logger poolLogger = (Logger) LoggerFactory.getLogger(SenderPool.class);
+ ListAppender appender = new ListAppender<>();
+ appender.start();
+ Level savedLevel = poolLogger.getLevel();
+ poolLogger.setLevel(Level.ALL);
+ poolLogger.addAppender(appender);
+ try {
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+
+ // Forge the exact leak symptom: close() returned with
+ // the flock still held (I/O thread refused to stop).
+ // Tear the delegate down for real first so the test
+ // leaks no native resources; close() is idempotent so
+ // discardBroken's re-close leaves the forged flag set.
+ Sender delegate = getDelegate(a);
+ delegate.close();
+ setBooleanField(delegate, "slotLockReleased", false);
+ invokeDiscardBroken(pool, a);
+
+ // Sanity: the slot really was retired as leaked.
+ Assert.assertEquals("precondition: one slot must leak",
+ 1, getIntField(pool, "leakedSlots"));
+ // The leak must be observable via public API (metric).
+ Assert.assertEquals("leaked slot must be observable via leakedSlotCount()",
+ 1, pool.leakedSlotCount());
+
+ // Contract under test: the leak must be observable.
+ boolean warned = appender.list.stream().anyMatch(e ->
+ e.getLevel().isGreaterOrEqual(Level.WARN)
+ && e.getFormattedMessage().toLowerCase().contains("slot"));
+ Assert.assertTrue(
+ "leakedSlots++ must emit a WARN naming the retired slot, "
+ + "otherwise capacity loss is invisible; captured events="
+ + appender.list,
+ warned);
+ }
+ } finally {
+ poolLogger.detachAppender(appender);
+ poolLogger.setLevel(savedLevel);
+ appender.stop();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throws Exception {
+ // Coverage twin of testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock,
+ // but the close-and-reclaim is driven through reapIdle()'s leaked
+ // branch -- reclaimSlot(s, " during idle reaping") -- rather than
+ // discardBroken(). reapIdle is the only one of the three
+ // close-and-reclaim paths whose flockReleased()==false branch had no
+ // test: the existing reap tests use live QWP delegates (flock IS
+ // released => free path), and testReapIdleSurvivesDelegateCloseError
+ // is HTTP (storeAndForward off => reclaimSlot never runs). Pin it: an
+ // idle delegate whose close() leaves the flock held must retire the
+ // slot permanently (leakedSlots++, slotInUse stays set), never hand
+ // the still-locked dir to a later borrow.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ // minSize=0 so reapIdle is free to evict; idleTimeout=1ms so a
+ // returned slot is immediately reap-eligible.
+ try (SenderPool pool = new SenderPool(config, 0, 2, 500, 1, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ Assert.assertTrue(Files.exists(slot("default-0")));
+
+ // Return it to the idle set with a LIVE delegate, then
+ // forge the exact leak symptom: tear the delegate down for
+ // real (so no native resources leak) and clear
+ // slotLockReleased. close() is idempotent, so reapIdle's
+ // re-close is a no-op that leaves the flock "still held".
+ pool.giveBack(a);
+ Sender delegate = getDelegate(a);
+ delegate.close();
+ setBooleanField(delegate, "slotLockReleased", false);
+
+ // Drive the sweep: the idle timeout has elapsed.
+ Thread.sleep(10);
+ pool.reapIdle();
+
+ // The reap leaked branch must have fired.
+ Assert.assertEquals("reapIdle must retire the still-locked slot as leaked",
+ 1, getIntField(pool, "leakedSlots"));
+ boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse");
+ Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]);
+ Assert.assertEquals("leaked slot must be observable via leakedSlotCount()",
+ 1, pool.leakedSlotCount());
+
+ // The next borrow must take a fresh index -- never reuse
+ // the still-locked default-0 dir.
+ PooledSender b = pool.borrow();
+ try {
+ Assert.assertTrue("new borrow must use a fresh slot dir",
+ Files.exists(slot("default-1")));
+ Assert.assertEquals(2, countSlotDirs());
+
+ // Capacity is permanently reduced by the leaked slot:
+ // max=2, one leaked + one live => the next borrow times
+ // out rather than colliding on the locked dir.
+ try {
+ pool.borrow();
+ Assert.fail("capacity must be reduced by the leaked slot");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out"));
+ }
+ } finally {
+ b.close();
+ }
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // Recovery: stable slot ids let a re-created pool re-adopt unacked data.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testRecoveryReplayThroughPooledSlot() throws Exception {
+ // Phase 1: write rows to a slot against a silent server (no acks), so
+ // the data persists unacked on disk under default-0. Close.
+ // Phase 2: a new pool against an ack-ing server re-adopts default-0
+ // (stable index) and replays the unacked frames. Stable, deterministic
+ // slot ids are exactly what make this recovery possible.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1 -- silent server.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=500;";
+ try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = pool.borrow();
+ for (int i = 0; i < 3; i++) {
+ s.table("recover").longColumn("v", i).atNow();
+ s.flush();
+ }
+ s.close();
+ }
+ }
+ // Data must be on disk, unacked, under default-0.
+ Assert.assertTrue("unacked data must persist on disk", hasSegmentFile(slot("default-0")));
+
+ // Phase 2 -- ack-ing server, brand-new pool, same sf_dir.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(cfg2, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = pool.borrow();
+ try {
+ // Drain replays the recovered, unacked frames.
+ s.drain(5_000);
+ Assert.assertTrue("recovered frames must be replayed to the new server",
+ awaitAtLeast(handler.frames, 1, 5_000));
+ } finally {
+ s.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception {
+ // M1 regression: a startup-recovery delegate runs on the PoolHousekeeper
+ // thread, so its build() must NOT inherit the user's SYNC initial-connect
+ // mode (auto-enabled by any reconnect_* knob). SYNC would retry the
+ // connect for the whole reconnect budget inside build() -- far past
+ // PoolHousekeeper.STOP_TIMEOUT_MILLIS -- so a close() landing during that
+ // build would make the housekeeper join time out and leave the recoverer
+ // holding the slot flock after close() returned. The recovery factory
+ // forces initial_connect_mode=OFF (at most one connect attempt); the
+ // normal factory must still honour the user's promoted SYNC mode.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ // reconnect_max_duration_millis set, initial_connect_mode unset
+ // -> the builder promotes to SYNC for ordinary senders.
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir
+ + ";reconnect_max_duration_millis=30000;";
+ // min=0 (no prewarm connect), no stranded data (recovery no-op).
+ try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ // Normal managed-slot delegate: inherits the promoted SYNC.
+ Sender normal = invokeBuildSlotDelegate(pool, "defaultSender", 0);
+ try {
+ Assert.assertEquals(
+ "ordinary pooled sender must honour the user's promoted SYNC mode",
+ Sender.InitialConnectMode.SYNC, readInitialConnectMode(normal));
+ } finally {
+ normal.close();
+ }
+ // Recovery delegate on a different slot: forced OFF.
+ Sender recoverer = invokeBuildSlotDelegate(pool, "defaultRecoverySender", 1);
+ try {
+ Assert.assertEquals(
+ "recovery delegate must force OFF so build() makes at most one connect attempt",
+ Sender.InitialConnectMode.OFF, readInitialConnectMode(recoverer));
+ } finally {
+ recoverer.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception {
+ // C1 regression: the startup recovery loop MUST mirror discardBroken /
+ // reapIdle. When a recoverer's delegate close() returns with the SF
+ // flock still held (the I/O thread refused to stop), the recovered slot
+ // index must be retired permanently (leakedSlots++, slotInUse stays
+ // set) -- NOT freed. Freeing it would let a later borrow re-pick the
+ // still-locked dir and resurrect "sf slot already in use", the exact
+ // failure class this PR exists to kill. Pre-fix the recovery finally
+ // set slotInUse[i]=false unconditionally; this test is RED until it
+ // consults flockReleased() like the other two close-and-reclaim paths.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: leave unacked data on disk under default-0 so startup
+ // recovery treats it as a candidate orphan and builds a recoverer.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=500;";
+ try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = pool.borrow();
+ for (int i = 0; i < 3; i++) {
+ s.table("recover").longColumn("v", i).atNow();
+ s.flush();
+ }
+ s.close();
+ }
+ }
+ Assert.assertTrue("unacked data must persist under default-0",
+ hasSegmentFile(slot("default-0")));
+
+ // Phase 2: ack-ing server + a new pool whose injected factory forges
+ // the exact leak symptom for the recovery build of slot 0. The
+ // factory returns a real, flock-holding QwpWebSocketSender but
+ // pre-sets closed=true, so the recovery close() is a complete no-op
+ // (checkNotClosed short-circuits drain too): the flock stays held
+ // and slotLockReleased never flips -- precisely a refused I/O-thread
+ // stop. flockReleased(recoverer) must therefore report false.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";";
+
+ AtomicReference forged = new AtomicReference<>();
+ IntFunction factory = idx -> {
+ Sender real = Sender.builder(cfg2).senderId("default-" + idx).build();
+ if (idx == 0) {
+ try {
+ setBooleanField(real, "closed", true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ forged.set(real);
+ }
+ return real;
+ };
+
+ // minSize=0 so prewarm never adopts slot 0 -- recovery is the
+ // only builder of slot 0. maxSize=2 so a later borrow can still
+ // get a fresh slot (default-1), proving capacity dropped by one.
+ SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory);
+ try {
+ // The forge must actually have reached recovery's build.
+ Assert.assertNotNull("recovery must have built slot 0", forged.get());
+ // The retire branch must have fired during construction.
+ Assert.assertEquals("recovery must retire the still-locked slot as leaked",
+ 1, getIntField(pool, "leakedSlots"));
+ boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse");
+ Assert.assertTrue("retired slot 0 must stay reserved", slotInUse[0]);
+ Assert.assertFalse("slot 1 must remain free", slotInUse[1]);
+
+ // A later borrow must take the fresh slot 1, never re-pick
+ // the still-locked default-0 (which would throw "sf slot
+ // already in use").
+ PooledSender b = pool.borrow();
+ try {
+ Assert.assertTrue("borrow must use a fresh slot dir",
+ Files.exists(slot("default-1")));
+ // Capacity is permanently reduced by the leaked slot:
+ // max=2, one leaked + one live => the next borrow times
+ // out rather than colliding on the locked dir.
+ try {
+ pool.borrow();
+ Assert.fail("capacity must be reduced by the leaked slot");
+ } catch (LineSenderException e) {
+ Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out"));
+ }
+ } finally {
+ b.close();
+ }
+ } finally {
+ pool.close();
+ // Release the forged recoverer's real flock + native
+ // resources: pool.close() never saw it (recovery never
+ // added the recoverer to `all`), so un-forge closed and
+ // close it for real, otherwise assertMemoryLeak trips.
+ Sender leaked = forged.get();
+ if (leaked != null) {
+ setBooleanField(leaked, "closed", false);
+ leaked.close();
+ }
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // Concurrency stress: borrow/return churn must never collide on a slot.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testConcurrentBorrowReturnStress() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ final int threads = 6;
+ final int iterations = 25;
+ try (SenderPool pool = new SenderPool(config, 1, 4, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ final CountDownLatch start = new CountDownLatch(1);
+ final CountDownLatch done = new CountDownLatch(threads);
+ final AtomicReference failure = new AtomicReference<>();
+ for (int t = 0; t < threads; t++) {
+ final int id = t;
+ Thread worker = new Thread(() -> {
+ try {
+ start.await();
+ for (int i = 0; i < iterations; i++) {
+ PooledSender s = pool.borrow();
+ try {
+ s.table("stress").longColumn("thread", id)
+ .longColumn("i", i).atNow();
+ s.flush();
+ } finally {
+ s.close();
+ }
+ }
+ } catch (Throwable e) {
+ failure.compareAndSet(null, e);
+ } finally {
+ done.countDown();
+ }
+ });
+ worker.start();
+ }
+ start.countDown();
+ Assert.assertTrue("workers must finish", done.await(60, TimeUnit.SECONDS));
+ if (failure.get() != null) {
+ throw new AssertionError("concurrent borrow/return failed", failure.get());
+ }
+ // Invariants after the storm.
+ Assert.assertTrue("totalSize within max", pool.totalSize() <= 4);
+ Assert.assertTrue("available <= total",
+ pool.availableSize() <= pool.totalSize());
+ Assert.assertTrue("no slot dir beyond max created", countSlotDirs() <= 4);
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testConcurrentFirstBorrowsWithMinZeroRaceOnSfDir() throws Exception {
+ // C2 regression: senderPoolMin(0) means no single-threaded pre-warm,
+ // so the shared parent sf_dir is NOT created at construction (the
+ // constructor probe only parses the config). The first concurrent
+ // borrows then race into build() -> Files.mkdir(sfDir) outside the
+ // pool lock. Pre-fix, the mkdir loser got a non-zero rc (EEXIST) and
+ // its borrow() threw "could not create sf_dir" on a perfectly healthy
+ // pool. Post-fix, a benign creation race is treated as success.
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ // minSize=0 -> no pre-warm -> sf_dir absent until first borrow.
+ try (SenderPool pool = new SenderPool(config, 0, 4, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ Assert.assertFalse("sf_dir must not exist before the first borrow",
+ Files.exists(sfDir));
+
+ final int threads = 4;
+ final CyclicBarrier barrier = new CyclicBarrier(threads);
+ final CountDownLatch done = new CountDownLatch(threads);
+ final AtomicReference failure = new AtomicReference<>();
+ final PooledSender[] borrowed = new PooledSender[threads];
+ for (int t = 0; t < threads; t++) {
+ final int id = t;
+ Thread worker = new Thread(() -> {
+ try {
+ // Align all first borrows so they race into the
+ // shared parent mkdir simultaneously.
+ barrier.await();
+ borrowed[id] = pool.borrow();
+ } catch (Throwable e) {
+ failure.compareAndSet(null, e);
+ } finally {
+ done.countDown();
+ }
+ });
+ worker.start();
+ }
+ Assert.assertTrue("workers must finish", done.await(30, TimeUnit.SECONDS));
+ try {
+ if (failure.get() != null) {
+ throw new AssertionError(
+ "concurrent first borrows must not race on sf_dir", failure.get());
+ }
+ Assert.assertEquals(threads, pool.totalSize());
+ Assert.assertEquals("one slot dir per borrow", threads, countSlotDirs());
+ } finally {
+ for (PooledSender s : borrowed) {
+ if (s != null) {
+ s.close();
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testFirstBorrowToleratesPreExistingSfDir() throws Exception {
+ // Deterministic complement to testConcurrentFirstBorrowsWithMinZeroRaceOnSfDir.
+ // That test only RAISES the probability of two threads racing into
+ // Files.mkdir(sfDir); on a run where one thread wins the mkdir cleanly
+ // before the others reach the exists() check, the benign-race branch
+ // in Sender.build() is never hit and the test would pass even on
+ // pre-fix code. This test removes the timing dependency: it pre-creates
+ // sfDir so EVERY first borrow finds the parent already present and must
+ // build successfully without throwing "could not create sf_dir".
+ TestUtils.assertMemoryLeak(() -> {
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
+ int port = server.getPort();
+ server.start();
+ Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
+
+ // Pre-create the shared parent: now build()'s mkdir guard is
+ // skipped on every borrow, deterministically asserting that a
+ // pre-existing sf_dir is tolerated rather than fatal.
+ Assert.assertEquals("pre-create sf_dir must succeed",
+ 0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT));
+ Assert.assertTrue(Files.exists(sfDir));
+
+ String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(config, 0, 4, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ try {
+ Assert.assertEquals(2, pool.totalSize());
+ Assert.assertTrue(Files.exists(slot("default-0")));
+ Assert.assertTrue(Files.exists(slot("default-1")));
+ } finally {
+ a.close();
+ b.close();
+ }
+ }
+ }
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // drain_orphans=on + pool: the pool must NOT treat its own sibling slots
+ // as drainable orphans, but MUST still drain genuine foreign leftovers.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testDrainOrphansPoolDoesNotCannibalizeSiblingSlots() throws Exception {
+ // Regression guard. The pool gives each SF sender a sibling slot
+ // -. With drain_orphans=on, every pooled build runs an
+ // orphan scan -- and before the namespace-exclusion fix that scan
+ // listed the pool's OWN siblings (default-1 holds unacked .sfa) as
+ // orphans and dispatched a background drainer at them. That drainer
+ // could win a sibling's flock and re-surface the exact
+ // "sf slot already in use" collision the per-slot ids were added to
+ // prevent. After the fix the pool fences off its whole "-"
+ // namespace, so building a drain_orphans pool over pre-existing
+ // sibling data is clean and the data is recovered by the pool itself.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0 AND default-1 via a
+ // plain (no drain_orphans) pool against a silent server.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=500;";
+ try (SenderPool seed = new SenderPool(seedCfg, 2, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = seed.borrow();
+ PooledSender b = seed.borrow();
+ a.table("recover").longColumn("v", 1L).atNow();
+ a.flush();
+ b.table("recover").longColumn("v", 2L).atNow();
+ b.flush();
+ b.close();
+ a.close();
+ }
+ }
+ Assert.assertTrue("default-0 must hold unacked data", hasSegmentFile(slot("default-0")));
+ Assert.assertTrue("default-1 must hold unacked data", hasSegmentFile(slot("default-1")));
+
+ // Phase 2: a drain_orphans=on pool over the same sf_dir. Pre-fix
+ // this construction could throw "sf slot already in use"; post-fix
+ // it is deterministically clean -- no drainer targets a sibling.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir
+ + ";drain_orphans=on;";
+ try (SenderPool pool = new SenderPool(cfg, 2, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ Assert.assertEquals(2, pool.totalSize());
+ Assert.assertEquals("no extra slot dirs spawned by a rogue drainer",
+ 2, countSlotDirs());
+ // A drainer must NOT have given up on a sibling slot.
+ Assert.assertFalse("sibling must not be flagged .failed",
+ Files.exists(slot("default-0") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ Assert.assertFalse("sibling must not be flagged .failed",
+ Files.exists(slot("default-1") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+
+ // The pool owns and recovers both slots: borrowing + draining
+ // replays the recovered frames through the legitimate senders.
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ try {
+ a.drain(5_000);
+ b.drain(5_000);
+ Assert.assertTrue("both slots' recovered data must replay",
+ awaitAtLeast(handler.frames, 2, 5_000));
+ } finally {
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDrainOrphansPoolStillDrainsForeignOrphan() throws Exception {
+ // The fix excludes only the pool's OWN "-" namespace -- a
+ // genuine foreign leftover (a different sender_id base) must still be
+ // adopted and drained, otherwise we would have silently disabled the
+ // drain_orphans feature for pooled deployments.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: a standalone sender with a DIFFERENT base leaves unacked
+ // data under /legacy.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ // close_flush_timeout_millis=0 => close() does not drain, so
+ // the flushed-but-unacked frames stay on disk (silent server
+ // never acks) and close() never throws a drain timeout.
+ String ghostCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";sender_id=legacy;close_flush_timeout_millis=0;";
+ try (Sender ghost = Sender.fromConfig(ghostCfg)) {
+ for (int i = 0; i < 3; i++) {
+ ghost.table("foreign").longColumn("v", i).atNow();
+ ghost.flush();
+ }
+ } catch (Exception ignored) {
+ // best-effort: we only need the unacked .sfa on disk
+ }
+ }
+ Assert.assertTrue("foreign leftover must hold unacked data",
+ hasSegmentFile(slot("legacy")));
+
+ // Phase 2: a drain_orphans=on pool with the default base. Its
+ // pooled senders are default-*, so "legacy" is NOT in the excluded
+ // namespace -- the background drainer must adopt it, replay its
+ // frames to the ack server, and clear the slot.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir
+ + ";drain_orphans=on;";
+ try (SenderPool pool = new SenderPool(cfg, 1, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = pool.borrow();
+ try {
+ Assert.assertTrue("foreign orphan frames must be replayed by a drainer",
+ awaitAtLeast(handler.frames, 1, 10_000));
+ Assert.assertTrue("foreign orphan slot must be drained (no unacked .sfa left)",
+ awaitNoSegmentFile(slot("legacy"), 10_000));
+ } finally {
+ s.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testShrinkingMaxSizeDrainsStrandedOutOfRangeSlots() throws Exception {
+ // The bug: a deployment that previously ran at maxSize=4 leaves unacked
+ // data in default-0..3. Restarting at maxSize=2 means default-2 and
+ // default-3 are out of the new [0,2) index range forever -- the pool
+ // never re-creates them. Before the fix the pool also fenced off the
+ // WHOLE "default-" prefix from draining, so default-2/3 were neither
+ // re-created nor drained: their store-and-forward data was silently
+ // stranded even with drain_orphans=on. After the fix the exclusion is
+ // bounded to [0,maxSize), so the out-of-range slots are adopted by a
+ // background drainer and recovered.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0..3 via a maxSize=4 pool
+ // against a silent (never-acks) server. close_flush_timeout_millis=0
+ // so close() leaves the flushed-but-unacked .sfa on disk.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 4, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender[] s = new PooledSender[4];
+ for (int i = 0; i < 4; i++) {
+ s[i] = seed.borrow();
+ }
+ for (int i = 0; i < 4; i++) {
+ s[i].table("recover").longColumn("v", i).atNow();
+ s[i].flush();
+ }
+ for (int i = 3; i >= 0; i--) {
+ s[i].close();
+ }
+ }
+ }
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue("default-" + i + " must hold unacked data",
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Phase 2: restart at maxSize=2 with drain_orphans=on against an ack
+ // server. The pool re-creates + self-recovers default-0/1; the
+ // out-of-range default-2/3 must be drained, not stranded.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir
+ + ";drain_orphans=on;";
+ try (SenderPool pool = new SenderPool(cfg, 1, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ try {
+ // The regression: the out-of-range slots must be adopted
+ // by a background drainer and emptied.
+ Assert.assertTrue("default-2 unacked data must be recovered, not stranded",
+ awaitNoSegmentFile(slot("default-2"), 15_000));
+ Assert.assertTrue("default-3 unacked data must be recovered, not stranded",
+ awaitNoSegmentFile(slot("default-3"), 15_000));
+ Assert.assertFalse("out-of-range slot must not be abandoned as .failed",
+ Files.exists(slot("default-2") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ Assert.assertFalse("out-of-range slot must not be abandoned as .failed",
+ Files.exists(slot("default-3") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ } finally {
+ b.close();
+ a.close();
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDefaultConfigRecoversOutOfRangeSlotsAfterShrink() throws Exception {
+ // Regression for review claim M1: with the out-of-the-box config
+ // (drain_orphans defaults to OFF), shrinking maxSize across a restart
+ // must STILL deliver the unacked data left in the now-out-of-range
+ // slots. recoverOneSlotStep() pass 2 adopts -i for
+ // i >= maxSize at construction, independent of drain_orphans.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0..3 via a maxSize=4 pool
+ // against a silent (never-acks) server.
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 4, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender[] s = new PooledSender[4];
+ for (int i = 0; i < 4; i++) {
+ s[i] = seed.borrow();
+ }
+ for (int i = 0; i < 4; i++) {
+ s[i].table("recover").longColumn("v", i).atNow();
+ s[i].flush();
+ }
+ for (int i = 3; i >= 0; i--) {
+ s[i].close();
+ }
+ }
+ }
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue("default-" + i + " must hold unacked data",
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Phase 2: restart at maxSize=2 with the DEFAULT config (NO
+ // drain_orphans) against a healthy ack server. minSize=0 so startup
+ // recovery -- not prewarm "normal use" -- is the recovery path for
+ // the in-range slots, and pass 2 is the only path for default-2/3.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(cfg, 0, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ // In-range [0,2): startup recovery pass 1 drains these.
+ Assert.assertTrue("in-range default-0 must be recovered at startup",
+ awaitNoSegmentFile(slot("default-0"), 15_000));
+ Assert.assertTrue("in-range default-1 must be recovered at startup",
+ awaitNoSegmentFile(slot("default-1"), 15_000));
+
+ // Out-of-range [2,4): pass 2 must deliver these too, under
+ // the default config (no drain_orphans).
+ Assert.assertTrue("default-2 unacked data must be recovered, not stranded",
+ awaitNoSegmentFile(slot("default-2"), 15_000));
+ Assert.assertTrue("default-3 unacked data must be recovered, not stranded",
+ awaitNoSegmentFile(slot("default-3"), 15_000));
+ Assert.assertFalse("out-of-range slot must not be abandoned as .failed",
+ Files.exists(slot("default-2") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ Assert.assertFalse("out-of-range slot must not be abandoned as .failed",
+ Files.exists(slot("default-3") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+
+ // Sanity: the pool is still usable for normal borrows.
+ PooledSender a = pool.borrow();
+ a.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception {
+ // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer
+ // never adopts a slot dir the pool intends to (re)create -- that is what
+ // prevents "sf slot already in use" (see
+ // testDrainOrphansPoolDoesNotCannibalizeSiblingSlots). The trade-off was
+ // that an in-range slot left holding unacked data by a previous run was
+ // recovered ONLY when the pool happened to (re)create that index: the
+ // pool pre-warms [0, minSize) and builds [minSize, maxSize) lazily on
+ // demand, so under steady low load a high in-range index was never
+ // rebuilt -- neither drained (excluded) nor recovered -- and its data
+ // was stranded on disk until a restart or load spike.
+ //
+ // The fix has the pool recover its own stranded managed slots once, at
+ // construction, under its own slot reservation (so the cannibalization
+ // race the exclusion guards against still cannot happen). This test
+ // seeds a busy run, then restarts under steady low load and asserts the
+ // idle in-range slots are recovered anyway.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: a busy run at maxSize=4 seeds unacked data into
+ // default-0..3 (silent server never acks; close_flush_timeout=0 so
+ // close() leaves the flushed-but-unacked .sfa on disk).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 4, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender[] s = new PooledSender[4];
+ for (int i = 0; i < 4; i++) {
+ s[i] = seed.borrow();
+ }
+ for (int i = 0; i < 4; i++) {
+ s[i].table("recover").longColumn("v", i).atNow();
+ s[i].flush();
+ }
+ for (int i = 3; i >= 0; i--) {
+ s[i].close();
+ }
+ }
+ }
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue("default-" + i + " must hold unacked data",
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Phase 2: restart at the SAME maxSize=4 (so default-0..3 stay in
+ // range) with steady low load. minSize=0 means prewarm builds
+ // nothing and the lowest-free allocator would never reach the high
+ // indices under a single in-flight borrow -- yet startup recovery
+ // must still empty every in-range slot.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";";
+ try (SenderPool pool = new SenderPool(cfg, 0, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ // All four in-range slots must be recovered by the startup
+ // pass, even though steady low load never grows the pool to
+ // their indices.
+ for (int i = 0; i < 4; i++) {
+ Assert.assertTrue("in-range idle default-" + i
+ + " must be recovered at startup, not stranded",
+ awaitNoSegmentFile(slot("default-" + i), 15_000));
+ Assert.assertFalse("recovered slot must not be flagged .failed",
+ Files.exists(slot("default-" + i) + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ }
+ // The recovered frames must have actually reached the server.
+ Assert.assertTrue("recovered frames must be replayed to the server",
+ awaitAtLeast(handler.frames, 4, 15_000));
+
+ // Sanity: the pool is still usable for normal borrows.
+ PooledSender a = pool.borrow();
+ a.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception {
+ // Regression for the startup-recovery budget (M1).
+ // recoverOneSlotStep() runs synchronously in the SenderPool
+ // constructor. A previous run can strand unacked data in EVERY in-range
+ // slot, and if the server is reachable but does not ack, each slot's
+ // drain blocks for the full acquireTimeoutMillis. Without a shared,
+ // whole-scan budget (and a short-circuit on the first drain that fails
+ // to ack) construction blocks for (maxSize - minSize) *
+ // acquireTimeoutMillis -- here 4 * 1s = 4s -- so QuestDB.build() stalls
+ // proportionally to the recovery backlog. The fix caps the TOTAL
+ // recovery at ~one acquireTimeoutMillis and stops scanning the moment a
+ // drain fails to ack, so construction must return well inside that
+ // ceiling no matter how many slots are stranded.
+ final long acquireTimeoutMillis = 1_000L;
+ final int maxSize = 4;
+
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0..3 against a silent
+ // server (never acks; close_flush_timeout=0 leaves the
+ // flushed-but-unacked .sfa on disk).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, maxSize, maxSize, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender[] s = new PooledSender[maxSize];
+ for (int i = 0; i < maxSize; i++) {
+ s[i] = seed.borrow();
+ }
+ for (int i = 0; i < maxSize; i++) {
+ s[i].table("recover").longColumn("v", i).atNow();
+ s[i].flush();
+ }
+ for (int i = maxSize - 1; i >= 0; i--) {
+ s[i].close();
+ }
+ }
+ }
+ for (int i = 0; i < maxSize; i++) {
+ Assert.assertTrue("default-" + i + " must hold unacked data",
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Phase 2: restart against a STILL-silent (reachable but
+ // never-acking) server. Construction triggers startup recovery over
+ // all four stranded slots. close_flush_timeout=0 makes each
+ // recoverer's close() a fast close, so the measured window isolates
+ // the drain budget: pre-fix it is maxSize * acquireTimeoutMillis;
+ // post-fix it is bounded by ~one acquireTimeoutMillis.
+ try (TestWebSocketServer silent2 = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent2.getPort();
+ silent2.start();
+ Assert.assertTrue(silent2.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+
+ long startNanos = System.nanoTime();
+ SenderPool pool = new SenderPool(cfg, 0, maxSize, acquireTimeoutMillis, Long.MAX_VALUE, Long.MAX_VALUE);
+ long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ try {
+ // Headline guarantee: total recovery is bounded by the
+ // shared budget, NOT maxSize * acquireTimeoutMillis. The 3x
+ // ceiling leaves generous CI margin over the ~1x post-fix
+ // cost while still decisively failing the pre-fix 4x stall.
+ Assert.assertTrue(
+ "startup recovery must be bounded by a shared budget, not per-slot: took "
+ + elapsedMillis + "ms with acquireTimeout=" + acquireTimeoutMillis
+ + "ms over " + maxSize + " stranded slots (pre-fix ~"
+ + (maxSize * acquireTimeoutMillis) + "ms)",
+ elapsedMillis < 3 * acquireTimeoutMillis);
+
+ // Durability, not loss: the silent server never acked, so
+ // the stranded data is deferred (still on disk for a later
+ // attempt), never dropped.
+ for (int i = 0; i < maxSize; i++) {
+ Assert.assertTrue(
+ "stranded data must be preserved on disk, not lost: default-" + i,
+ hasSegmentFile(slot("default-" + i)));
+ }
+ } finally {
+ pool.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testRecoveryStepStaysBoundedWithDrainOrphansAgainstNonAckingServer() throws Exception {
+ // M-A regression: a startup-recovery delegate must NOT inherit
+ // drain_orphans=on. If it did, building the recoverer (which connects
+ // OK against a reachable server because initial_connect_mode is forced
+ // OFF) would run an orphan scan and dispatch a BackgroundDrainerPool at
+ // any foreign/out-of-range orphan. Against a reachable-but-not-acking
+ // server those background drainers never reach their target, so the
+ // recoverer's close() -- called inside the recovery step, on the
+ // housekeeper thread, BEFORE cursorEngine.close() releases the slot
+ // flock -- blocks in BackgroundDrainerPool.close() for ~3s
+ // (GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS). That makes one step
+ // ~1s drain + ~3s drainerPool.close ~= 4s, far past
+ // RECOVERY_DRAIN_BUDGET_MILLIS and PoolHousekeeper.STOP_TIMEOUT_MILLIS
+ // (2s) -- and a close() landing mid-step would return with the flock
+ // still held. After the fix recovery delegates force drain_orphans=off,
+ // so no drainer pool is created and the step stays bounded by the drain
+ // budget alone.
+ final long acquireTimeoutMillis = 1_000L;
+ final int maxSize = 2;
+
+ TestUtils.assertMemoryLeak(() -> {
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String addr = "localhost:" + silentPort;
+
+ // Phase 1a: strand unacked data in an in-range managed slot
+ // (default-0) so the recovery step has a slot to process.
+ String seedCfg = "ws::addr=" + addr + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 1, maxSize, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = seed.borrow();
+ s.table("recover").longColumn("v", 1L).atNow();
+ s.flush();
+ s.close();
+ }
+ Assert.assertTrue("default-0 must hold unacked data", hasSegmentFile(slot("default-0")));
+
+ // Phase 1b: strand a FOREIGN orphan (different base) so a
+ // recovery delegate that inherited drain_orphans=on would
+ // dispatch a background drainer at it.
+ String ghostCfg = "ws::addr=" + addr + ";sf_dir=" + sfDir
+ + ";sender_id=legacy;close_flush_timeout_millis=0;";
+ try (Sender ghost = Sender.fromConfig(ghostCfg)) {
+ for (int i = 0; i < 3; i++) {
+ ghost.table("foreign").longColumn("v", i).atNow();
+ ghost.flush();
+ }
+ } catch (Exception ignored) {
+ // best-effort: we only need the unacked .sfa on disk
+ }
+ Assert.assertTrue("foreign leftover must hold unacked data", hasSegmentFile(slot("legacy")));
+
+ // Phase 2: a deferred drain_orphans=on pool against the SAME
+ // reachable-but-not-acking server. Drive ONE recovery step and
+ // time it: the step builds a recovery delegate on default-0,
+ // which pre-fix would dispatch a drainer at the foreign orphan
+ // and then block ~3s in drainerPool.close().
+ String cfg = "ws::addr=" + addr + ";sf_dir=" + sfDir
+ + ";drain_orphans=on;close_flush_timeout_millis=0;";
+ SenderPool pool = newDeferredPool(cfg, 0, maxSize, acquireTimeoutMillis);
+ try {
+ long startNanos = System.nanoTime();
+ invokeRunStartupRecoveryStep(pool);
+ long stepMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+
+ // Headline guarantee: a recovery delegate must not stand up a
+ // BackgroundDrainerPool, so the step is bounded by the drain
+ // budget (~1s) -- comfortably under STOP_TIMEOUT_MILLIS. The
+ // 2.5s ceiling clears the ~1s post-fix cost with CI margin
+ // while decisively failing the pre-fix ~4s overrun.
+ Assert.assertTrue(
+ "recovery step must stay bounded with drain_orphans=on against a "
+ + "non-acking server (no BackgroundDrainerPool overrun): took "
+ + stepMillis + "ms",
+ stepMillis < 2_500L);
+
+ // Durability, not loss: the foreign orphan stays on disk for
+ // a later live-sender drainer; the recovery delegate must
+ // not have abandoned it as .failed either.
+ Assert.assertTrue("foreign orphan data must be preserved on disk, not lost",
+ hasSegmentFile(slot("legacy")));
+ Assert.assertFalse("foreign orphan must not be flagged .failed by a recovery delegate",
+ Files.exists(slot("legacy") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+ } finally {
+ pool.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Exception {
+ // M2 fix: when recovery is deferred (the pooled QuestDB handle's path),
+ // constructing the SenderPool must NOT run startup recovery inline, so
+ // build() never blocks on a reachable-but-not-acking server -- not even
+ // when every in-range slot holds stranded data. Recovery is driven later,
+ // off the build() thread (by the housekeeper in production; explicitly
+ // here).
+ final long acquireTimeoutMillis = 1_000L;
+ final int maxSize = 4;
+
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0..3 against a silent
+ // server (never acks; close_flush_timeout=0 leaves the .sfa on disk).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, maxSize, maxSize, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender[] s = new PooledSender[maxSize];
+ for (int i = 0; i < maxSize; i++) {
+ s[i] = seed.borrow();
+ }
+ for (int i = 0; i < maxSize; i++) {
+ s[i].table("recover").longColumn("v", i).atNow();
+ s[i].flush();
+ }
+ for (int i = maxSize - 1; i >= 0; i--) {
+ s[i].close();
+ }
+ }
+ }
+ for (int i = 0; i < maxSize; i++) {
+ Assert.assertTrue("default-" + i + " must hold unacked data",
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Phase 2: construct a DEFERRED pool against a STILL-silent (reachable
+ // but never-acking) server. Pre-fix, inline recovery would block the
+ // constructor for ~one acquireTimeoutMillis here; deferred, it returns
+ // effectively immediately because recovery has not run yet.
+ try (TestWebSocketServer silent2 = new TestWebSocketServer(new SilentHandler())) {
+ int port = silent2.getPort();
+ silent2.start();
+ Assert.assertTrue(silent2.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+
+ long startNanos = System.nanoTime();
+ SenderPool pool = newDeferredPool(cfg, 0, maxSize, acquireTimeoutMillis);
+ long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ try {
+ // Headline: deferred construction does not pay the recovery
+ // drain budget. A whole acquireTimeout of margin is plenty --
+ // pre-fix this would have been ~acquireTimeoutMillis.
+ Assert.assertTrue(
+ "deferred construction must not block on recovery: took " + elapsedMillis
+ + "ms with acquireTimeout=" + acquireTimeoutMillis + "ms",
+ elapsedMillis < acquireTimeoutMillis);
+
+ // Recovery has not been driven yet, so every stranded slot is
+ // still on disk -- proving construction skipped inline recovery.
+ for (int i = 0; i < maxSize; i++) {
+ Assert.assertTrue(
+ "deferred construction must NOT recover inline: default-" + i,
+ hasSegmentFile(slot("default-" + i)));
+ }
+
+ // Driving recovery against the still-silent server is bounded
+ // by the shared budget and preserves the data (durable, not
+ // lost) for a later attempt -- exercising the deferred path's
+ // concurrency-safe slot reservation too.
+ long recoverStart = System.nanoTime();
+ invokeRunStartupRecoveryOnce(pool);
+ long recoverMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStart);
+ Assert.assertTrue(
+ "driven recovery must be bounded by the shared budget: took " + recoverMillis
+ + "ms (acquireTimeout=" + acquireTimeoutMillis + "ms)",
+ recoverMillis < 3 * acquireTimeoutMillis);
+ for (int i = 0; i < maxSize; i++) {
+ Assert.assertTrue(
+ "stranded data must be preserved on disk, not lost: default-" + i,
+ hasSegmentFile(slot("default-" + i)));
+ }
+ } finally {
+ pool.close();
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception {
+ // Deferring recovery off the constructor must not lose it: once driven
+ // (by the housekeeper in production; explicitly here) against an acking
+ // server, a deferred pool recovers its stranded managed slots exactly as
+ // the inline path would. The drive is also idempotent.
+ TestUtils.assertMemoryLeak(() -> {
+ // Phase 1: seed unacked data into default-0 (silent server).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender s = seed.borrow();
+ s.table("recover").longColumn("v", 1).atNow();
+ s.flush();
+ s.close();
+ }
+ }
+ Assert.assertTrue("default-0 must hold unacked data", hasSegmentFile(slot("default-0")));
+
+ // Phase 2: deferred pool against an ACKING server.
+ CountingAckHandler handler = new CountingAckHandler();
+ try (TestWebSocketServer ack = new TestWebSocketServer(handler)) {
+ int ackPort = ack.getPort();
+ ack.start();
+ Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS));
+ String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";";
+ SenderPool pool = newDeferredPool(cfg, 0, 2, 5_000);
+ try {
+ // Deferred: not recovered until driven.
+ Assert.assertTrue("deferred construction must NOT recover inline: default-0",
+ hasSegmentFile(slot("default-0")));
+
+ // Drive it (what the housekeeper does on its first tick).
+ invokeRunStartupRecoveryOnce(pool);
+ Assert.assertTrue("driven recovery must empty default-0",
+ awaitNoSegmentFile(slot("default-0"), 15_000));
+ Assert.assertTrue("recovered frames must reach the server",
+ awaitAtLeast(handler.frames, 1, 15_000));
+ Assert.assertFalse("recovered slot must not be flagged .failed",
+ Files.exists(slot("default-0") + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
+
+ // Idempotent: a second drive is a no-op and must not throw.
+ invokeRunStartupRecoveryOnce(pool);
+ Assert.assertFalse("default-0 stays recovered", hasSegmentFile(slot("default-0")));
+
+ // Pool still usable for normal borrows.
+ PooledSender a = pool.borrow();
+ a.close();
+ } finally {
+ pool.close();
+ }
+ }
+ });
+ }
+
+ @Test(timeout = 60_000)
+ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws Exception {
+ // C1 regression. The housekeeper drives startup recovery one slot per
+ // step on its own thread. QuestDBImpl.close() raises the pool's shutdown
+ // signal (markClosing) BEFORE stopping the housekeeper, and every step
+ // re-checks it, so a close() landing while a recoverer is mid-drain must
+ // stop recovery from building the NEXT slot -- no more "keeps building
+ // senders on a logically-closed pool". A fake recoverer parks slot-0's
+ // drain so we can raise the signal deterministically inside the window.
+ TestUtils.assertMemoryLeak(() -> {
+ // Seed REAL unacked data into default-0 and default-1 (two candidates).
+ try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) {
+ int silentPort = silent.getPort();
+ silent.start();
+ Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS));
+ String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir
+ + ";close_flush_timeout_millis=0;";
+ try (SenderPool seed = new SenderPool(seedCfg, 2, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) {
+ PooledSender a = seed.borrow();
+ PooledSender b = seed.borrow();
+ a.table("recover").longColumn("v", 0).atNow();
+ a.flush();
+ b.table("recover").longColumn("v", 1).atNow();
+ b.flush();
+ b.close();
+ a.close();
+ }
+ }
+ Assert.assertTrue(hasSegmentFile(slot("default-0")));
+ Assert.assertTrue(hasSegmentFile(slot("default-1")));
+
+ final CountDownLatch slot0DrainStarted = new CountDownLatch(1);
+ final CountDownLatch releaseSlot0Drain = new CountDownLatch(1);
+ final java.util.List builtSlots =
+ java.util.Collections.synchronizedList(new java.util.ArrayList<>());
+ IntFunction factory = idx -> {
+ builtSlots.add(idx);
+ return blockingFakeSender(idx, slot0DrainStarted, releaseSlot0Drain);
+ };
+
+ // minSize=0 -> recovery is the only factory caller. Generous
+ // acquireTimeout so the ONLY reason slot 1 could be skipped is the
+ // shutdown signal being honoured.
+ final SenderPool pool = newDeferredPoolWithFactory(
+ "ws::addr=localhost:1;sf_dir=" + sfDir + ";", 0, 2, 30_000, factory);
+ // Mimic the housekeeper: drive steps back-to-back until done/closing.
+ Thread recovery = new Thread(() -> {
+ try {
+ while (invokeRunStartupRecoveryStep(pool)) {
+ // keep stepping
+ }
+ } catch (Exception ignored) {
+ }
+ }, "test-recovery");
+ recovery.start();
+
+ Assert.assertTrue("slot-0 recoverer must reach drain()",
+ slot0DrainStarted.await(10, TimeUnit.SECONDS));
+
+ // Raise the shutdown signal mid-drain, exactly as QuestDBImpl.close()
+ // does before stopping the housekeeper.
+ invokeMarkClosing(pool);
+ releaseSlot0Drain.countDown();
+ recovery.join(TimeUnit.SECONDS.toMillis(10));
+ Assert.assertFalse("recovery thread must finish", recovery.isAlive());
+
+ Assert.assertTrue("sanity: the in-flight slot-0 recoverer was built",
+ builtSlots.contains(0));
+ Assert.assertFalse(
+ "recovery built a recoverer for slot 1 after the pool was signalled "
+ + "closing; builtSlots=" + builtSlots,
+ builtSlots.contains(1));
+
+ pool.close();
+ });
+ }
+
+ // ----------------------------------------------------------------------
+ // Helpers.
+ // ----------------------------------------------------------------------
+
+ private String slot(String name) {
+ return sfDir + "/" + name;
+ }
+
+ private int countSlotDirs() {
+ if (!Files.exists(sfDir)) {
+ return 0;
+ }
+ int count = 0;
+ long find = Files.findFirst(sfDir);
+ if (find <= 0) {
+ return 0;
+ }
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ rc = Files.findNext(find);
+ if (name == null || ".".equals(name) || "..".equals(name)) {
+ continue;
+ }
+ // Slot dirs are the only children the pool creates under sfDir.
+ if (Files.exists(sfDir + "/" + name + "/.lock")) {
+ count++;
+ }
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ return count;
+ }
+
+ private static boolean hasSegmentFile(String slotPath) {
+ if (!Files.exists(slotPath)) {
+ return false;
+ }
+ long find = Files.findFirst(slotPath);
+ if (find <= 0) {
+ return false;
+ }
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ rc = Files.findNext(find);
+ if (name != null && name.endsWith(".sfa")) {
+ return true;
+ }
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ return false;
+ }
+
+ private static boolean awaitAtLeast(AtomicInteger counter, int target, long timeoutMillis)
+ throws InterruptedException {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (counter.get() >= target) {
+ return true;
+ }
+ Thread.sleep(10);
+ }
+ return counter.get() >= target;
+ }
+
+ private static boolean awaitNoSegmentFile(String slotPath, long timeoutMillis)
+ throws InterruptedException {
+ long deadline = System.currentTimeMillis() + timeoutMillis;
+ while (System.currentTimeMillis() < deadline) {
+ if (!hasSegmentFile(slotPath)) {
+ return true;
+ }
+ Thread.sleep(10);
+ }
+ return !hasSegmentFile(slotPath);
+ }
+
+ private static void rmDir(String dir) {
+ if (dir == null || !Files.exists(dir)) {
+ return;
+ }
+ long find = Files.findFirst(dir);
+ if (find > 0) {
+ try {
+ int rc = 1;
+ while (rc > 0) {
+ String name = Files.utf8ToString(Files.findName(find));
+ if (name != null && !".".equals(name) && !"..".equals(name)) {
+ String child = dir + "/" + name;
+ if (!Files.remove(child)) {
+ rmDir(child);
+ }
+ }
+ rc = Files.findNext(find);
+ }
+ } finally {
+ Files.findClose(find);
+ }
+ }
+ Files.remove(dir);
+ }
+
+ private static Sender getDelegate(PooledSender ps) throws Exception {
+ Field f = PooledSender.class.getDeclaredField("delegate");
+ f.setAccessible(true);
+ return (Sender) f.get(ps);
+ }
+
+ // Invokes one of the pool's private managed-slot delegate factories
+ // (defaultSender / defaultRecoverySender) so a test can inspect the raw
+ // delegate it would build for a given slot index.
+ private static Sender invokeBuildSlotDelegate(SenderPool pool, String methodName, int slotIndex)
+ throws Exception {
+ Method m = SenderPool.class.getDeclaredMethod(methodName, int.class);
+ m.setAccessible(true);
+ return (Sender) m.invoke(pool, slotIndex);
+ }
+
+ // Reads the resolved initial-connect mode a built QwpWebSocketSender delegate
+ // is using (the value after the builder's SYNC auto-promotion / explicit
+ // override has been applied).
+ private static Sender.InitialConnectMode readInitialConnectMode(Sender delegate) throws Exception {
+ Field f = delegate.getClass().getDeclaredField("initialConnectMode");
+ f.setAccessible(true);
+ return (Sender.InitialConnectMode) f.get(delegate);
+ }
+
+ private static void setBooleanField(Object target, String name, boolean value) throws Exception {
+ Field f = target.getClass().getDeclaredField(name);
+ f.setAccessible(true);
+ f.setBoolean(target, value);
+ }
+
+ private static int getIntField(Object target, String name) throws Exception {
+ Field f = target.getClass().getDeclaredField(name);
+ f.setAccessible(true);
+ return f.getInt(target);
+ }
+
+ private static Object getField(Object target, String name) throws Exception {
+ Field f = target.getClass().getDeclaredField(name);
+ f.setAccessible(true);
+ return f.get(target);
+ }
+
+ private static void invokeDiscardBroken(SenderPool pool, PooledSender ps) throws Exception {
+ Method m = SenderPool.class.getDeclaredMethod("discardBroken", PooledSender.class);
+ m.setAccessible(true);
+ m.invoke(pool, ps);
+ }
+
+ // Reaches the package-private senderFactory test seam by reflection so a
+ // test can inject a fake/forged delegate (mirrors SenderPoolErrorSafetyTest).
+ private static SenderPool newPoolWithFactory(
+ String cfg, int min, int max, long acquireMs, IntFunction senderFactory
+ ) throws Exception {
+ Constructor c = SenderPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class, IntFunction.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, senderFactory);
+ }
+
+ // Reaches the package-private 8-arg constructor (deferStartupRecovery=true)
+ // by reflection so a test can build a pool whose SF startup recovery is NOT
+ // run inline -- mirroring the pooled QuestDB handle, which defers it to the
+ // housekeeper. senderFactory=null -> the real defaultSender().
+ private static SenderPool newDeferredPool(String cfg, int min, int max, long acquireMs) throws Exception {
+ Constructor c = SenderPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class,
+ IntFunction.class, boolean.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, null, true);
+ }
+
+ // Drives a deferred pool's startup recovery to completion (the housekeeper
+ // drives it one slot per tick; tests drive the whole backlog in one call).
+ private static void invokeRunStartupRecoveryOnce(SenderPool pool) throws Exception {
+ Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryToCompletion");
+ m.setAccessible(true);
+ m.invoke(pool);
+ }
+
+ // Drives a SINGLE recovery step (the housekeeper's per-tick unit); returns
+ // whether more stranded slots remain.
+ private static boolean invokeRunStartupRecoveryStep(SenderPool pool) throws Exception {
+ Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryStep");
+ m.setAccessible(true);
+ return (Boolean) m.invoke(pool);
+ }
+
+ // Raises the pool's shutdown signal early, exactly as QuestDBImpl.close()
+ // does before stopping the housekeeper.
+ private static void invokeMarkClosing(SenderPool pool) throws Exception {
+ Method m = SenderPool.class.getDeclaredMethod("markClosing");
+ m.setAccessible(true);
+ m.invoke(pool);
+ }
+
+ // Deferred pool (deferStartupRecovery=true) WITH an injected factory, so a
+ // test can drive the housekeeper recovery path against fully controlled
+ // (fake) recoverers.
+ private static SenderPool newDeferredPoolWithFactory(
+ String cfg, int min, int max, long acquireMs, IntFunction factory) throws Exception {
+ Constructor c = SenderPool.class.getDeclaredConstructor(
+ String.class, int.class, int.class, long.class, long.class, long.class,
+ IntFunction.class, boolean.class);
+ c.setAccessible(true);
+ return c.newInstance(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, factory, true);
+ }
+
+ // Fake Sender whose drain() (for slot 0 only) parks until released, opening a
+ // deterministic shutdown-during-recovery window. Holds no native resources.
+ private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, CountDownLatch release) {
+ return (Sender) java.lang.reflect.Proxy.newProxyInstance(
+ Sender.class.getClassLoader(),
+ new Class[]{Sender.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "drain":
+ if (idx == 0) {
+ drainStarted.countDown();
+ release.await();
+ }
+ return true;
+ case "close":
+ return null;
+ case "toString":
+ return "BlockingFakeSender-" + idx;
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "equals":
+ return proxy == args[0];
+ default:
+ Class rt = method.getReturnType();
+ if (rt == boolean.class) return false;
+ if (rt == int.class) return 0;
+ if (rt == long.class) return 0L;
+ if (rt.isInstance(proxy)) return proxy;
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Acks every binary frame with a per-connection running sequence (each
+ * pooled sender is its own WebSocket connection, so each needs its own
+ * FSN counter), and counts total frames received across all connections.
+ */
+ private static final class CountingAckHandler implements TestWebSocketServer.WebSocketServerHandler {
+ final AtomicInteger frames = new AtomicInteger();
+ private final Map seqByClient =
+ new ConcurrentHashMap<>();
+
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ frames.incrementAndGet();
+ AtomicLong seq = seqByClient.computeIfAbsent(client, c -> new AtomicLong(0));
+ try {
+ client.sendBinary(buildAck(seq.getAndIncrement()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static byte[] buildAck(long seq) {
+ byte[] buf = new byte[1 + 8 + 2];
+ ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
+ bb.put((byte) 0x00); // STATUS_OK
+ bb.putLong(seq);
+ bb.putShort((short) 0);
+ return buf;
+ }
+ }
+
+ private static final class SilentHandler implements TestWebSocketServer.WebSocketServerHandler {
+ @Override
+ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
+ // No ack -- frames stay unacked on disk for the recovery test.
+ }
+ }
+}
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java
index 7c58943a..85952f85 100644
--- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java
@@ -26,12 +26,17 @@
import io.questdb.client.Sender;
import io.questdb.client.cutlass.line.LineSenderException;
+import io.questdb.client.impl.PooledSender;
import io.questdb.client.impl.SenderPool;
+import io.questdb.client.test.tools.TestUtils;
import org.junit.Assert;
import org.junit.Test;
+import java.lang.reflect.Field;
+import java.lang.reflect.Proxy;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -487,4 +492,115 @@ public void testThreadAffinityIsPerThread() throws InterruptedException {
pool.releaseCurrentThread();
}
}
+
+ // ----------------------------------------------------------------------
+ // Teardown robustness: a delegate close() can throw an Error (e.g. an
+ // -ea AssertionError), not just a RuntimeException. The pool's best-effort
+ // teardown paths used to catch only RuntimeException, so such an Error
+ // escaped: reapIdle() aborted mid-loop (stranding siblings) and the cap
+ // accounting was skipped, permanently understating capacity. These tests
+ // pin the fix: an Error during delegate close() must not abort the loop,
+ // must not propagate, and must leave the pool fully usable.
+ // ----------------------------------------------------------------------
+
+ @Test
+ public void testReapIdleSurvivesDelegateCloseError() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ // idleTimeout=1ms so every idle slot is reap-eligible after a short sleep.
+ try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 0, 3, 1_000, 1, Long.MAX_VALUE)) {
+ AtomicInteger closeAttempts = new AtomicInteger();
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ PooledSender c = pool.borrow();
+ installFailingCloseDelegate(a, closeAttempts);
+ installFailingCloseDelegate(b, closeAttempts);
+ installFailingCloseDelegate(c, closeAttempts);
+ pool.giveBack(a);
+ pool.giveBack(b);
+ pool.giveBack(c);
+ Assert.assertEquals(3, pool.totalSize());
+
+ Thread.sleep(10);
+ // Must NOT throw even though every delegate.close() raises an Error.
+ pool.reapIdle();
+
+ // The loop attempted to close ALL three delegates -- it did not abort
+ // after the first Error.
+ Assert.assertEquals("reap loop must not abort on a delegate close() Error",
+ 3, closeAttempts.get());
+ // Every reaped slot left `all`; capacity was not stranded.
+ Assert.assertEquals(0, pool.totalSize());
+ Assert.assertEquals(0, pool.availableSize());
+
+ // Pool still grows back to full capacity -- no permanent leak.
+ PooledSender d = pool.borrow();
+ PooledSender e = pool.borrow();
+ PooledSender f = pool.borrow();
+ Assert.assertEquals(3, pool.totalSize());
+ pool.giveBack(d);
+ pool.giveBack(e);
+ pool.giveBack(f);
+ }
+ });
+ }
+
+ @Test
+ public void testCloseSurvivesDelegateCloseError() throws Exception {
+ TestUtils.assertMemoryLeak(() -> {
+ SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 2, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE);
+ AtomicInteger closeAttempts = new AtomicInteger();
+ PooledSender a = pool.borrow();
+ PooledSender b = pool.borrow();
+ installFailingCloseDelegate(a, closeAttempts);
+ installFailingCloseDelegate(b, closeAttempts);
+ pool.giveBack(a);
+ pool.giveBack(b);
+
+ // close() must not propagate an Error and must attempt every delegate.
+ pool.close();
+ Assert.assertEquals("close() must attempt to close every delegate despite an Error",
+ 2, closeAttempts.get());
+ // Idempotent second close stays clean.
+ pool.close();
+ });
+ }
+
+ /**
+ * Reflectively replaces the wrapper's delegate with a {@link Proxy} that
+ * throws an {@link AssertionError} (an {@link Error}, not a
+ * {@link RuntimeException}) on {@code close()} and forwards every other
+ * call to the real delegate. Models an -ea assertion firing during native
+ * teardown.
+ *
+ * The proxy still forwards {@code close()} to the real delegate (freeing
+ * its native {@code HttpClient} buffers) before throwing, so the
+ * pool sees the injected {@link Error} exactly as it would in production
+ * while the test does not leak native memory.
+ */
+ private static void installFailingCloseDelegate(PooledSender ps, AtomicInteger closeAttempts) throws Exception {
+ Field f = PooledSender.class.getDeclaredField("delegate");
+ f.setAccessible(true);
+ Sender real = (Sender) f.get(ps);
+ Sender failing = (Sender) Proxy.newProxyInstance(
+ Sender.class.getClassLoader(),
+ new Class[]{Sender.class},
+ (proxy, method, args) -> {
+ if ("close".equals(method.getName()) && (args == null || args.length == 0)) {
+ closeAttempts.incrementAndGet();
+ // Free the real delegate's native resources first, then
+ // surface the injected Error -- the pool's teardown loop
+ // still observes it, but no native memory is leaked.
+ try {
+ real.close();
+ } catch (Throwable ignore) {
+ // real.close() on an empty HTTP buffer is a no-op;
+ // swallow anything so the injected Error is what the
+ // pool sees.
+ }
+ throw new AssertionError("injected delegate close() Error");
+ }
+ return method.invoke(real, args);
+ });
+ f.set(ps, failing);
+ }
}
diff --git a/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketHandshakeOverflowTest.java b/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketHandshakeOverflowTest.java
index 6d76ab49..25b138bd 100644
--- a/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketHandshakeOverflowTest.java
+++ b/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketHandshakeOverflowTest.java
@@ -124,7 +124,7 @@ private static JavaTlsClientSocket newSocket() throws Exception {
public static final class HandshakeOverflowProvider extends Provider {
public HandshakeOverflowProvider() {
- super(PROVIDER_NAME, "1.0", "test-only");
+ super(PROVIDER_NAME, 1.0, "test-only");
put("SSLContext.TLS", HandshakeOverflowSslContextSpi.class.getName());
}
}
diff --git a/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketTest.java b/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketTest.java
index 7b2736e3..506ce783 100644
--- a/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketTest.java
+++ b/core/src/test/java/io/questdb/client/test/network/JavaTlsClientSocketTest.java
@@ -204,7 +204,7 @@ private static void invoke(Object obj, String methodName) throws Exception {
}
private static JavaTlsClientSocket newSocket() throws Exception {
- var constructor = JavaTlsClientSocket.class.getDeclaredConstructor(
+ java.lang.reflect.Constructor constructor = JavaTlsClientSocket.class.getDeclaredConstructor(
NetworkFacade.class,
org.slf4j.Logger.class,
ClientTlsConfiguration.class
diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java
index f86f1f17..1c461525 100644
--- a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java
+++ b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java
@@ -753,7 +753,7 @@ public void testFitsInStorageSizeCombinatorics() {
{Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, false, false, false, false, false, true},
};
Decimal256 d = new Decimal256();
- for (var combination : combinations) {
+ for (Object[] combination : combinations) {
d.of((long) combination[0], (long) combination[1], (long) combination[2], (long) combination[3], 0);
Assert.assertEquals("Expected " + d + " to fit in a byte", combination[4], d.fitsInStorageSizePow2(0));
Assert.assertEquals("Expected " + d + " to fit in a short", combination[5], d.fitsInStorageSizePow2(1));
diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
index ffb1366e..60cbc9ef 100644
--- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
+++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java
@@ -172,10 +172,21 @@ public static String ipv4ToString(int ip) {
return ((ip >> 24) & 0xff) + "." + ((ip >> 16) & 0xff) + "." + ((ip >> 8) & 0xff) + "." + (ip & 0xff);
}
+ /**
+ * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11).
+ */
+ public static String repeat(CharSequence s, int count) {
+ StringBuilder sb = new StringBuilder(s.length() * Math.max(0, count));
+ for (int i = 0; i < count; i++) {
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
// Useful for debugging
@SuppressWarnings("unused")
public static String reverseBeHex(String hex) {
- var sb = new char[hex.length()];
+ char[] sb = new char[hex.length()];
for (int i = 0; i < hex.length(); i += 2) {
sb[hex.length() - i - 1] = hex.charAt(i + 1);
sb[hex.length() - i - 2] = hex.charAt(i);
diff --git a/core/src/test/java/module-info.java b/core/src/test/java/module-info.java
index e9997b3d..e42b73ff 100644
--- a/core/src/test/java/module-info.java
+++ b/core/src/test/java/module-info.java
@@ -34,6 +34,7 @@
requires org.postgresql.jdbc;
requires jmh.core;
requires ch.qos.logback.classic;
+ requires ch.qos.logback.core;
exports io.questdb.client.test;
exports io.questdb.client.test.cairo;