Lazy null-index fill on the no-nulls fast path

bluestreak01 · claude · bluestreak01 · commit 6a99163fb012 · 2026-04-21T10:14:52.000+01:00
parseNullSection now skips the per-row "nonNullIdx[i] = i" array fill
when the column carries no nulls in this batch. Saves O(rowCount *
columnCount) trivial assignments per batch on the most common decode
path. For a 16K-row x 100-column wide result that's 1.6M skipped
iterations every batch.

QwpColumnLayout gains a denseIndex(row) helper that returns row
directly when nullBitmapAddr == 0, otherwise reads nonNullIdx[row].
All 13 typed accessors in QwpColumnBatch (getBool, getByteValue,
getCharValue, getDecimal128High/Low, getDouble, getFloat, getIntValue,
getLong, getLong256Word, getLongArray, getLongValue, getShortValue,
getUuidHi/Lo, lookupBinaryBytes, lookupStringBytes) now go through
denseIndex. parseArrayColumn and parseSymbolColumn hoist the
discriminator out of their per-row loops.

The public raw-API nonNullIndex(col) lazily materialises the identity
array on first call when the column has no nulls, so the existing
fragmentation / compression / credit-flow tests that consume the raw
API keep working with no per-call cost amortised across reuse.

Also drops QwpSpscQueue.clearAndWakeConsumer: zero callers in the
codebase and the implementation wrote the consumer-only tail field
from a context the doc explicitly described as the producer.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java
@@ -118,7 +118,7 @@ public io.questdb.client.std.bytes.DirectByteSequence getBinaryB(int col, int ro
     public boolean getBool(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return false;
-        int denseIdx = l.nonNullIdx[row];
+        int denseIdx = l.denseIndex(row);
         // Bit-packed: 8 values per byte, LSB-first
         byte b = Unsafe.getUnsafe().getByte(l.valuesAddr + (denseIdx >>> 3));
         return (b & (1 << (denseIdx & 7))) != 0;
@@ -131,7 +131,7 @@ public boolean getBool(int col, int row) {
     public byte getByteValue(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0;
-        return Unsafe.getUnsafe().getByte(l.valuesAddr + l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getByte(l.valuesAddr + l.denseIndex(row));
     }
 
     /**
@@ -140,7 +140,7 @@ public byte getByteValue(int col, int row) {
     public char getCharValue(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0;
-        return (char) Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.nonNullIdx[row]);
+        return (char) Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.denseIndex(row));
     }
 
     public int getColumnCount() {
@@ -161,7 +161,7 @@ public byte getColumnWireType(int col) {
     public long getDecimal128High(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.nonNullIdx[row] + 8L);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row) + 8L);
     }
 
     /**
@@ -170,7 +170,7 @@ public long getDecimal128High(int col, int row) {
     public long getDecimal128Low(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row));
     }
 
     public int getDecimalScale(int col) {
@@ -180,7 +180,7 @@ public int getDecimalScale(int col) {
     public double getDouble(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return Double.NaN;
-        return Unsafe.getUnsafe().getDouble(l.valuesAddr + 8L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getDouble(l.valuesAddr + 8L * l.denseIndex(row));
     }
 
     /**
@@ -209,7 +209,7 @@ public double[] getDoubleArrayElements(int col, int row) {
     public float getFloat(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return Float.NaN;
-        return Unsafe.getUnsafe().getFloat(l.valuesAddr + 4L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getFloat(l.valuesAddr + 4L * l.denseIndex(row));
     }
 
     public int getGeohashPrecisionBits(int col) {
@@ -223,7 +223,7 @@ public int getGeohashPrecisionBits(int col) {
     public int getIntValue(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0;
-        return Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * l.denseIndex(row));
     }
 
     /**
@@ -238,7 +238,7 @@ public long getLong(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
         byte wt = l.info.wireType;
-        int denseIdx = l.nonNullIdx[row];
+        int denseIdx = l.denseIndex(row);
         if (wt == QwpConstants.TYPE_LONG || wt == QwpConstants.TYPE_DATE
                 || wt == QwpConstants.TYPE_TIMESTAMP || wt == QwpConstants.TYPE_TIMESTAMP_NANOS
                 || wt == QwpConstants.TYPE_DECIMAL64) {
@@ -274,7 +274,7 @@ public long getLong(int col, int row) {
     public long getLong256Word(int col, int row, int wordIndex) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 32L * l.nonNullIdx[row] + 8L * wordIndex);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 32L * l.denseIndex(row) + 8L * wordIndex);
     }
 
     // Raw column-address API -- for zero-branch hot inner loops.
@@ -299,7 +299,7 @@ public long[] getLongArray(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return null;
         byte wt = l.info.wireType;
-        int denseIdx = l.nonNullIdx[row];
+        int denseIdx = l.denseIndex(row);
         if (wt == QwpConstants.TYPE_UUID || wt == QwpConstants.TYPE_DECIMAL128) {
             long base = l.valuesAddr + 16L * denseIdx;
             return new long[]{Unsafe.getUnsafe().getLong(base), Unsafe.getUnsafe().getLong(base + 8)};
@@ -324,7 +324,7 @@ public long[] getLongArray(int col, int row) {
     public long getLongValue(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 8L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 8L * l.denseIndex(row));
     }
 
     public int getRowCount() {
@@ -337,7 +337,7 @@ public int getRowCount() {
     public short getShortValue(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0;
-        return Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getShort(l.valuesAddr + 2L * l.denseIndex(row));
     }
 
     /**
@@ -379,7 +379,7 @@ public String getString(int col, int row) {
     public long getUuidHi(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.nonNullIdx[row] + 8L);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row) + 8L);
     }
 
     /**
@@ -388,7 +388,7 @@ public long getUuidHi(int col, int row) {
     public long getUuidLo(int col, int row) {
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (isLayoutNull(l, row)) return 0L;
-        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.nonNullIdx[row]);
+        return Unsafe.getUnsafe().getLong(l.valuesAddr + 16L * l.denseIndex(row));
     }
 
     /**
@@ -468,9 +468,23 @@ public int nonNullCount(int col) {
      * column's non-null values, or -1 if the row is NULL. Array length equals
      * {@link #getRowCount()}. Valid only during the current {@code onBatch}
      * callback; do not retain.
+     * <p>
+     * For columns with no nulls in this batch the decoder skips populating
+     * this array (saves O(rowCount * columnCount) per batch on the typical
+     * no-nulls hot path). This accessor lazy-materialises an identity mapping
+     * on demand for raw-API callers; the result is cached on the layout so
+     * repeated calls within the same batch reuse the allocation. The typed
+     * accessors ({@link #getLong}, {@link #getDouble}, etc.) avoid this
+     * allocation entirely via {@link QwpColumnLayout#denseIndex}.
      */
     public int[] nonNullIndex(int col) {
-        return columnLayouts.getQuick(col).nonNullIdx;
+        QwpColumnLayout l = columnLayouts.getQuick(col);
+        if (l.nullBitmapAddr == 0 && l.nonNullIdx == null) {
+            int[] arr = new int[rowCount];
+            for (int i = 0; i < rowCount; i++) arr[i] = i;
+            l.nonNullIdx = arr;
+        }
+        return l.nonNullIdx;
     }
 
     /**
@@ -532,7 +546,7 @@ private io.questdb.client.std.bytes.DirectByteSequence lookupBinaryBytes(
         if (isNull(col, row)) return null;
         QwpColumnLayout l = columnLayouts.getQuick(col);
         if (l.info.wireType != QwpConstants.TYPE_BINARY) return null;
-        int denseIdx = l.nonNullIdx[row];
+        int denseIdx = l.denseIndex(row);
         int startOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * denseIdx);
         int endOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * (denseIdx + 1));
         return view.of(l.stringBytesAddr + startOff, endOff - startOff);
@@ -547,7 +561,7 @@ private DirectUtf8Sequence lookupStringBytes(int col, int row, DirectUtf8String
         if (isNull(col, row)) return null;
         QwpColumnLayout l = columnLayouts.getQuick(col);
         byte wt = l.info.wireType;
-        int denseIdx = l.nonNullIdx[row];
+        int denseIdx = l.denseIndex(row);
         if (wt == QwpConstants.TYPE_STRING || wt == QwpConstants.TYPE_VARCHAR) {
             int startOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * denseIdx);
             int endOff = Unsafe.getUnsafe().getInt(l.valuesAddr + 4L * (denseIdx + 1));
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnLayout.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnLayout.java
@@ -117,6 +117,21 @@ public class QwpColumnLayout implements QuietCloseable {
     private long timestampDecodeAddr;
     private int timestampDecodeCapacity;
 
+    /**
+     * Returns the dense index of {@code row} into the non-null values array.
+     * For columns with no nulls in this batch ({@code nullBitmapAddr == 0}),
+     * dense index equals row and {@link #nonNullIdx} is left unread (the
+     * decoder skips the per-row array fill on this path). Otherwise the
+     * pre-computed slot is returned.
+     * <p>
+     * Caller MUST have null-checked the cell first via the surrounding
+     * {@code isNull} / {@code isLayoutNull} guard -- this method does not
+     * detect null rows on its own.
+     */
+    public int denseIndex(int row) {
+        return nullBitmapAddr == 0 ? row : nonNullIdx[row];
+    }
+
     public void clear() {
         info = null;
         valuesAddr = 0;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java
@@ -512,8 +512,13 @@ private long parseArrayColumn(QwpColumnLayout layout, int rowCount, long p, long
         layout.arrayRowAddr = ensureLongArray(layout.arrayRowAddr, rowCount);
         layout.arrayRowLen = ensureIntArray(layout.arrayRowLen, rowCount);
         layout.valuesAddr = p;
+        // Hoist the no-nulls discriminator out of the per-row loop -- when the
+        // column has no nulls in this batch, every row is non-null and the
+        // null-skip branch is dead.
+        boolean noNulls = layout.nullBitmapAddr == 0;
+        int[] nonNullIdx = layout.nonNullIdx;
         for (int i = 0; i < rowCount; i++) {
-            if (layout.nonNullIdx[i] < 0) {
+            if (!noNulls && nonNullIdx[i] < 0) {
                 layout.arrayRowAddr[i] = 0;
                 layout.arrayRowLen[i] = 0;
                 continue;
@@ -668,16 +673,25 @@ private long parseDeltaSymbolDict(long p, long limit) throws QwpDecodeException
     private long parseNullSection(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException {
         if (p >= limit) throw new QwpDecodeException("truncated null flag");
         byte flag = Unsafe.getUnsafe().getByte(p++);
-        layout.nonNullIdx = ensureIntArray(layout.nonNullIdx, rowCount);
         if (flag == 0) {
+            // No nulls in this column -- skip the per-row "nonNullIdx[i] = i"
+            // array fill entirely. Accessors detect the no-nulls case via
+            // {@code nullBitmapAddr == 0} and treat dense-index == row directly
+            // (see {@link QwpColumnLayout#denseIndex}), so the array is unread
+            // on this path. For a 16K-row x 100-column wide result this saves
+            // 1.6M trivial assignments per batch. nonNullIdx is nulled so a
+            // raw-API caller of {@code QwpColumnBatch.nonNullIndex(col)} can
+            // distinguish "needs identity-fill" from "fully populated by a
+            // prior with-nulls batch"; that path lazy-materialises on demand.
             layout.nullBitmapAddr = 0;
+            layout.nonNullIdx = null;
             layout.nonNullCount = rowCount;
-            for (int i = 0; i < rowCount; i++) layout.nonNullIdx[i] = i;
             return p;
         }
         int bitmapBytes = (rowCount + 7) >>> 3;
         if (p + bitmapBytes > limit) throw new QwpDecodeException("truncated null bitmap");
         layout.nullBitmapAddr = p;
+        layout.nonNullIdx = ensureIntArray(layout.nonNullIdx, rowCount);
         int denseIdx = 0;
         for (int i = 0; i < rowCount; i++) {
             int bi = i >>> 3;
@@ -732,9 +746,13 @@ private long parseSymbolColumn(QwpColumnLayout layout, int rowCount, long p, lon
         layout.symbolDictSize = dictSize;
         // Materialise per-row IDs into int[rowCount] so random access is O(1).
         layout.symbolRowIds = ensureIntArray(layout.symbolRowIds, rowCount);
+        // Hoist the no-nulls discriminator out of the per-row loop -- when the
+        // column has no nulls in this batch, every row carries an id and the
+        // null-skip branch is dead.
+        boolean noNulls = layout.nullBitmapAddr == 0;
+        int[] nonNullIdx = layout.nonNullIdx;
         for (int i = 0; i < rowCount; i++) {
-            int denseIdx = layout.nonNullIdx[i];
-            if (denseIdx < 0) continue; // NULL row; leave slot stale
+            if (!noNulls && nonNullIdx[i] < 0) continue; // NULL row; leave slot stale
             decodeVarint(p, limit);
             p = varintPos;
             int id = (int) varintValue;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSpscQueue.java
@@ -69,25 +69,6 @@ public QwpSpscQueue(int capacity) {
         this.mask = pow2 - 1;
     }
 
-    /**
-     * Drops any queued items and wakes the consumer if it is currently parked.
-     * Safe to call concurrently with poll/take only if the caller is the
-     * producer; otherwise external synchronisation is required.
-     */
-    public void clearAndWakeConsumer() {
-        long h = head;
-        long t = tail;
-        while (t < h) {
-            slots[(int) (t & mask)] = null;
-            t++;
-        }
-        tail = t;
-        Thread consumer = consumerThread;
-        if (consumer != null) {
-            LockSupport.unpark(consumer);
-        }
-    }
-
     /**
      * Publishes {@code value} to the consumer. Returns {@code false} when the
      * ring is full (caller may retry or spin externally). Never blocks.
