Skip to content

Commit 6d029fe

Browse files
mtopolnikclaude
mtopolnik
and
claude
committed
Encode null presence inline in column data
Replace the nullable type-code flag (0x80 high bit) with an inline null-count byte at the start of each column's data. QwpColumnWriter.writeNullHeader() now writes a single byte (0 = no nulls, 1 = has nulls) followed by the bitmap only when nulls are present. QwpColumnDef no longer stores or OR's a nullable flag into the type code. The hasNullBitmap field and the 3-argument constructor are removed. QwpSchemaHash hashes only the base type code. QwpTableBuffer.ColumnBuffer defers null bitmap expansion to addNull() calls instead of checking capacity on every row. Non-null addXxx() methods no longer touch the bitmap at all. The bitmap is tail-expanded to the full row count at serialization time in writeNullHeader(). Safety fixes: isNull() returns false for indices beyond bitmap capacity, truncateTo() and clearToEmptyFast() clamp to the allocated bitmap size. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent aa55767 commit 6d029fe

9 files changed

Lines changed: 52 additions & 108 deletions

File tree

‎core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,7 @@ private void encodeColumn(
5858
) {
5959
long dataAddr = col.getDataAddress();
6060

61-
if (colDef.hasNullBitmap()) {
62-
writeNullBitmap(col, rowCount);
63-
}
61+
writeNullHeader(col, rowCount, rowCount - valueCount);
6462

6563
switch (col.getType()) {
6664
case TYPE_BOOLEAN:
@@ -281,16 +279,15 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) {
281279
}
282280
}
283281

284-
private void writeNullBitmap(QwpTableBuffer.ColumnBuffer col, int rowCount) {
285-
long nullAddr = col.getNullBitmapAddress();
286-
if (nullAddr != 0) {
282+
private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) {
283+
if (nullCount > 0) {
284+
buffer.putByte((byte) 1);
285+
col.ensureNullBitmapCapacity(rowCount);
286+
long nullAddr = col.getNullBitmapAddress();
287287
int bitmapSize = (rowCount + 7) / 8;
288288
buffer.putBlockOfBytes(nullAddr, bitmapSize);
289289
} else {
290-
int bitmapSize = (rowCount + 7) / 8;
291-
for (int i = 0; i < bitmapSize; i++) {
292-
buffer.putByte((byte) 0);
293-
}
290+
buffer.putByte((byte) 0);
294291
}
295292
}
296293

‎core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpColumnDef.java‎

Lines changed: 7 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -34,33 +34,17 @@
3434
*/
3535
public final class QwpColumnDef {
3636
private final String name;
37-
private final boolean hasNullBitmap;
3837
private final byte typeCode;
3938

4039
/**
4140
* Creates a column definition.
4241
*
4342
* @param name the column name (UTF-8)
44-
* @param typeCode the QWP v1 type code (0x01-0x0F, optionally OR'd with 0x80 for null bitmap)
43+
* @param typeCode the QWP v1 type code (0x01-0x16)
4544
*/
4645
public QwpColumnDef(String name, byte typeCode) {
4746
this.name = name;
48-
// Extract null bitmap flag (high bit) and base type
49-
this.hasNullBitmap = (typeCode & 0x80) != 0;
50-
this.typeCode = (byte) (typeCode & 0x7F);
51-
}
52-
53-
/**
54-
* Creates a column definition with explicit null bitmap flag.
55-
*
56-
* @param name the column name
57-
* @param typeCode the base type code (0x01-0x0F)
58-
* @param hasNullBitmap whether the column has a null bitmap
59-
*/
60-
public QwpColumnDef(String name, byte typeCode, boolean hasNullBitmap) {
61-
this.name = name;
62-
this.typeCode = (byte) (typeCode & 0x7F);
63-
this.hasNullBitmap = hasNullBitmap;
47+
this.typeCode = typeCode;
6448
}
6549

6650
@Override
@@ -69,7 +53,6 @@ public boolean equals(Object o) {
6953
if (o == null || getClass() != o.getClass()) return false;
7054
QwpColumnDef that = (QwpColumnDef) o;
7155
return typeCode == that.typeCode &&
72-
hasNullBitmap == that.hasNullBitmap &&
7356
name.equals(that.name);
7457
}
7558

@@ -81,9 +64,9 @@ public String getName() {
8164
}
8265

8366
/**
84-
* Gets the base type code (without null bitmap flag).
67+
* Gets the base type code.
8568
*
86-
* @return type code 0x01-0x0F
69+
* @return type code 0x01-0x16
8770
*/
8871
public byte getTypeCode() {
8972
return typeCode;
@@ -97,37 +80,24 @@ public String getTypeName() {
9780
}
9881

9982
/**
100-
* Gets the wire type code (with null bitmap flag if applicable).
83+
* Gets the wire type code.
10184
*
10285
* @return type code as sent on wire
10386
*/
10487
public byte getWireTypeCode() {
105-
return hasNullBitmap ? (byte) (typeCode | 0x80) : typeCode;
88+
return typeCode;
10689
}
10790

10891
@Override
10992
public int hashCode() {
11093
int result = name.hashCode();
11194
result = 31 * result + typeCode;
112-
result = 31 * result + (hasNullBitmap ? 1 : 0);
11395
return result;
11496
}
11597

116-
/**
117-
* Returns true if this column has a null bitmap.
118-
*/
119-
public boolean hasNullBitmap() {
120-
return hasNullBitmap;
121-
}
122-
12398
@Override
12499
public String toString() {
125-
StringBuilder sb = new StringBuilder();
126-
sb.append(name).append(':').append(getTypeName());
127-
if (hasNullBitmap) {
128-
sb.append('?');
129-
}
130-
return sb.toString();
100+
return name + ':' + getTypeName();
131101
}
132102

133103
/**

‎core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ private QwpConstants() {
269269
* @return size in bytes, 0 for bit-packed (BOOLEAN), or -1 for variable-width types
270270
*/
271271
public static int getFixedTypeSize(byte typeCode) {
272-
int code = typeCode & TYPE_MASK;
272+
int code = typeCode;
273273
switch (code) {
274274
case TYPE_BOOLEAN:
275275
return 0; // Special: bit-packed
@@ -308,8 +308,7 @@ public static int getFixedTypeSize(byte typeCode) {
308308
* @return type name
309309
*/
310310
public static String getTypeName(byte typeCode) {
311-
int code = typeCode & TYPE_MASK;
312-
boolean hasNullBitmap = (typeCode & TYPE_NULLABLE_FLAG) != 0;
311+
int code = typeCode;
313312
String name;
314313
switch (code) {
315314
case TYPE_BOOLEAN:
@@ -382,7 +381,7 @@ public static String getTypeName(byte typeCode) {
382381
name = "UNKNOWN(" + code + ")";
383382
break;
384383
}
385-
return hasNullBitmap ? name + "?" : name;
384+
return name;
386385
}
387386

388387
/**
@@ -392,7 +391,7 @@ public static String getTypeName(byte typeCode) {
392391
* @return true if fixed-width
393392
*/
394393
public static boolean isFixedWidthType(byte typeCode) {
395-
int code = typeCode & TYPE_MASK;
394+
int code = typeCode;
396395
return code == TYPE_BOOLEAN ||
397396
code == TYPE_BYTE ||
398397
code == TYPE_SHORT ||

‎core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaHash.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,9 +152,8 @@ public static long computeSchemaHashDirect(io.questdb.client.std.ObjList<QwpTabl
152152
hasher.update((byte) (0x80 | (c & 0x3F)));
153153
}
154154
}
155-
// Wire type code: type | (useNullBitmap ? 0x80 : 0)
156-
byte wireType = (byte) (col.getType() | (col.useNullBitmap ? 0x80 : 0));
157-
hasher.update(wireType);
155+
// Wire type code: just the base type, no nullable flag
156+
hasher.update(col.getType());
158157
}
159158

160159
return hasher.getValue();

‎core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java‎

Lines changed: 12 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ public QwpColumnDef[] getColumnDefs() {
176176
cachedColumnDefs = new QwpColumnDef[columns.size()];
177177
for (int i = 0; i < columns.size(); i++) {
178178
ColumnBuffer col = columns.get(i);
179-
cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type, col.useNullBitmap);
179+
cachedColumnDefs[i] = new QwpColumnDef(col.name, col.type);
180180
}
181181
columnDefsCacheValid = true;
182182
}
@@ -633,14 +633,12 @@ public ColumnBuffer(String name, byte type, boolean useNullBitmap) {
633633
}
634634

635635
public void addBoolean(boolean value) {
636-
ensureNullBitmapCapacity();
637636
dataBuffer.putByte(value ? (byte) 1 : (byte) 0);
638637
valueCount++;
639638
size++;
640639
}
641640

642641
public void addByte(byte value) {
643-
ensureNullBitmapCapacity();
644642
dataBuffer.putByte(value);
645643
valueCount++;
646644
size++;
@@ -651,7 +649,6 @@ public void addDecimal128(Decimal128 value) {
651649
addNull();
652650
return;
653651
}
654-
ensureNullBitmapCapacity();
655652
if (decimalScale == -1) {
656653
decimalScale = (byte) value.getScale();
657654
} else if (decimalScale != value.getScale()) {
@@ -684,7 +681,6 @@ public void addDecimal256(Decimal256 value) {
684681
addNull();
685682
return;
686683
}
687-
ensureNullBitmapCapacity();
688684
Decimal256 src = value;
689685
if (decimalScale == -1) {
690686
decimalScale = (byte) value.getScale();
@@ -711,7 +707,6 @@ public void addDecimal64(Decimal64 value) {
711707
addNull();
712708
return;
713709
}
714-
ensureNullBitmapCapacity();
715710
if (decimalScale == -1) {
716711
decimalScale = (byte) value.getScale();
717712
dataBuffer.putLong(value.getValue());
@@ -737,7 +732,6 @@ public void addDecimal64(Decimal64 value) {
737732
}
738733

739734
public void addDouble(double value) {
740-
ensureNullBitmapCapacity();
741735
dataBuffer.putDouble(value);
742736
valueCount++;
743737
size++;
@@ -844,7 +838,6 @@ public void addDoubleArrayPayload(long ptr, long len) {
844838
}
845839

846840
public void addFloat(float value) {
847-
ensureNullBitmapCapacity();
848841
dataBuffer.putFloat(value);
849842
valueCount++;
850843
size++;
@@ -867,28 +860,24 @@ public void addGeoHash(long value, int precision) {
867860
"GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision
868861
);
869862
}
870-
ensureNullBitmapCapacity();
871863
dataBuffer.putLong(value);
872864
valueCount++;
873865
size++;
874866
}
875867

876868
public void addInt(int value) {
877-
ensureNullBitmapCapacity();
878869
dataBuffer.putInt(value);
879870
valueCount++;
880871
size++;
881872
}
882873

883874
public void addLong(long value) {
884-
ensureNullBitmapCapacity();
885875
dataBuffer.putLong(value);
886876
valueCount++;
887877
size++;
888878
}
889879

890880
public void addLong256(long l0, long l1, long l2, long l3) {
891-
ensureNullBitmapCapacity();
892881
dataBuffer.putLong(l0);
893882
dataBuffer.putLong(l1);
894883
dataBuffer.putLong(l2);
@@ -995,7 +984,7 @@ public void addLongArray(LongArray array) {
995984

996985
public void addNull() {
997986
if (useNullBitmap) {
998-
ensureNullBitmapCapacity();
987+
ensureNullBitmapCapacity(size + 1);
999988
markNull(size);
1000989
} else {
1001990
// For non-nullable columns, store a sentinel/default value
@@ -1069,18 +1058,16 @@ public void addNull() {
10691058
}
10701059

10711060
public void addShort(short value) {
1072-
ensureNullBitmapCapacity();
10731061
dataBuffer.putShort(value);
10741062
valueCount++;
10751063
size++;
10761064
}
10771065

10781066
public void addString(CharSequence value) {
10791067
if (value == null && useNullBitmap) {
1080-
ensureNullBitmapCapacity();
1068+
ensureNullBitmapCapacity(size + 1);
10811069
markNull(size);
10821070
} else {
1083-
ensureNullBitmapCapacity();
10841071
if (value != null) {
10851072
stringData.putUtf8(value);
10861073
}
@@ -1100,7 +1087,6 @@ public void addSymbol(CharSequence value) {
11001087
addSymbolWithGlobalId(value, globalId);
11011088
return;
11021089
}
1103-
ensureNullBitmapCapacity();
11041090
int idx = getOrAddLocalSymbol(value);
11051091
dataBuffer.putInt(idx);
11061092
valueCount++;
@@ -1128,7 +1114,6 @@ public void addSymbolUtf8(long ptr, int len) {
11281114
addSymbolWithGlobalId(lookupSink, globalId);
11291115
return;
11301116
}
1131-
ensureNullBitmapCapacity();
11321117
int idx = getOrAddLocalSymbol(lookupSink);
11331118
dataBuffer.putInt(idx);
11341119
valueCount++;
@@ -1140,7 +1125,6 @@ public void addSymbolWithGlobalId(CharSequence value, int globalId) {
11401125
addNull();
11411126
return;
11421127
}
1143-
ensureNullBitmapCapacity();
11441128
int localIdx = getOrAddLocalSymbol(value);
11451129
dataBuffer.putInt(localIdx);
11461130

@@ -1158,7 +1142,6 @@ public void addSymbolWithGlobalId(CharSequence value, int globalId) {
11581142
}
11591143

11601144
public void addUuid(long high, long low) {
1161-
ensureNullBitmapCapacity();
11621145
// Store in wire order: lo first, hi second
11631146
dataBuffer.putLong(low);
11641147
dataBuffer.putLong(high);
@@ -1332,7 +1315,7 @@ public boolean hasSymbol(CharSequence value) {
13321315
}
13331316

13341317
public boolean isNull(int index) {
1335-
if (nullBufPtr == 0) {
1318+
if (nullBufPtr == 0 || index >= nullBufCapRows) {
13361319
return false;
13371320
}
13381321
long longAddr = nullBufPtr + ((long) (index >>> 6)) * 8;
@@ -1422,7 +1405,7 @@ public void truncateTo(int newSize) {
14221405
}
14231406
}
14241407
// Clear null bits for truncated rows
1425-
for (int i = newSize; i < size; i++) {
1408+
for (int i = newSize; i < Math.min(size, nullBufCapRows); i++) {
14261409
long longAddr = nullBufPtr + ((long) (i >>> 6)) * 8;
14271410
int bitIndex = i & 63;
14281411
long current = Unsafe.getUnsafe().getLong(longAddr);
@@ -1617,7 +1600,8 @@ private void clearToEmptyFast() {
16171600
int sizeBefore = size;
16181601
clearValuePayload();
16191602
if (nullBufPtr != 0 && sizeBefore > 0) {
1620-
long usedLongs = ((long) sizeBefore + 63) >>> 6;
1603+
int rowsToClear = Math.min(sizeBefore, nullBufCapRows);
1604+
long usedLongs = ((long) rowsToClear + 63) >>> 6;
16211605
Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0);
16221606
}
16231607
size = 0;
@@ -1650,7 +1634,8 @@ private void compactNullBitmap(int sourceRow) {
16501634
}
16511635

16521636
boolean retainedNull = isNull(sourceRow);
1653-
long usedLongs = ((long) size + 63) >>> 6;
1637+
int rowsToClear = Math.min(size, nullBufCapRows);
1638+
long usedLongs = ((long) rowsToClear + 63) >>> 6;
16541639
Vect.memset(nullBufPtr, usedLongs * Long.BYTES, 0);
16551640
if (retainedNull) {
16561641
Unsafe.getUnsafe().putLong(nullBufPtr, 1L);
@@ -1664,11 +1649,6 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
16641649
arrayDims = Arrays.copyOf(arrayDims, arrayDims.length * 2);
16651650
}
16661651

1667-
// Ensure null bitmap capacity
1668-
if (useNullBitmap) {
1669-
ensureNullBitmapCapacity();
1670-
}
1671-
16721652
// Ensure shape array capacity
16731653
int requiredShapeCapacity = arrayShapeOffset + nDims;
16741654
if (arrayShapes == null) {
@@ -1694,11 +1674,11 @@ private void ensureArrayCapacity(int nDims, int dataElements) {
16941674
}
16951675
}
16961676

1697-
private void ensureNullBitmapCapacity() {
1698-
if (nullBufPtr == 0 || nullBufCapRows > size) {
1677+
public void ensureNullBitmapCapacity(int minRows) {
1678+
if (nullBufPtr == 0 || nullBufCapRows >= minRows) {
16991679
return;
17001680
}
1701-
int newCapRows = Math.max(nullBufCapRows * 2, ((size + 64) >>> 6) << 6);
1681+
int newCapRows = Math.max(nullBufCapRows * 2, ((minRows + 63) >>> 6) << 6);
17021682
long newSizeBytes = (long) newCapRows >>> 3;
17031683
long oldSizeBytes = (long) nullBufCapRows >>> 3;
17041684
nullBufPtr = Unsafe.realloc(nullBufPtr, oldSizeBytes, newSizeBytes, MemoryTag.NATIVE_ILP_RSS);

0 commit comments

Comments
 (0)