Skip to content

Commit 24fbbd5

Browse files
authored
feat(qwp): scope the SYMBOL dictionary to a single query (#58)
1 parent 6c336d1 commit 24fbbd5

5 files changed

Lines changed: 281 additions & 10 deletions

File tree

core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,14 +384,16 @@ public void submitQuery(
384384
long initialCredit,
385385
int bindCount,
386386
long bindPayloadPtr,
387-
long bindPayloadLen
387+
long bindPayloadLen,
388+
long queryFlags
388389
) throws InterruptedException {
389390
pendingRequest.sql = sql;
390391
pendingRequest.requestId = requestId;
391392
pendingRequest.initialCredit = initialCredit;
392393
pendingRequest.bindCount = bindCount;
393394
pendingRequest.bindPayloadPtr = bindPayloadPtr;
394395
pendingRequest.bindPayloadLen = bindPayloadLen;
396+
pendingRequest.queryFlags = queryFlags;
395397
requests.put(pendingRequest);
396398
}
397399

@@ -715,6 +717,11 @@ private void sendQueryRequest(QueryRequest req) {
715717
if (req.bindCount > 0 && req.bindPayloadLen > 0) {
716718
sendScratch.putBlockOfBytes(req.bindPayloadPtr, req.bindPayloadLen);
717719
}
720+
// Optional query_flags trailer; omitted when zero so a baseline frame
721+
// stays byte-identical and the server defaults the flags to 0.
722+
if (req.queryFlags != 0) {
723+
sendScratch.putVarint(req.queryFlags);
724+
}
718725
wsClient.sendBinary(sendScratch.getBufferPtr(), sendScratch.getPosition());
719726
sendScratch.reset();
720727
}
@@ -773,6 +780,7 @@ private static final class QueryRequest {
773780
long bindPayloadLen;
774781
long bindPayloadPtr;
775782
long initialCredit;
783+
long queryFlags;
776784
long requestId;
777785
CharSequence sql;
778786
}

core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ public final class QwpEgressMsgKind {
3838
*/
3939
public static final byte CACHE_RESET = 0x17;
4040
public static final byte CANCEL = 0x14;
41+
/**
42+
* {@code SERVER_INFO.capabilities} bit: the server parses the optional
43+
* {@code query_flags:varint} trailer on {@code QUERY_REQUEST}. The client
44+
* appends the trailer only when this bit is set. Mirrors the server-side
45+
* constant {@code io.questdb.cutlass.qwp.codec.QwpEgressMsgKind#CAP_QUERY_FLAGS}.
46+
*/
47+
public static final int CAP_QUERY_FLAGS = 0x00000002;
4148
/**
4249
* {@code SERVER_INFO.capabilities} bit advertising that the frame ends with
4350
* an additional {@code zone_id:u16_len+utf8} field after {@code node_id}.
@@ -54,6 +61,12 @@ public final class QwpEgressMsgKind {
5461
*/
5562
public static final byte EXEC_DONE = 0x16;
5663
public static final byte QUERY_ERROR = 0x13;
64+
/**
65+
* {@code QUERY_REQUEST.query_flags} bit: reset the connection-scoped SYMBOL
66+
* dict before this query, scoping it to the query. Sent only when the server
67+
* advertised {@link #CAP_QUERY_FLAGS}.
68+
*/
69+
public static final int QUERY_FLAG_RESET_DICT = 0x01;
5770
public static final byte QUERY_REQUEST = 0x10;
5871
/**
5972
* Reset mask bit: clear the connection-scoped SYMBOL dict.

core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -801,7 +801,18 @@ public synchronized void connect() {
801801
* before the reset should be discarded by the handler.
802802
*/
803803
public void execute(CharSequence sql, QwpColumnBatchHandler handler) {
804-
execute(sql, null, handler);
804+
execute(sql, null, handler, false);
805+
}
806+
807+
/**
808+
* As {@link #execute(CharSequence, QwpColumnBatchHandler)}, but when
809+
* {@code resetSymbolDict} is set the server resets the connection-scoped
810+
* SYMBOL dict before this query, scoping the dict to this query's symbols.
811+
* The flag reaches the wire only when the server advertised
812+
* {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; otherwise it is silently ignored.
813+
*/
814+
public void execute(CharSequence sql, QwpColumnBatchHandler handler, boolean resetSymbolDict) {
815+
execute(sql, null, handler, resetSymbolDict);
805816
}
806817

807818
/**
@@ -816,6 +827,17 @@ public void execute(CharSequence sql, QwpColumnBatchHandler handler) {
816827
* defeats this reuse.
817828
*/
818829
public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler) {
830+
execute(sql, binds, handler, false);
831+
}
832+
833+
/**
834+
* As {@link #execute(CharSequence, QwpBindSetter, QwpColumnBatchHandler)},
835+
* but when {@code resetSymbolDict} is set the server resets the
836+
* connection-scoped SYMBOL dict before this query, scoping the dict to this
837+
* query's symbols. The flag reaches the wire only when the server advertised
838+
* {@link QwpEgressMsgKind#CAP_QUERY_FLAGS}; otherwise it is silently ignored.
839+
*/
840+
public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) {
819841
if (!executing.compareAndSet(false, true)) {
820842
throw new IllegalStateException(
821843
"QwpQueryClient.execute called while another execute is in flight; one query at a time per client");
@@ -827,7 +849,7 @@ public void execute(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler
827849
// is intentionally NOT cleared inside executeOnce().
828850
pendingCancel = false;
829851
try {
830-
executeImpl(sql, binds, handler);
852+
executeImpl(sql, binds, handler, resetSymbolDict);
831853
} finally {
832854
executing.set(false);
833855
}
@@ -1422,7 +1444,7 @@ private void connectToEndpoint(Endpoint ep) {
14221444
}
14231445
}
14241446

1425-
private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler) {
1447+
private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHandler handler, boolean resetSymbolDict) {
14261448
if (closedFlag.get()) {
14271449
throw new IllegalStateException("QwpQueryClient is closed");
14281450
}
@@ -1446,7 +1468,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa
14461468
while (true) {
14471469
attempt++;
14481470
FailoverProbeHandler probe = new FailoverProbeHandler(handler);
1449-
executeOnce(sql, binds, probe);
1471+
executeOnce(sql, binds, probe, resetSymbolDict);
14501472
if (!probe.transportFailureIntercepted) {
14511473
return;
14521474
}
@@ -1539,7 +1561,7 @@ private void executeImpl(CharSequence sql, QwpBindSetter binds, QwpColumnBatchHa
15391561
* the user's handler in a {@link FailoverProbeHandler} so that the outer
15401562
* loop can intercept transport failures before they reach the user.
15411563
*/
1542-
private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHandler probe) {
1564+
private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHandler probe, boolean resetSymbolDict) {
15431565
// Cache the I/O thread reference at entry: close() may null the field while
15441566
// we are inside this loop, so reading the field per-iteration would NPE
15451567
// exactly when the user is mid-execute() and close() races. The queue and
@@ -1583,7 +1605,8 @@ private void executeOnce(CharSequence sql, QwpBindSetter binds, FailoverProbeHan
15831605
io.requestCancel(requestId);
15841606
}
15851607
try {
1586-
io.submitQuery(sql, requestId, initialCreditBytes, bindValues.count(), bindValues.bufferPtr(), bindValues.bufferLen());
1608+
io.submitQuery(sql, requestId, initialCreditBytes, bindValues.count(), bindValues.bufferPtr(), bindValues.bufferLen(),
1609+
resolveQueryFlags(resetSymbolDict));
15871610
while (true) {
15881611
QueryEvent ev = io.takeEvent();
15891612
try {
@@ -1774,6 +1797,16 @@ private void reconnectViaTracker() {
17741797
+ ", lastError=" + (lastError == null ? "<none>" : lastError.getMessage()) + ']');
17751798
}
17761799

1800+
private long resolveQueryFlags(boolean resetSymbolDict) {
1801+
if (!resetSymbolDict) {
1802+
return 0L;
1803+
}
1804+
QwpServerInfo info = serverInfo;
1805+
return info != null && (info.getCapabilities() & QwpEgressMsgKind.CAP_QUERY_FLAGS) != 0
1806+
? QwpEgressMsgKind.QUERY_FLAG_RESET_DICT
1807+
: 0L;
1808+
}
1809+
17771810
private void runUpgradeWithTimeout(Endpoint ep) {
17781811
// Connect first, OUTSIDE the upgrade try. A connect-phase failure --
17791812
// including a connect_timeout overage flagged via flagAsTimeout() -- must
Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
/*+*****************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2026 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.client.test.cutlass.qwp.client;
26+
27+
import io.questdb.client.cutlass.qwp.client.QwpColumnBatch;
28+
import io.questdb.client.cutlass.qwp.client.QwpColumnBatchHandler;
29+
import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind;
30+
import io.questdb.client.cutlass.qwp.client.QwpQueryClient;
31+
import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
32+
import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
33+
import org.junit.Assert;
34+
import org.junit.Test;
35+
36+
import java.io.IOException;
37+
import java.nio.ByteBuffer;
38+
import java.nio.ByteOrder;
39+
import java.util.Arrays;
40+
import java.util.concurrent.TimeUnit;
41+
import java.util.concurrent.atomic.AtomicReference;
42+
43+
/**
44+
* Wire coverage for the per-query {@code query_flags} trailer that
45+
* {@link QwpQueryClient#execute(CharSequence, QwpColumnBatchHandler, boolean)}
46+
* adds. The mock server captures the raw {@code QUERY_REQUEST} the client emits
47+
* and replies with an {@code EXEC_DONE} so {@code execute} returns; the test
48+
* then inspects the captured bytes. Pins:
49+
* <ul>
50+
* <li>{@code resetSymbolDict=true} against a server advertising
51+
* {@link QwpEgressMsgKind#CAP_QUERY_FLAGS} appends the
52+
* {@link QwpEgressMsgKind#QUERY_FLAG_RESET_DICT} trailer;</li>
53+
* <li>the trailer is the only difference from the flag-off baseline, which
54+
* stays byte-identical;</li>
55+
* <li>without the capability the flag is dropped (no trailer);</li>
56+
* <li>the flag-less overloads never append a trailer.</li>
57+
* </ul>
58+
* The decode -&gt; dict-reset effect on the server, and the end-to-end honouring
59+
* of the flag, are covered against a live server in the questdb repo
60+
* (QwpEgressQueryFlagsResetTest, QwpEgressCacheResetWireTest).
61+
*/
62+
public class QwpQueryClientQueryFlagsTest {
63+
64+
private static final QwpColumnBatchHandler NOOP_HANDLER = new QwpColumnBatchHandler() {
65+
@Override
66+
public void onBatch(QwpColumnBatch batch) {
67+
}
68+
69+
@Override
70+
public void onEnd(long totalRows) {
71+
}
72+
73+
@Override
74+
public void onError(byte status, String message) {
75+
}
76+
};
77+
78+
@Test
79+
public void testFlagDroppedWhenServerLacksCapability() throws Exception {
80+
byte[] frame = runAndCapture(0, c -> c.execute("SELECT 1", NOOP_HANDLER, true));
81+
Assert.assertEquals("flag must be dropped when the server does not advertise CAP_QUERY_FLAGS",
82+
-1L, queryFlagsTrailer(frame));
83+
}
84+
85+
@Test
86+
public void testLegacyThreeArgOverloadSendsNoTrailer() throws Exception {
87+
byte[] frame = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", null, NOOP_HANDLER));
88+
Assert.assertEquals(-1L, queryFlagsTrailer(frame));
89+
}
90+
91+
@Test
92+
public void testLegacyTwoArgOverloadSendsNoTrailer() throws Exception {
93+
byte[] frame = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER));
94+
Assert.assertEquals(-1L, queryFlagsTrailer(frame));
95+
}
96+
97+
@Test
98+
public void testResetFlagAppendsTrailerWhenCapabilityAdvertised() throws Exception {
99+
byte[] frame = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER, true));
100+
Assert.assertEquals((long) QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, queryFlagsTrailer(frame));
101+
}
102+
103+
@Test
104+
public void testResetTrailerIsTheOnlyDifferenceFromBaseline() throws Exception {
105+
byte[] off = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER, false));
106+
byte[] on = runAndCapture(QwpEgressMsgKind.CAP_QUERY_FLAGS, c -> c.execute("SELECT 1", NOOP_HANDLER, true));
107+
// request_id increments per execute; both run on a fresh client so they
108+
// already match, but zero it to keep the comparison about the trailer.
109+
zeroRequestId(off);
110+
zeroRequestId(on);
111+
Assert.assertEquals("reset frame must be the baseline plus a one-byte trailer", off.length + 1, on.length);
112+
Assert.assertArrayEquals("baseline bytes must be untouched", off, Arrays.copyOf(on, off.length));
113+
Assert.assertEquals(QwpEgressMsgKind.QUERY_FLAG_RESET_DICT, on[on.length - 1]);
114+
}
115+
116+
private static byte[] buildExecDone(byte[] queryRequest) {
117+
int bodyLen = 1 + 8 + 1 + 1; // msg_kind + request_id + op_type + rows_affected varint
118+
byte[] frame = new byte[QwpConstants.HEADER_SIZE + bodyLen];
119+
ByteBuffer bb = ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN);
120+
bb.put((byte) 'Q').put((byte) 'W').put((byte) 'P').put((byte) '1');
121+
bb.put((byte) 1); // version
122+
bb.put((byte) 0); // flags
123+
bb.putShort((short) 0); // table_count
124+
bb.putInt(bodyLen); // payload_length
125+
bb.put(QwpEgressMsgKind.EXEC_DONE);
126+
bb.put(queryRequest, 1, 8); // echo request_id verbatim
127+
bb.put((byte) 0); // op_type
128+
bb.put((byte) 0); // rows_affected = 0
129+
return frame;
130+
}
131+
132+
/**
133+
* Parses the {@code QUERY_REQUEST} the client emitted and returns the
134+
* decoded {@code query_flags} trailer, or {@code -1} when no trailer was
135+
* appended. Assumes no binds (the tests use none).
136+
*/
137+
private static long queryFlagsTrailer(byte[] f) {
138+
Assert.assertTrue("captured frame is too short", f.length > 1 + 8);
139+
Assert.assertEquals("captured frame must be a QUERY_REQUEST", QwpEgressMsgKind.QUERY_REQUEST, f[0]);
140+
int[] p = {1 + 8}; // skip msg_kind + request_id
141+
long sqlLen = readVarint(f, p);
142+
p[0] += (int) sqlLen;
143+
readVarint(f, p); // initial_credit
144+
long bindCount = readVarint(f, p);
145+
Assert.assertEquals("test frames carry no binds", 0L, bindCount);
146+
if (p[0] >= f.length) {
147+
return -1L;
148+
}
149+
return readVarint(f, p);
150+
}
151+
152+
private static long readVarint(byte[] buf, int[] pos) {
153+
long value = 0;
154+
int shift = 0;
155+
while (true) {
156+
byte b = buf[pos[0]++];
157+
value |= (long) (b & 0x7F) << shift;
158+
if ((b & 0x80) == 0) {
159+
return value;
160+
}
161+
shift += 7;
162+
}
163+
}
164+
165+
private static byte[] runAndCapture(int serverCapabilities, ExecuteAction action) throws Exception {
166+
CapturingQueryServer handler = new CapturingQueryServer();
167+
try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
168+
server.setSendServerInfo(true);
169+
server.setCapabilities(serverCapabilities);
170+
int port = server.getPort();
171+
server.start();
172+
Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS));
173+
try (QwpQueryClient client = QwpQueryClient.fromConfig(
174+
"ws::addr=localhost:" + port + ";auth_timeout_ms=2000;")) {
175+
client.connect();
176+
Assert.assertTrue("client must connect", client.isConnected());
177+
action.run(client);
178+
}
179+
}
180+
byte[] frame = handler.captured.get();
181+
Assert.assertNotNull("server never received a QUERY_REQUEST", frame);
182+
return frame;
183+
}
184+
185+
private static void zeroRequestId(byte[] frame) {
186+
Arrays.fill(frame, 1, 1 + 8, (byte) 0);
187+
}
188+
189+
@FunctionalInterface
190+
private interface ExecuteAction {
191+
void run(QwpQueryClient client);
192+
}
193+
194+
private static final class CapturingQueryServer implements TestWebSocketServer.WebSocketServerHandler {
195+
final AtomicReference<byte[]> captured = new AtomicReference<>();
196+
197+
@Override
198+
public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) {
199+
if (data.length == 0 || data[0] != QwpEgressMsgKind.QUERY_REQUEST) {
200+
return;
201+
}
202+
captured.compareAndSet(null, data);
203+
try {
204+
client.sendBinary(buildExecDone(data));
205+
} catch (IOException e) {
206+
// best-effort: a failed reply surfaces to the client as a transport error
207+
}
208+
}
209+
}
210+
}

0 commit comments

Comments
 (0)