Skip to content

Commit 9700754

Browse files
authored
refactor(qwp): rename sf_max_bytes to sf_max_segment_bytes (#70)
Renames the store-and-forward connect-string key sf_max_bytes to sf_max_segment_bytes: the key caps a single SF segment, and the new name says so; it also pairs cleanly with sf_max_total_bytes. The builder method follows: storeAndForwardMaxBytes() is now storeAndForwardMaxSegmentBytes(). Internal identifiers, error messages, and test assertions rename in step, so git grep -iE 'sf[_-]?max[_-]?bytes|storeandforwardmaxbytes' returns zero.
1 parent 1d54a25 commit 9700754

10 files changed

Lines changed: 81 additions & 37 deletions

File tree

core/src/main/java/io/questdb/client/Sender.java

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1128,7 +1128,7 @@ public int getConnectTimeout() {
11281128
// Durability contract for SF append/flush. FLUSH and APPEND remain
11291129
// deferred follow-ups; PERIODIC uses the segment manager.
11301130
private SfDurability sfDurability = SfDurability.MEMORY;
1131-
private long sfMaxBytes = PARAMETER_NOT_SET_EXPLICITLY;
1131+
private long sfMaxSegmentBytes = PARAMETER_NOT_SET_EXPLICITLY;
11321132
private long sfMaxTotalBytes = PARAMETER_NOT_SET_EXPLICITLY;
11331133
private long sfSyncIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY;
11341134
private boolean shouldDestroyPrivKey;
@@ -1452,17 +1452,17 @@ public Sender build() {
14521452
// (same lock-free architecture, no disk involvement).
14531453
// Durability-combination validation lives in validateParameters
14541454
// so build() and no-connect validation apply the same rules.
1455-
long actualSfMaxBytes = sfMaxBytes == PARAMETER_NOT_SET_EXPLICITLY
1455+
long actualSfMaxSegmentBytes = sfMaxSegmentBytes == PARAMETER_NOT_SET_EXPLICITLY
14561456
? DEFAULT_SEGMENT_BYTES
1457-
: sfMaxBytes;
1457+
: sfMaxSegmentBytes;
14581458
// Default cap depends on backing: RAM (memory mode) is tight
14591459
// by default; disk (SF mode) is cheap so the default is
14601460
// generous enough that normal traffic never hits it.
14611461
long defaultMaxTotal = sfDir == null
14621462
? DEFAULT_MAX_BYTES_MEMORY
14631463
: DEFAULT_MAX_BYTES_SF;
14641464
long actualSfMaxTotalBytes = sfMaxTotalBytes == PARAMETER_NOT_SET_EXPLICITLY
1465-
? Math.max(defaultMaxTotal, actualSfMaxBytes * 2)
1465+
? Math.max(defaultMaxTotal, actualSfMaxSegmentBytes * 2)
14661466
: sfMaxTotalBytes;
14671467
long actualCloseFlushTimeoutMillis = closeFlushTimeoutMillis == CLOSE_FLUSH_TIMEOUT_NOT_SET
14681468
? DEFAULT_CLOSE_FLUSH_TIMEOUT_MILLIS
@@ -1556,7 +1556,7 @@ public Sender build() {
15561556
? DEFAULT_SF_SYNC_INTERVAL_MILLIS : sfSyncIntervalMillis) * 1_000_000L
15571557
: 0L;
15581558
CursorSendEngine cursorEngine = new CursorSendEngine(
1559-
slotPath, actualSfMaxBytes,
1559+
slotPath, actualSfMaxSegmentBytes,
15601560
actualSfMaxTotalBytes, actualSfAppendDeadlineNanos,
15611561
actualSfSyncIntervalNanos);
15621562
int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY
@@ -1638,7 +1638,7 @@ public Sender build() {
16381638
connected.startOrphanDrainers(
16391639
orphans,
16401640
maxBackgroundDrainers,
1641-
actualSfMaxBytes,
1641+
actualSfMaxSegmentBytes,
16421642
actualSfMaxTotalBytes,
16431643
actualSfSyncIntervalNanos);
16441644
}
@@ -2770,19 +2770,21 @@ public LineSenderBuilder storeAndForwardDurability(SfDurability durability) {
27702770
}
27712771

27722772
/**
2773-
* Maximum bytes per segment file before rotation. Defaults to
2774-
* {@code DEFAULT_SEGMENT_BYTES}
2775-
* (4 MiB). Smaller segments mean faster trim of acked data; larger
2776-
* segments mean fewer rotations.
2773+
* Maximum bytes per segment file before rotation, the builder form of
2774+
* the {@code sf_max_segment_bytes} connect-string key. Smaller segments
2775+
* mean faster trim of acked data; larger segments mean fewer rotations.
2776+
* Default: {@code 4 MiB}. WebSocket transport only.
2777+
*
2778+
* @param maxSegmentBytes per-segment cap in bytes; must be positive
27772779
*/
2778-
public LineSenderBuilder storeAndForwardMaxBytes(long maxBytes) {
2780+
public LineSenderBuilder storeAndForwardMaxSegmentBytes(long maxSegmentBytes) {
27792781
if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) {
27802782
throw new LineSenderException("store_and_forward is only supported for WebSocket transport");
27812783
}
2782-
if (maxBytes <= 0) {
2783-
throw new LineSenderException("sf_max_bytes must be positive: ").put(maxBytes);
2784+
if (maxSegmentBytes <= 0) {
2785+
throw new LineSenderException("sf_max_segment_bytes must be positive: ").put(maxSegmentBytes);
27842786
}
2785-
this.sfMaxBytes = maxBytes;
2787+
this.sfMaxSegmentBytes = maxSegmentBytes;
27862788
return this;
27872789
}
27882790

@@ -3387,12 +3389,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) {
33873389
}
33883390
pos = getValue(configurationString, pos, sink, "sender_id");
33893391
senderId(sink.toString());
3390-
} else if (Chars.equals("sf_max_bytes", sink)) {
3392+
} else if (Chars.equals("sf_max_segment_bytes", sink)) {
33913393
if (protocol != PROTOCOL_WEBSOCKET) {
3392-
throw new LineSenderException("sf_max_bytes is only supported for WebSocket transport");
3394+
throw new LineSenderException("sf_max_segment_bytes is only supported for WebSocket transport");
33933395
}
3394-
pos = getValue(configurationString, pos, sink, "sf_max_bytes");
3395-
storeAndForwardMaxBytes(parseSizeValue(sink, "sf_max_bytes"));
3396+
pos = getValue(configurationString, pos, sink, "sf_max_segment_bytes");
3397+
storeAndForwardMaxSegmentBytes(parseSizeValue(sink, "sf_max_segment_bytes"));
33963398
} else if (Chars.equals("sf_max_total_bytes", sink)) {
33973399
if (protocol != PROTOCOL_WEBSOCKET) {
33983400
throw new LineSenderException("sf_max_total_bytes is only supported for WebSocket transport");
@@ -3730,8 +3732,8 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString)
37303732
if (view.has("sf_append_deadline_millis")) {
37313733
sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis"));
37323734
}
3733-
if (view.has("sf_max_bytes")) {
3734-
storeAndForwardMaxBytes(wsSize(view, v, "sf_max_bytes"));
3735+
if (view.has("sf_max_segment_bytes")) {
3736+
storeAndForwardMaxSegmentBytes(wsSize(view, v, "sf_max_segment_bytes"));
37353737
}
37363738
if (view.has("sf_max_total_bytes")) {
37373739
storeAndForwardMaxTotalBytes(wsSize(view, v, "sf_max_total_bytes"));
@@ -3892,7 +3894,7 @@ public java.util.Map<String, Object> wsConfigSnapshotForTest() {
38923894
m.put("request_durable_ack", requestDurableAck);
38933895
m.put("sender_id", senderId);
38943896
m.put("sf_dir", sfDir);
3895-
m.put("sf_max_bytes", sfMaxBytes);
3897+
m.put("sf_max_segment_bytes", sfMaxSegmentBytes);
38963898
m.put("sf_max_total_bytes", sfMaxTotalBytes);
38973899
m.put("sf_durability", sfDurability == null ? null : sfDurability.name());
38983900
m.put("sf_append_deadline_millis", sfAppendDeadlineMillis);

core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
183183
// drained from the head every time a STATUS_DURABLE_ACK frame advances
184184
// any watermark; an entry pops when every (name, seqTxn) it carries is
185185
// covered by durableTableWatermarks. Bounded in practice by the SF on-disk
186-
// cap: once the producer hits sf_max_bytes it blocks, which caps how far
186+
// cap: once the producer hits sf_max_segment_bytes it blocks, which caps how far
187187
// the durable watermark can lag behind the OK watermark.
188188
private final ArrayDeque<PendingDurableEntry> pendingDurable = new ArrayDeque<>();
189189
private final ArrayDeque<PendingDurableEntry> pendingDurablePool = new ArrayDeque<>();

core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long
219219
// the JVM).
220220
if (!ff.allocate(fd, sizeBytes)) {
221221
ff.close(fd);
222-
// Unlink the partially-created file so a sf_max_bytes-sized
222+
// Unlink the partially-created file so a sf_max_segment_bytes-sized
223223
// empty file does not survive the failure. Under sustained
224224
// disk-full pressure with the manager polling, hundreds would
225225
// otherwise accumulate.

core/src/main/java/io/questdb/client/impl/ConfigSchema.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public final class ConfigSchema {
8686
str("sf_append_deadline_millis", Side.INGRESS);
8787
str("sf_dir", Side.INGRESS);
8888
str("sf_durability", Side.INGRESS);
89-
str("sf_max_bytes", Side.INGRESS);
89+
str("sf_max_segment_bytes", Side.INGRESS);
9090
str("sf_max_total_bytes", Side.INGRESS);
9191
str("sf_sync_interval_millis", Side.INGRESS);
9292
str("transaction", Side.INGRESS);

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
/**
3939
* Tests for WebSocket transport support in the Sender.builder() API.
4040
* These tests verify the builder configuration and validation,
41-
* not actual WebSocket connectivity (which requires a running server).
41+
* not actual WebSocket connectivity (that requires a running server).
4242
*/
4343
public class LineSenderBuilderWebSocketTest extends AbstractTest {
4444

@@ -634,6 +634,39 @@ public void testRetryTimeout_notSupportedForWebSocket() {
634634
"not supported for WebSocket");
635635
}
636636

637+
@Test
638+
public void testStoreAndForwardMaxSegmentBytes() {
639+
Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET);
640+
641+
Assert.assertSame(builder, builder.storeAndForwardMaxSegmentBytes(64 * 1024L));
642+
Assert.assertEquals(
643+
64 * 1024L,
644+
((Number) builder.wsConfigSnapshotForTest().get("sf_max_segment_bytes")).longValue()
645+
);
646+
}
647+
648+
@Test
649+
public void testStoreAndForwardMaxSegmentBytesRejectsNonPositiveValues() {
650+
long[] rejected = {0L, -1L};
651+
for (long value : rejected) {
652+
assertThrows("sf_max_segment_bytes must be positive: " + value,
653+
() -> Sender.builder(Sender.Transport.WEBSOCKET).storeAndForwardMaxSegmentBytes(value));
654+
}
655+
}
656+
657+
@Test
658+
public void testStoreAndForwardMaxSegmentBytesRejectedForNonWebSocketTransports() {
659+
Sender.Transport[] rejected = {
660+
Sender.Transport.HTTP,
661+
Sender.Transport.TCP,
662+
Sender.Transport.UDP
663+
};
664+
for (Sender.Transport transport : rejected) {
665+
assertThrows("store_and_forward is only supported for WebSocket transport",
666+
() -> Sender.builder(transport).storeAndForwardMaxSegmentBytes(64 * 1024L));
667+
}
668+
}
669+
637670
@Test
638671
public void testSyncModeAutoFlushDefaults() throws Exception {
639672
// Regression test: connect() must not hardcode autoFlush to 0.
@@ -809,6 +842,15 @@ public void testWsConfigString_withPath_fails() {
809842
"unknown configuration key: path");
810843
}
811844

845+
@Test
846+
public void testWsConfigString_withSfMaxBytes_fails() {
847+
// sf_max_bytes is the spelling this key carries in the 1.2.1-1.3.5 jars.
848+
// sf_max_segment_bytes replaces it with no alias, so the old spelling
849+
// rejects as an unknown key rather than silently configuring nothing.
850+
assertBadConfig("ws::addr=localhost:9000;sf_max_bytes=4096;",
851+
"unknown configuration key: sf_max_bytes");
852+
}
853+
812854
@Test
813855
public void testWsConfigString_withEgressOnlyKeysSilentlyAccepted() {
814856
// connect-string.md "Query client keys" and "Multi-host failover": these

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -720,7 +720,7 @@ public void testIngressOnlyKeysSilentlyAcceptedOnEgress() {
720720
"sf_append_deadline_millis=30000",
721721
"sf_dir=/var/lib/qdb-sf",
722722
"sf_durability=memory",
723-
"sf_max_bytes=4m",
723+
"sf_max_segment_bytes=4m",
724724
"sf_max_total_bytes=10g",
725725
"transaction=on",
726726
};

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void testRestartReplaysSealedSegmentsAgainstFreshServer() throws Exceptio
8686
String pad = repeat("x", 64);
8787
String cfg1 = "ws::addr=localhost:" + port1
8888
+ ";sf_dir=" + sfDir
89-
+ ";sf_max_bytes=4096"
89+
+ ";sf_max_segment_bytes=4096"
9090
+ ";close_flush_timeout_millis=0;";
9191
try (Sender s1 = Sender.fromConfig(cfg1)) {
9292
for (int i = 0; i < 50; i++) {

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public void testSfDirOnTcpRejected() throws Exception {
9393
}
9494

9595
@Test
96-
public void testSfMaxBytesParsing() throws Exception {
96+
public void testSfMaxSegmentBytesParsing() throws Exception {
9797
TestUtils.assertMemoryLeak(() -> {
9898
AckHandler handler = new AckHandler();
9999
try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
@@ -102,7 +102,7 @@ public void testSfMaxBytesParsing() throws Exception {
102102

103103
int port = server.getPort();
104104
String config = "ws::addr=localhost:" + port
105-
+ ";sf_dir=" + sfDir + ";sf_max_bytes=131072;";
105+
+ ";sf_dir=" + sfDir + ";sf_max_segment_bytes=131072;";
106106
try (Sender sender = Sender.fromConfig(config)) {
107107
// Write enough data that segments rotate at ~128 KiB boundary.
108108
for (int i = 0; i < 50; i++) {
@@ -140,7 +140,7 @@ public void testNoSfDirMeansNoSf() throws Exception {
140140
}
141141

142142
/**
143-
* Regression test for the connect-string {@code sf_max_bytes} /
143+
* Regression test for the connect-string {@code sf_max_segment_bytes} /
144144
* {@code sf_max_total_bytes} parser accepting values larger than
145145
* {@code Integer.MAX_VALUE}. The pre-cursor parser used parseInt which
146146
* artificially capped the SF size from the connect string at ~2 GiB.
@@ -349,7 +349,7 @@ public void testSfSyncIntervalRejectsOnNonWebSocketTransport() throws Exception
349349
}
350350

351351
@Test
352-
public void testSfMaxBytesAcceptsSizeSuffixes() throws Exception {
352+
public void testSfMaxSegmentBytesAcceptsSizeSuffixes() throws Exception {
353353
TestUtils.assertMemoryLeak(() -> {
354354
AckHandler handler = new AckHandler();
355355
try (TestWebSocketServer server = new TestWebSocketServer(handler)) {
@@ -360,7 +360,7 @@ public void testSfMaxBytesAcceptsSizeSuffixes() throws Exception {
360360
// 64m / 4g should parse identically to their byte-count equivalents.
361361
String config = "ws::addr=localhost:" + port
362362
+ ";sf_dir=" + sfDir
363-
+ ";sf_max_bytes=64m"
363+
+ ";sf_max_segment_bytes=64m"
364364
+ ";sf_max_total_bytes=4g;";
365365
try (Sender sender = Sender.fromConfig(config)) {
366366
sender.table("foo").longColumn("v", 1L).atNow();
@@ -473,14 +473,14 @@ public void testSenderIdInvalidCharRejected() throws Exception {
473473
}
474474

475475
@Test
476-
public void testSfMaxBytesInvalidSizeSuffixRejected() throws Exception {
476+
public void testSfMaxSegmentBytesInvalidSizeSuffixRejected() throws Exception {
477477
TestUtils.assertMemoryLeak(() -> {
478-
String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";sf_max_bytes=64x;";
478+
String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";sf_max_segment_bytes=64x;";
479479
try (Sender ignored = Sender.fromConfig(config)) {
480480
Assert.fail("expected rejection of unknown unit suffix");
481481
} catch (LineSenderException expected) {
482482
Assert.assertTrue(expected.getMessage(),
483-
expected.getMessage().contains("invalid sf_max_bytes"));
483+
expected.getMessage().contains("invalid sf_max_segment_bytes"));
484484
}
485485
});
486486
}

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1004,7 +1004,7 @@ public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Ex
10041004

10051005
/**
10061006
* Open-time sort regression: at the documented {@code sf_max_total_bytes
1007-
* / sf_max_bytes} ceiling (~16K segments), an O(N²) sort over the
1007+
* / sf_max_segment_bytes} ceiling (~16K segments), an O(N²) sort over the
10081008
* recovered segments delays the I/O thread. This test guards the sort
10091009
* with a deterministic comparison count.
10101010
* <p>

core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public void testEveryIngressKeyIsHonored() {
6161
assertHonored("request_durable_ack=on", "request_durable_ack", true);
6262
assertHonored("sender_id=probe-1", "sender_id", "probe-1");
6363
assertHonored("sf_dir=/var/probe", "sf_dir", "/var/probe");
64-
assertHonored("sf_max_bytes=4096", "sf_max_bytes", 4096L);
64+
assertHonored("sf_max_segment_bytes=4096", "sf_max_segment_bytes", 4096L);
6565
assertHonored("sf_max_total_bytes=8192", "sf_max_total_bytes", 8192L);
6666
assertHonored("sf_durability=periodic", "sf_durability", "PERIODIC");
6767
assertHonored("sf_append_deadline_millis=1500", "sf_append_deadline_millis", 1500L);

0 commit comments

Comments
 (0)