Skip to content

Commit 2986827

Browse files
committed
test(qwp): close M8 coverage gaps -- mutant-verified pins, down-then-up outage e2e, carried-over burn-downs
Delta-path gaps (each verified by applying the named mutant and watching exactly the new test fail): - testRoleRejectGrantsFreshWallClockToNextGapEpisode pins the WALL-CLOCK half of the role-reject episode reset; a counter-only-reset mutant previously survived all 18 retry tests. - testRequestStopInterruptsLongBackoffParkPromptly pins requestStop() promptness out of a 5s park; a no-unpark/monolithic-park mutant previously survived all 29 drainer tests. - BackgroundDrainerTransportOutageRecoveryTest conjoins the two Invariant B halves e2e: real ECONNREFUSED sweeps for 3x the settle budget, then the server returns on the SAME port (new TestWebSocketServer port-choosing ctor) and the drain completes -- no sentinel, no escalation. Carried-over gaps: - ConfigViewTest: getBool accepts true/on/false/off, rejects garbage and uppercase TRUE (case-sensitivity pinned). - JavaTlsClientSocketTest: NEED_TASK -> NOT_HANDSHAKING terminal delegated-task exit pinned (mutant-verified: clause removal busy-spins). - TlsProxy deleted: 248 lines, never instantiated since initial commit. - assertMemoryLeak sweep: 32 test methods wrapped across 12 test/impl files; ConfigViewTest/ConfStringParserTest/WsSenderConfigHonoredTest exempt (pure-JVM); no latent leaks surfaced (117/117 green).
1 parent 0623f9f commit 2986827

18 files changed

Lines changed: 1534 additions & 1161 deletions

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -736,6 +736,115 @@ public void testTransportWindowDoesNotBurnCapabilityGapWallClock() throws Except
736736
});
737737
}
738738

739+
@Test
740+
public void testRoleRejectGrantsFreshWallClockToNextGapEpisode() {
741+
// Companion to testRoleRejectResetsCapabilityGapEpisode, which pins the
742+
// ATTEMPT-counter half of the episode reset but runs under a 60s budget
743+
// where the wall-clock half is unobservable: a mutant that resets only
744+
// capabilityGapAttempts (leaving capabilityGapElapsedNanos /
745+
// lastCapabilityGapNanos ticking) passes it. This test pins the
746+
// WALL-CLOCK half: gap sweeps burn most of the budget, a role reject
747+
// proves the topology churned, and the next gap episode must start
748+
// from a zero wall clock -- under the counter-only mutant the stale
749+
// elapsed (plus the still-anchored lastCapabilityGapNanos charging
750+
// straight across the role-reject window) exhausts the budget and
751+
// quarantines a cluster that was about to settle.
752+
long budgetMillis = 800L;
753+
CountingListener listener = new CountingListener();
754+
AtomicInteger sweeps = new AtomicInteger();
755+
ScriptedFactory factory = ScriptedFactory.failingTimes(5, () -> {
756+
switch (sweeps.incrementAndGet()) {
757+
case 2:
758+
// Burn ~600ms of the 800ms budget inside the first gap
759+
// episode (charged by this sweep's gap-to-gap interval).
760+
sleepQuietly(600);
761+
return new QwpDurableAckMismatchException("h", 1234, "primary");
762+
case 3:
763+
// Topology churn: the settle budget must restart in full.
764+
return new QwpIngressRoleRejectedException(
765+
QwpIngressRoleRejectedException.ROLE_REPLICA, "127.0.0.1", 9000);
766+
case 5:
767+
// Second episode burns ~350ms -- well inside a fresh 800ms
768+
// budget, but 600 + 350 > 800 under the mutant's carried-over
769+
// wall clock.
770+
sleepQuietly(350);
771+
return new QwpDurableAckMismatchException("h", 1234, "primary");
772+
default:
773+
return new QwpDurableAckMismatchException("h", 1234, "primary");
774+
}
775+
});
776+
BackgroundDrainer drainer = newDrainerWithBudgets(
777+
factory, budgetMillis, FAST_BACKOFF_MILLIS, FAST_BACKOFF_MAX_MILLIS);
778+
drainer.setListener(listener);
779+
WebSocketClient out = drainer.connectWithDurableAckRetry();
780+
assertSame("role reject restarts the episode wall clock -- the second gap "
781+
+ "episode must get the full settle budget, not the first "
782+
+ "episode's leftovers",
783+
factory.successSentinel(), out);
784+
// gap, gap(+600ms), roleReject, gap, gap(+350ms), success = 6 sweeps.
785+
assertEquals(6, factory.attempts());
786+
assertEquals(BackgroundDrainer.DrainOutcome.PENDING, drainer.outcome());
787+
assertEquals("a settling cluster must never see a persistent-failure escalation",
788+
0, listener.persistentFailures.get());
789+
assertFalse("no .failed sentinel: both gap episodes stayed inside their budgets",
790+
Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
791+
// Per-mode attempt numbering across the reset: gaps 1,2 -- role reject
792+
// 1 -- fresh gap episode 1,2.
793+
assertEquals(java.util.Arrays.asList(1, 2, 1, 1, 2), listener.unavailableAttempts);
794+
}
795+
796+
@Test
797+
public void testRequestStopInterruptsLongBackoffParkPromptly() throws Exception {
798+
// Pins the stop-promptness contract of the backoff park: requestStop()
799+
// must break the drainer out of a LONG park (unpark, backstopped by
800+
// the 50ms STOP_CHECK_PARK_CHUNK_NANOS chunking) instead of sleeping
801+
// out the remainder. testStopRequestedDuringRetryAbortsWithStoppedOutcome
802+
// cannot see this: its 5-10ms backoffs complete faster than any
803+
// reasonable join timeout, so a monolithic park with no unpark passes
804+
// it. Here the backoff is 5s and the exit bound is 2s -- an
805+
// implementation that parks the full backoff in one shot fails.
806+
CountDownLatch firstFailureSeen = new CountDownLatch(1);
807+
ScriptedFactory factory = ScriptedFactory.alwaysFailing(() -> {
808+
firstFailureSeen.countDown();
809+
// Transport error: the un-clamped (boundedByBudget=false) sleep
810+
// path, so the park is backoff+jitter (5-10s), never trimmed to
811+
// the wall-clock budget.
812+
return new LineSenderException(
813+
"Failed to connect: all 2 endpoint(s) unreachable; last=127.0.0.1:9000");
814+
});
815+
BackgroundDrainer drainer = newDrainerWithBudgets(
816+
factory, /*reconnectMaxDurationMillis*/ 60_000L,
817+
/*backoffInit*/ 5_000L, /*backoffMax*/ 5_000L);
818+
Thread t = new Thread(drainer::connectWithDurableAckRetry, "long-park-stop-drainer");
819+
t.setDaemon(true);
820+
t.start();
821+
Assert.assertTrue("first failure must occur promptly",
822+
firstFailureSeen.await(2, TimeUnit.SECONDS));
823+
// Give the drainer a moment to enter the 5-10s park. If requestStop()
824+
// instead lands before the park, the pre-park stopRequested check
825+
// skips it entirely -- either way the exit must be prompt.
826+
Thread.sleep(100);
827+
long stopNanos = System.nanoTime();
828+
drainer.requestStop();
829+
t.join(2_000);
830+
long exitMillis = (System.nanoTime() - stopNanos) / 1_000_000L;
831+
Assert.assertFalse("requestStop() must break the drainer out of a 5-10s "
832+
+ "backoff park promptly (exit took >" + exitMillis + "ms); "
833+
+ "a monolithic park with no unpark sleeps out the full backoff",
834+
t.isAlive());
835+
assertEquals(BackgroundDrainer.DrainOutcome.STOPPED, drainer.outcome());
836+
assertFalse("stop is not a failure: no .failed sentinel",
837+
Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME));
838+
}
839+
840+
private static void sleepQuietly(long millis) {
841+
try {
842+
Thread.sleep(millis);
843+
} catch (InterruptedException ignored) {
844+
Thread.currentThread().interrupt();
845+
}
846+
}
847+
739848
private BackgroundDrainer newDrainer(ScriptedFactory factory) {
740849
return newDrainerWithBudgets(
741850
factory, FAST_RECONNECT_MAX_DURATION_MILLIS,

0 commit comments

Comments
 (0)