fix(client): role-reject evidence outranks latched non-421 upgrade error in mixed connect sweep

In a mixed sweep -- e.g. [replica(421+role), replica(421+role),
node(503)] -- the exhausted-round epilogue threw the latched non-421
WebSocketUpgradeException before checking lastRoleReject, misclassifying
a transient failover/promotion window as terminal: dead foreground
sender, or a drainer slot quarantine via BackgroundDrainer markFailed.

Demote a plain non-421 WebSocketUpgradeException below role-reject
evidence: when any endpoint answered 421+role in the same sweep, surface
the retriable QwpRoleMismatchException so the connect/reconnect loops
keep rows in store-and-forward and retry through the promotion window.
The demoted upgrade error rides along as a suppressed diagnostic.

Typed capability gaps are NOT demoted: QwpVersionMismatchException and
QwpDurableAckMismatchException extend HttpClientException directly, so
they fall through the instanceof check and stay terminal even when
replicas role-rejected in the same sweep -- preserving the documented
durable-ack contract.

Covered by WriteFailoverTest#testMixedSweepRoleRejectOutranksLatchedTerminalUpgradeError,
which also asserts the suppressed 503 diagnostic.

Author: bluestreak01

core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java

@@ -2761,8 +2761,13 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
         // non-421 WebSocketUpgradeException (4xx/5xx). The catch block walks
         // remaining endpoints in case the failure is per-endpoint, then surfaces
         // this latched typed exception when the round ends without a successful
-        // connect. Auth failures are NOT latched here -- they throw immediately
-        // because a rejected credential is uniformly rejected across the cluster.
+        // connect -- except that a plain non-421 WebSocketUpgradeException is
+        // demoted below role-reject evidence in the epilogue: when any endpoint
+        // answered 421+role in the same sweep, the window is transient and the
+        // retriable role-mismatch classification wins (the demoted error rides
+        // along as a suppressed diagnostic). Auth failures are NOT latched here
+        // -- they throw immediately because a rejected credential is uniformly
+        // rejected across the cluster.
         HttpClientException terminalUpgradeError = null;
         QwpIngressRoleRejectedException lastRoleReject = null;
         Endpoint lastEndpoint = null;
@@ -2963,15 +2968,32 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
                     null, SenderConnectionEvent.NO_PORT,
                     SenderConnectionEvent.NO_ATTEMPT_NUMBER, roundSeq, lastError);
         }
-        if (terminalUpgradeError != null) {
+        // Role-reject evidence outranks a latched plain non-421 upgrade
+        // error. In a mixed sweep -- e.g. [replica(421+role),
+        // replica(421+role), node(503)] -- the co-occurring 421+role
+        // responses are positive evidence of a transient failover/promotion
+        // window; throwing the latched 5xx/4xx here would misclassify that
+        // window as terminal (dead foreground sender, or a drainer slot
+        // quarantine via BackgroundDrainer markFailed). Only plain
+        // WebSocketUpgradeException is demoted: the typed capability gaps
+        // (QwpVersionMismatchException, QwpDurableAckMismatchException)
+        // extend HttpClientException directly, so they fall through this
+        // check and stay terminal even when replicas role-rejected in the
+        // same sweep -- the contract the durable-ack paragraph below
+        // documents and relies on.
+        if (terminalUpgradeError != null
+                && !(lastRoleReject != null
+                && terminalUpgradeError instanceof WebSocketUpgradeException)) {
             throw terminalUpgradeError;
         }
         if (lastRoleReject != null) {
-            // Every endpoint role-rejected the /write/v4 upgrade: right now the
-            // reachable nodes are all replicas (or primary-catchup). That is a
-            // TRANSIENT failover window, not a terminal condition -- a replica
-            // can be promoted and a primary will reappear. Surface it as a
-            // retriable QwpRoleMismatchException so the SYNC/ASYNC connect and
+            // Every endpoint either role-rejected the /write/v4 upgrade or
+            // failed with a demoted non-421 upgrade error: right now the
+            // reachable, role-classified nodes are all replicas (or
+            // primary-catchup). That is a TRANSIENT failover window, not a
+            // terminal condition -- a replica can be promoted and a primary
+            // will reappear. Surface it as a retriable
+            // QwpRoleMismatchException so the SYNC/ASYNC connect and
             // reconnect loops keep the rows in store-and-forward and retry
             // within reconnect_max_duration_millis (for an SF sender the only
             // terminal condition is SF exhaustion).
@@ -2993,6 +3015,11 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) {
                             + "; last observed role=" + lastRoleReject.getRole()
                             + " at " + lastRoleReject.getHost() + ':' + lastRoleReject.getPort());
             ex.initCause(lastRoleReject);
+            if (terminalUpgradeError != null) {
+                // Keep the demoted non-421 upgrade error observable for
+                // diagnostics without changing the surfaced classification.
+                ex.addSuppressed(terminalUpgradeError);
+            }
             throw ex;
         }
         LineSenderException ex = new LineSenderException(lastError);

core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WriteFailoverTest.java

@@ -306,6 +306,84 @@ public void testRoleMismatchExceptionWhenAllReplicas() throws Exception {
         }
     }
 
+    @Test
+    public void testMixedSweepRoleRejectOutranksLatchedTerminalUpgradeError() throws Exception {
+        // Mixed connect sweep: [replica(421+role), replica(421+role),
+        // node(503)]. The co-occurring 421 role rejects are positive
+        // evidence of a transient failover/promotion window -- a replica
+        // can be promoted and a primary will reappear -- so the round
+        // epilogue must surface the retriable QwpRoleMismatchException,
+        // NOT the latched non-421 terminal upgrade error. Preferring the
+        // 503 turns a transient window into a dead sender (or a drainer
+        // slot quarantine on the background path).
+        AckHandler ack = new AckHandler();
+        TestWebSocketServer r1 = new TestWebSocketServer(ack);
+        int port1 = r1.getPort();
+        r1.setRejectWithRole("REPLICA");
+        TestWebSocketServer r2 = new TestWebSocketServer(ack);
+        int port2 = r2.getPort();
+        r2.setRejectWithRole("REPLICA");
+        TestWebSocketServer sick = new TestWebSocketServer(ack);
+        int port3 = sick.getPort();
+        sick.setRejectWithStatus(503, "Service Unavailable");
+        try {
+            r1.start();
+            r2.start();
+            sick.start();
+            Assert.assertTrue(r1.awaitStart(5, TimeUnit.SECONDS));
+            Assert.assertTrue(r2.awaitStart(5, TimeUnit.SECONDS));
+            Assert.assertTrue(sick.awaitStart(5, TimeUnit.SECONDS));
+
+            QwpRoleMismatchException observed = null;
+            // Off-mode walk sweeps every endpoint once and classifies the
+            // exhausted round at build(). Endpoint pick order does not
+            // matter: 421 role rejects and the 503 both `continue` the walk,
+            // so all three endpoints are always visited before the epilogue.
+            try (Sender ignored = Sender.builder(Sender.Transport.WEBSOCKET)
+                    .address("localhost:" + port1)
+                    .address("localhost:" + port2)
+                    .address("localhost:" + port3)
+                    .build()) {
+                Assert.fail("expected the mixed-sweep connect to fail");
+            } catch (QwpRoleMismatchException e) {
+                observed = e;
+            } catch (WebSocketUpgradeException e) {
+                throw new AssertionError(
+                        "mixed sweep misclassified as terminal: latched non-421 upgrade error "
+                                + "(status=" + e.getStatusCode()
+                                + ") outranked role-reject evidence: " + e.getMessage(), e);
+            } catch (LineSenderException e) {
+                throw new AssertionError(
+                        "expected QwpRoleMismatchException but got LineSenderException: "
+                                + e.getMessage(), e);
+            }
+            Assert.assertNotNull("expected a role-mismatch surface", observed);
+            Assert.assertEquals("PRIMARY", observed.getTargetRole());
+            String msg = observed.getMessage();
+            Assert.assertNotNull(msg);
+            Assert.assertTrue("error must mention the observed replica role: " + msg,
+                    msg.contains("REPLICA"));
+            // The demoted non-421 upgrade error must stay observable for
+            // diagnostics: it rides along as a suppressed exception on the
+            // surfaced role-mismatch classification.
+            WebSocketUpgradeException demoted = null;
+            for (Throwable s : observed.getSuppressed()) {
+                if (s instanceof WebSocketUpgradeException) {
+                    demoted = (WebSocketUpgradeException) s;
+                    break;
+                }
+            }
+            Assert.assertNotNull(
+                    "expected the demoted 503 upgrade error as a suppressed diagnostic",
+                    demoted);
+            Assert.assertEquals(503, demoted.getStatusCode());
+        } finally {
+            r1.close();
+            r2.close();
+            sick.close();
+        }
+    }
+
     @Test
     public void testStandaloneIsTreatedAsWritable() throws Exception {
         // OSS / single-node deployments advertise STANDALONE. The client