Fix WebSocket mask key byte order

WebSocketFrameWriter.writeHeader() wrote the mask key via
Unsafe.putInt() in native byte order. maskPayload() also
extracted mask bytes in native order. RFC 6455 specifies
network (big-endian) byte order for the mask key on the wire.

writeHeader() now writes the 4 mask key bytes individually in
big-endian order. maskPayload() converts to native order for
bulk XOR and extracts per-byte mask in big-endian order.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: mtopolnik

core/src/main/java/io/questdb/client/cutlass/qwp/websocket/WebSocketFrameWriter.java

@@ -26,6 +26,8 @@
 
 import io.questdb.client.std.Unsafe;
 
+import java.nio.ByteOrder;
+
 /**
  * Zero-allocation WebSocket frame writer.
  * Writes WebSocket frames according to RFC 6455.
@@ -37,6 +39,7 @@
 public final class WebSocketFrameWriter {
     // Frame header bits
     private static final int FIN_BIT = 0x80;
+    private static final boolean IS_BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN;
     private static final int MASK_BIT = 0x80;
 
     private WebSocketFrameWriter() {
@@ -70,9 +73,13 @@ public static int headerSize(long payloadLength, boolean masked) {
      * @param maskKey the 4-byte mask key
      */
     public static void maskPayload(long buf, long len, int maskKey) {
-        // Process 8 bytes at a time when possible
+        // maskKey is in big-endian convention: MSB = wire byte 0 = mask byte for position 0.
+        // For bulk XOR via getInt/getLong (native byte order), convert to native order
+        // so that memory position 0 XORs with mask byte 0, position 1 with mask byte 1, etc.
+        int nativeMask = IS_BIG_ENDIAN ? maskKey : Integer.reverseBytes(maskKey);
+        long longMask = ((long) nativeMask << 32) | (nativeMask & 0xFFFFFFFFL);
+
         long i = 0;
-        long longMask = ((long) maskKey << 32) | (maskKey & 0xFFFFFFFFL);
 
         // Process 8-byte chunks
         while (i + 8 <= len) {
@@ -84,14 +91,14 @@ public static void maskPayload(long buf, long len, int maskKey) {
         // Process 4-byte chunk if remaining
         if (i + 4 <= len) {
             int value = Unsafe.getUnsafe().getInt(buf + i);
-            Unsafe.getUnsafe().putInt(buf + i, value ^ maskKey);
+            Unsafe.getUnsafe().putInt(buf + i, value ^ nativeMask);
             i += 4;
         }
 
-        // Process remaining bytes (0-3 bytes) - extract mask byte inline to avoid allocation
+        // Process remaining bytes - extract mask byte in big-endian order
         while (i < len) {
             byte b = Unsafe.getUnsafe().getByte(buf + i);
-            int maskByte = (maskKey >> (((int) i & 3) << 3)) & 0xFF;
+            int maskByte = (maskKey >>> ((3 - ((int) i & 3)) << 3)) & 0xFF;
             Unsafe.getUnsafe().putByte(buf + i, (byte) (b ^ maskByte));
             i++;
         }
@@ -144,7 +151,11 @@ public static int writeHeader(long buf, boolean fin, int opcode, long payloadLen
      */
     public static int writeHeader(long buf, boolean fin, int opcode, long payloadLength, int maskKey) {
         int offset = writeHeader(buf, fin, opcode, payloadLength, true);
-        Unsafe.getUnsafe().putInt(buf + offset, maskKey);
+        // Write mask key in network byte order (big-endian) per RFC 6455
+        Unsafe.getUnsafe().putByte(buf + offset, (byte) (maskKey >>> 24));
+        Unsafe.getUnsafe().putByte(buf + offset + 1, (byte) (maskKey >>> 16));
+        Unsafe.getUnsafe().putByte(buf + offset + 2, (byte) (maskKey >>> 8));
+        Unsafe.getUnsafe().putByte(buf + offset + 3, (byte) maskKey);
         return offset + 4;
     }
 }