Increase data packet size check to 64KB in raw data packets (#948)

* feat: remove data packet size check

* fix: address spotless format

* fix: add changeset

* Separate target size and put back max data size at 64kb

* test

* Update changeset

* spotless

---------

Co-authored-by: davidliu <davidliu@deviange.net>

Author: 1egoman

.changeset/tricky-carpets-fall.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Increased max data packet size for `LocalParticipant.publishData` to 65535 bytes (64KB - 1)

livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt

@@ -771,6 +771,10 @@ internal constructor(
 
                 val packetBytes = dataPacket.toByteArray()
 
+                if (packetBytes.size > MAX_DATA_PACKET_SIZE) {
+                    return Result.failure(IllegalArgumentException("packet size (${packetBytes.size}) exceeds the max size (${MAX_DATA_PACKET_SIZE})"))
+                }
+
                 if (isReliable && this.connectionState == ConnectionState.RECONNECTING) {
                     reliableMessageBuffer.queue(DataPacketItem(ByteBuffer.wrap(packetBytes), dataPacket.sequence))
                     reliableDataSequence++
@@ -1049,7 +1053,17 @@ internal constructor(
          */
         @VisibleForTesting
         const val LOSSY_DATA_CHANNEL_LABEL = "_lossy"
-        internal const val MAX_DATA_PACKET_SIZE = 15 * 1024 // 15 KB
+        internal const val TARGET_DATA_PACKET_SIZE = 15 * 1024 // 15 KB
+
+        /**
+         * Corresponds to the max-message-size in SDP. Attempting to send packets
+         * over this size will cause the data channel to close, so this must be enforced
+         * within the SDK. Note that [DataChannel.send] will still report true
+         * even if a huge packet is sent, so it's not a usable signal.
+         *
+         * TODO: get max-message-size from SDP or equivalent from libwebrtc.
+         */
+        internal const val MAX_DATA_PACKET_SIZE = 64 * 1024 - 1 // 64 KB
         private const val MAX_RECONNECT_RETRIES = 30
         private const val MAX_RECONNECT_TIMEOUT = 60 * 1000
         private const val MAX_ICE_CONNECT_TIMEOUT_MS = 20000

livekit-android-sdk/src/main/java/io/livekit/android/room/datastream/outgoing/OutgoingDataStreamManager.kt

@@ -289,7 +289,7 @@ constructor(
             if (!isOpen) {
                 return Result.failure(StreamException.TerminatedException("Stream is closed!"))
             }
-            val chunks = chunker.invoke(data, RTCEngine.MAX_DATA_PACKET_SIZE)
+            val chunks = chunker.invoke(data, RTCEngine.TARGET_DATA_PACKET_SIZE)
 
             for (chunk in chunks) {
                 val result = sendChunk(streamId, chunk)

livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt

@@ -964,7 +964,7 @@ internal constructor(
 
     /**
      * Publish a new data payload to the room. Data will be forwarded to each participant in the room.
-     * Each payload must not exceed 15k in size
+     * Each payload must not exceed 65535 bytes (64KB - 1) in size.
      *
      * @param data payload to send
      * @param reliability for delivery guarantee, use RELIABLE. for fastest delivery without guarantee, use LOSSY
@@ -981,10 +981,6 @@ internal constructor(
         topic: String? = null,
         identities: List<Identity>? = null,
     ): Result<Unit> {
-        if (data.size > RTCEngine.MAX_DATA_PACKET_SIZE) {
-            return Result.failure(IllegalArgumentException("cannot publish data larger than " + RTCEngine.MAX_DATA_PACKET_SIZE))
-        }
-
         val kind = when (reliability) {
             DataPublishReliability.RELIABLE -> DataPacket.Kind.RELIABLE
             DataPublishReliability.LOSSY -> DataPacket.Kind.LOSSY
@@ -1108,7 +1104,7 @@ internal constructor(
         // one second to complete, even after accounting for round-trip latency.
         val minEffectiveTimeout = 1.seconds
 
-        if (payload.byteLength() > RTCEngine.MAX_DATA_PACKET_SIZE) {
+        if (payload.byteLength() > RpcError.MAX_V1_PAYLOAD_BYTES) {
             throw RpcError.BuiltinRpcError.REQUEST_PAYLOAD_TOO_LARGE.create()
         }
 
@@ -1209,8 +1205,8 @@ internal constructor(
         payload: String,
         responseTimeout: Duration = 10.seconds,
     ): Result<Unit> {
-        if (payload.byteLength() > RTCEngine.MAX_DATA_PACKET_SIZE) {
-            throw IllegalArgumentException("cannot publish data larger than " + RTCEngine.MAX_DATA_PACKET_SIZE)
+        if (payload.byteLength() > RpcError.MAX_V1_PAYLOAD_BYTES) {
+            return Result.failure(RpcError.BuiltinRpcError.REQUEST_PAYLOAD_TOO_LARGE.create())
         }
 
         val dataPacket = with(DataPacket.newBuilder()) {
@@ -1237,8 +1233,8 @@ internal constructor(
         payload: String?,
         error: RpcError?,
     ): Result<Unit> {
-        if (payload.byteLength() > RTCEngine.MAX_DATA_PACKET_SIZE) {
-            throw IllegalArgumentException("cannot publish data larger than " + RTCEngine.MAX_DATA_PACKET_SIZE)
+        if (payload.byteLength() > RpcError.MAX_V1_PAYLOAD_BYTES) {
+            return Result.failure(RpcError.BuiltinRpcError.RESPONSE_PAYLOAD_TOO_LARGE.create())
         }
 
         val dataPacket = with(DataPacket.newBuilder()) {
@@ -1359,7 +1355,7 @@ internal constructor(
                 ),
             )
 
-            if (response.byteLength() > RTCEngine.MAX_DATA_PACKET_SIZE) {
+            if (response.byteLength() > RpcError.MAX_V1_PAYLOAD_BYTES) {
                 responseError = RpcError.BuiltinRpcError.RESPONSE_PAYLOAD_TOO_LARGE.create()
                 LKLog.w { "RPC Response payload too large for $method" }
             } else {
@@ -1980,7 +1976,8 @@ private fun isBackupCodec(codecName: String) = backupCodecs.contains(codecName)
 
 /**
  * A handler that processes an RPC request and returns a string
- * that will be sent back to the requester.
+ * that will be sent back to the requester. The payload must
+ * be less than 15KB in size.
  *
  * Throwing an [RpcError] will send the error back to the requester.
  *

livekit-android-sdk/src/main/java/io/livekit/android/rpc/RpcError.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2025 LiveKit, Inc.
+ * Copyright 2025-2026 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 
 package io.livekit.android.rpc
 
-import io.livekit.android.room.RTCEngine
+import io.livekit.android.rpc.RpcError.Companion.MAX_V1_PAYLOAD_BYTES
 import io.livekit.android.util.truncateBytes
 import livekit.LivekitModels
 
@@ -57,11 +57,19 @@ data class RpcError(
         CONNECTION_TIMEOUT(1501, "Connection timeout"),
         RESPONSE_TIMEOUT(1502, "Response timeout"),
         RECIPIENT_DISCONNECTED(1503, "Recipient disconnected"),
+
+        /**
+         * @see [MAX_V1_PAYLOAD_BYTES]
+         */
         RESPONSE_PAYLOAD_TOO_LARGE(1504, "Response payload too large"),
         SEND_FAILED(1505, "Failed to send"),
 
         UNSUPPORTED_METHOD(1400, "Method not supported at destination"),
         RECIPIENT_NOT_FOUND(1401, "Recipient not found"),
+
+        /**
+         * @see [MAX_V1_PAYLOAD_BYTES]
+         */
         REQUEST_PAYLOAD_TOO_LARGE(1402, "Request payload too large"),
         UNSUPPORTED_SERVER(1403, "RPC not supported by server"),
         UNSUPPORTED_VERSION(1404, "Unsupported RPC version"),
@@ -75,11 +83,16 @@ data class RpcError(
     companion object {
         const val MAX_MESSAGE_BYTES = 256
 
+        /**
+         * The maximum payload size for v1 RPC requests and responses in bytes.
+         */
+        const val MAX_V1_PAYLOAD_BYTES = 15 * 1024 // 15KB
+
         fun fromProto(proto: LivekitModels.RpcError): RpcError {
             return RpcError(
                 code = proto.code,
                 message = (proto.message ?: "").truncateBytes(MAX_MESSAGE_BYTES),
-                data = proto.data.truncateBytes(RTCEngine.MAX_DATA_PACKET_SIZE),
+                data = proto.data.truncateBytes(MAX_V1_PAYLOAD_BYTES),
             )
         }
     }

livekit-android-test/src/test/java/io/livekit/android/room/RTCEngineMockE2ETest.kt

@@ -416,6 +416,23 @@ class RTCEngineMockE2ETest : MockE2ETest() {
         assertEquals(TestData.JOIN.join.participant.sid, sid)
     }
 
+    @Test
+    fun publishDataRejectsLargePacket() = runTest {
+        connect()
+        val pubDataChannel = getPublisherPeerConnection()
+            .dataChannels[RTCEngine.RELIABLE_DATA_CHANNEL_LABEL] as MockDataChannel
+
+        val oversizedPayload = ByteArray(65 * 1024) // See RTCEngine.MAX_DATA_PACKET_SIZE
+        val result = room.localParticipant.publishData(oversizedPayload)
+
+        assertTrue(result.isFailure)
+        assertTrue(
+            "Expected IllegalArgumentException, got ${result.exceptionOrNull()}",
+            result.exceptionOrNull() is IllegalArgumentException,
+        )
+        assertEquals(0, pubDataChannel.sentBuffers.size)
+    }
+
     @Test
     fun resendReliableMessagesReplaysFullPayloadAcrossMultipleResumes() = runTest {
         connect()