Report a malformed frame in one more case

(cherry picked from commit 6936879)

Author: michaelklishin

src/main/java/com/rabbitmq/client/impl/Frame.java

@@ -109,9 +109,9 @@ public static Frame readFrom(DataInputStream is, int maxPayloadSize) throws IOEx
 
         channel = is.readUnsignedShort();
         int payloadSize = is.readInt();
-        if (payloadSize >= maxPayloadSize) {
-            throw new IllegalStateException(format(
-                "Frame body is too large (%d), maximum configured size is %d. " +
+        if (payloadSize < 0 || payloadSize >= maxPayloadSize) {
+            throw new MalformedFrameException(format(
+                "Frame body size is invalid (%d), maximum configured size is %d. " +
                     "See ConnectionFactory#setMaxInboundMessageBodySize " +
                     "if you need to increase the limit.",
                 payloadSize, maxPayloadSize

src/main/java/com/rabbitmq/client/impl/NettyFrameHandlerFactory.java

@@ -469,10 +469,10 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         int type = m.readUnsignedByte();
         int channel = m.readUnsignedShort();
         int payloadSize = m.readInt();
-        if (payloadSize >= maxPayloadSize) {
-          throw new IllegalStateException(
+        if (payloadSize < 0 || payloadSize >= maxPayloadSize) {
+          throw new MalformedFrameException(
               format(
-                  "Frame body is too large (%d), maximum configured size is %d. "
+                  "Frame body size is invalid (%d), maximum configured size is %d. "
                       + "See ConnectionFactory#setMaxInboundMessageBodySize "
                       + "if you need to increase the limit.",
                   payloadSize, maxPayloadSize));