[fix] ensure buffer capacity on an overflow

Author: kares

src/main/java/org/jruby/ext/openssl/SSLSocket.java

@@ -589,6 +589,7 @@ private IRubyObject doHandshake(final boolean blocking, final boolean exception)
                 assert status != SSLEngineResult.Status.BUFFER_UNDERFLOW;
                 if (status == SSLEngineResult.Status.BUFFER_OVERFLOW) {
                     netWriteData.compact();
+                    netWriteData = Utils.ensureCapacity(netWriteData, engine.getSession().getPacketBufferSize());
                     netWriteData.flip();
                     if (handshakeStatus != SSLEngineResult.HandshakeStatus.NEED_UNWRAP || flushData(blocking)) {
                         sel = waitSelect(SelectionKey.OP_WRITE, blocking, exception);
@@ -602,7 +603,7 @@ private IRubyObject doHandshake(final boolean blocking, final boolean exception)
         }
     }
 
-    private void doWrap(boolean blocking) throws IOException {
+    private void doWrap(final boolean blocking) throws IOException {
         netWriteData.clear();
         SSLEngineResult result = engine.wrap(dummy, netWriteData);
         netWriteData.flip();
@@ -623,6 +624,11 @@ private void doTasks() {
         verifyResult = sslContext.getLastVerifyResult();
     }
 
+    /**
+     * @param blocking
+     * @return whether buffer has remaining data
+     * @throws IOException
+     */
     private boolean flushData(boolean blocking) throws IOException {
         try {
             writeToChannel(netWriteData, blocking);

src/main/java/org/jruby/ext/openssl/Utils.java

@@ -28,6 +28,7 @@
 package org.jruby.ext.openssl;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.HashSet;
 
 import org.jruby.*;
@@ -176,8 +177,12 @@ public void visit(IRubyObject key, IRubyObject value) {
         return ret;
     }
 
-    static IRubyObject extractKeywordArg(ThreadContext context, String keyword, RubyHash opts) {
-        return opts.op_aref(context, context.runtime.newSymbol(keyword));
+    static ByteBuffer ensureCapacity(final ByteBuffer buffer, final int size) {
+        if (size <= buffer.capacity()) return buffer;
+        buffer.flip();
+        ByteBuffer newBuffer = ByteBuffer.allocate(size);
+        newBuffer.put(buffer);
+        return newBuffer;
     }
 
 }// Utils