JSSE: extract peek logic to internal function. Adds flag to allow

skipping TLS record peek if continuing with the same TLS record. Update
bufferUnderflow guards.

Author: rlm2002

src/java/com/wolfssl/provider/jsse/WolfSSLEngine.java

@@ -110,6 +110,7 @@ public class WolfSSLEngine extends SSLEngine {
 
     /* session stored (WOLFSSL_SESSION), relevant on client side */
     private boolean sessionStored = false;
+    private boolean contPartialRecord = false;
 
     /* TLS 1.3 session ticket received (on client side) */
     private boolean sessionTicketReceived = false;
@@ -1289,6 +1290,47 @@ private byte[] getRecvAppDataBuf(int minSz) {
         return this.recvAppDataBuf;
     }
 
+    /**
+     * TLS-only: peek at the record header and
+     * return BUFFER_UNDERFLOW before calling into
+     * JNI when the full record is not yet present.
+     * Without this, native wolfSSL consumes partial
+     * record bytes via the I/O callback, violating
+     * the JSSE contract that BUFFER_UNDERFLOW must
+     * report bytesConsumed() == 0. DTLS still
+     * relies on the native WANT_READ path.
+     *
+     * @param in input buffer
+     * @param inRemaining bytes remaining in input buffer.
+     * @param handshaking whether the function is being called during
+     *                    handshake.
+     *
+     * @return true if buffer underflow detected, false otherwise
+     */
+    private boolean peekTlsRecordHeader(ByteBuffer in, int inRemaining) {
+        boolean bufferUnderflow = false;
+        if (inRemaining > 0 && (this.ssl.dtls() == 0)) {
+            int pos = in.position();
+            if (inRemaining < TLS_RECORD_HEADER_LEN) {
+                /* Not enough for TLS record header */
+                bufferUnderflow = true;
+            } else {
+                /* Peek at record length from header
+                    * bytes 3-4 (big-endian) */
+                int recLen =
+                    ((in.get(pos + TLS_RECORD_LEN_HI_OFF)
+                        & 0xFF) << 8) |
+                    (in.get(pos + TLS_RECORD_LEN_LO_OFF)
+                        & 0xFF);
+                if (recLen <= WolfSSL.MAX_RECORD_SIZE &&
+                    inRemaining < TLS_RECORD_HEADER_LEN + recLen) {
+                    bufferUnderflow = true;
+                }
+            }
+        }
+        return bufferUnderflow;
+    }
+
     @Override
     public synchronized SSLEngineResult unwrap(ByteBuffer in, ByteBuffer out)
             throws SSLException {
@@ -1313,6 +1355,7 @@ public synchronized SSLEngineResult unwrap(ByteBuffer in, ByteBuffer[] out,
         long dtlsPrevDropCount = 0;
         long dtlsCurrDropCount = 0;
         int prevSessionTicketCount = 0;
+        boolean bufferUnderflow = false;
         final int tmpRet;
 
         /* Set initial status for SSLEngineResult return */
@@ -1463,38 +1506,22 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
                     if (this.handshakeFinished == false) {
                         WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
                             () -> "starting or continuing handshake");
-                        ret = DoHandshake(false);
+                        if (!this.contPartialRecord) {
+                            bufferUnderflow =
+                                this.getUseClientMode() &&
+                                peekTlsRecordHeader(in, inRemaining);
+                        }
+                        if (bufferUnderflow) {
+                            status = SSLEngineResult.Status.BUFFER_UNDERFLOW;
+                        }
+                        else {
+                            ret = DoHandshake(false);
+                        }
                     }
                     else {
-                        /* TLS-only: peek at the record header and
-                         * return BUFFER_UNDERFLOW before calling into
-                         * JNI when the full record is not yet present.
-                         * Without this, native wolfSSL consumes partial
-                         * record bytes via the I/O callback, violating
-                         * the JSSE contract that BUFFER_UNDERFLOW must
-                         * report bytesConsumed() == 0. DTLS still
-                         * relies on the native WANT_READ path. */
-                        boolean bufferUnderflow = false;
-                        if (inRemaining > 0 && (this.ssl.dtls() == 0)) {
-                            synchronized (netDataLock) {
-                                int pos = in.position();
-                                if (inRemaining < TLS_RECORD_HEADER_LEN) {
-                                    /* Not enough for TLS record header */
-                                    bufferUnderflow = true;
-                                } else {
-                                    /* Peek at record length from header
-                                     * bytes 3-4 (big-endian) */
-                                    int recLen =
-                                        ((in.get(pos + TLS_RECORD_LEN_HI_OFF)
-                                            & 0xFF) << 8) |
-                                        (in.get(pos + TLS_RECORD_LEN_LO_OFF)
-                                            & 0xFF);
-                                    if (inRemaining <
-                                        TLS_RECORD_HEADER_LEN + recLen) {
-                                        bufferUnderflow = true;
-                                    }
-                                }
-                            }
+                        if (!this.contPartialRecord) {
+                            bufferUnderflow =
+                                    peekTlsRecordHeader(in, inRemaining);
                         }
 
                         /* Serve stashed data from previous
@@ -1629,8 +1656,8 @@ else if (inRemaining > 0 &&
                         }
                     } /* end DoHandshake() / RecvAppData() */
 
-                    if (outBoundOpen == false || this.closeNotifySent ||
-                        this.closeNotifyReceived) {
+                    if ((outBoundOpen == false || this.closeNotifySent ||
+                        this.closeNotifyReceived) && !bufferUnderflow) {
                         /* Mark SSLEngine status as CLOSED */
                         status = SSLEngineResult.Status.CLOSED;
                         /* Handshake has finished and SSLEngine is closed,
@@ -1697,7 +1724,8 @@ else if (ret < 0 &&
                         }
                     }
                     else if (!this.handshakeFinished && (ret == 0) &&
-                        (err == 0 || err == WolfSSL.SSL_ERROR_ZERO_RETURN)) {
+                        (err == 0 || err == WolfSSL.SSL_ERROR_ZERO_RETURN)
+                         && !bufferUnderflow) {
 
                         boolean gotCloseNotify = false;
                         synchronized (ioLock) {
@@ -2650,6 +2678,16 @@ protected synchronized int internalRecvCb(ByteBuffer toRead, int sz) {
                 if (this.ssl.dtls() == 1 && this.handshakeFinished) {
                     this.nativeWantsToRead = 1;
                 }
+                /* wolfSSL asked for bytes but we have none remaining
+                 * (netData is valid but empty). Mark as mid-record so
+                 * the next unwrap() skips the TLS header peek and
+                 * passes the continuation bytes directly to DoHandshake.
+                 * Only set when netData is non-null (netData==null means
+                 * we are inside wrap(), not a mid-record scenario).
+                 * Prevents stale state from being used. */
+                if (this.netData != null) {
+                    this.contPartialRecord = true;
+                }
                 return WolfSSL.WOLFSSL_CBIO_ERR_WANT_READ;
             }
 
@@ -2676,6 +2714,8 @@ protected synchronized int internalRecvCb(ByteBuffer toRead, int sz) {
                 toRead.position(toReadPos);
             }
 
+            this.contPartialRecord = (max < sz);
+
             return max;
         }
     }