Skip to content

unwrap() header peek in SSLEngine to fix BUFFER_UNDERFLOW #352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cconlon merged 2 commits into from
Apr 17, 2026
Merged

unwrap() header peek in SSLEngine to fix BUFFER_UNDERFLOW #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a62bc09
JSSE: extract peek logic to internal function. Adds flag to allow
rlm2002 Apr 15, 2026
099a55a
update regression test to better test when inRemaining == 0 and add o…
rlm2002 Apr 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions native/com_wolfssl_WolfSSL.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions src/java/com/wolfssl/WolfSSL.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,23 @@ public enum TLS_VERSION {
/** Maximum SSL record size (16KB) as defined by the protocol. */
public static final int MAX_RECORD_SIZE = 16384;

/** TLS record header is: type(1) + version(2) + length(2) */
public static final int TLS_RECORD_HEADER_LEN = 5;
/** TLS record header length high byte offset */
public static final int TLS_RECORD_LEN_HI_OFF = 3;
/** TLS record header length lower byte offset */
public static final int TLS_RECORD_LEN_LO_OFF = 4;
/** TLS record header protocol version offset */
public static final int TLS_RECORD_VERS_OFF = 1;
/** TLS record header protocol version major */
public static final int TLS_RECORD_VERS_MAJOR = 0x03;
/** TLS record header valid content type
* (RFC 8446 B.1, RFC 6520), (change_cipher_spec) */
public static final int TLS_RECORD_CT_MIN = 20;
/** TLS record header valid content type
* (RFC 8446 B.1, RFC 6520), (heartbeat) */
public static final int TLS_RECORD_CT_MAX = 24;

/* ------------------ TLS extension specific ------------------------ */
/** SNI Host name type, for UseSNI() */
public static final int WOLFSSL_SNI_HOST_NAME = 0;
Expand Down
134 changes: 97 additions & 37 deletions src/java/com/wolfssl/provider/jsse/WolfSSLEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ public class WolfSSLEngine extends SSLEngine {

/* session stored (WOLFSSL_SESSION), relevant on client side */
private boolean sessionStored = false;
/* Skip record header peek in next unwrap, continue same TLS record */
private boolean contPartialRecord = false;

/* TLS 1.3 session ticket received (on client side) */
private boolean sessionTicketReceived = false;
Expand Down Expand Up @@ -142,11 +144,6 @@ public class WolfSSLEngine extends SSLEngine {
* and expanded only when a larger output window requires it. */
private byte[] recvAppDataBuf = new byte[WolfSSL.MAX_RECORD_SIZE];

/* TLS record header is: type(1) + version(2) + length(2) */
private static final int TLS_RECORD_HEADER_LEN = 5;
private static final int TLS_RECORD_LEN_HI_OFF = 3;
private static final int TLS_RECORD_LEN_LO_OFF = 4;

/* Default size of internalIOSendBuf, 16k to match TLS record size.
* TODO - add upper bound on I/O send buf resize allocations. */
private static final int INTERNAL_IOSEND_BUF_SZ = WolfSSL.MAX_RECORD_SIZE;
Expand Down Expand Up @@ -365,7 +362,7 @@ private List<SNIServerName> parseRequestedServerNamesFromNetData() {

synchronized (netDataLock) {
if (this.netData == null ||
this.netData.remaining() < TLS_RECORD_HEADER_LEN) {
this.netData.remaining() < WolfSSL.TLS_RECORD_HEADER_LEN) {
return null;
}
in = this.netData.asReadOnlyBuffer();
Expand Down Expand Up @@ -1289,6 +1286,70 @@ private byte[] getRecvAppDataBuf(int minSz) {
return this.recvAppDataBuf;
}

/**
* Peek at the record header and return BUFFER_UNDERFLOW before calling
* into JNI when the full record is not yet present. Without this, native
* wolfSSL consumes partial record bytes via the I/O callback, violating
* the JSSE contract that BUFFER_UNDERFLOW must report
* bytesConsumed() == 0.
*
* Skipped if using DTLS.
*
* @param in input buffer.
* @param inRemaining bytes remaining in input buffer.
*
* @return true if buffer underflow detected, false otherwise.
*/
private boolean peekTlsRecordHeader(ByteBuffer in, int inRemaining) {
boolean bufferUnderflow = false;
/* DTLS still relies on the native WANT_READ path. */
if (inRemaining > 0 && (this.ssl.dtls() == 0)) {
int pos = in.position();
if (inRemaining < WolfSSL.TLS_RECORD_HEADER_LEN) {
/* Not enough for TLS record header */
bufferUnderflow = true;
}
else {
/* Check if header is plausible. Content type between
* change_cipher_spec and heartbeat and
* version major corresponds to TLS. */
byte headerByte = in.get(pos);
if ((headerByte & 0xFF) >= WolfSSL.TLS_RECORD_CT_MIN &&
(headerByte & 0xFF) <= WolfSSL.TLS_RECORD_CT_MAX &&
(in.get(pos + WolfSSL.TLS_RECORD_VERS_OFF)
& 0xFF) == WolfSSL.TLS_RECORD_VERS_MAJOR) {
/* Peek at record length from header
* bytes 3-4 (big-endian). */
int recLen =
((in.get(pos + WolfSSL.TLS_RECORD_LEN_HI_OFF)
& 0xFF) << 8) |
(in.get(pos + WolfSSL.TLS_RECORD_LEN_LO_OFF)
& 0xFF);
/* Fall through if recLen exceeds max packet buffer size
* or inRemaining is less than reported record length */
int packetBufSz = this.getSession().getPacketBufferSize();
if (recLen <= packetBufSz &&
inRemaining < WolfSSL.TLS_RECORD_HEADER_LEN + recLen) {
bufferUnderflow = true;
}
else if (recLen > packetBufSz) {
WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
() -> "recLen: " + recLen +
" exceeds max packet buffer size, " +
"skipping underflow check.");
}
}
else {
WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
() -> "Peeked content-type or version " +
"major outside TLS range, skipping " +
"underflow check");
}
}
}
return bufferUnderflow;
}

@Override
public synchronized SSLEngineResult unwrap(ByteBuffer in, ByteBuffer out)
throws SSLException {
Expand All @@ -1313,6 +1374,7 @@ public synchronized SSLEngineResult unwrap(ByteBuffer in, ByteBuffer[] out,
long dtlsPrevDropCount = 0;
long dtlsCurrDropCount = 0;
int prevSessionTicketCount = 0;
boolean bufferUnderflow = false;
final int tmpRet;

/* Set initial status for SSLEngineResult return */
Expand Down Expand Up @@ -1463,38 +1525,26 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
if (this.handshakeFinished == false) {
WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
() -> "starting or continuing handshake");
ret = DoHandshake(false);
if (!this.contPartialRecord) {
/* Client only: TLS 1.3 ssl_accept() I/O callback
* patterns conflict with the pre-peek
* mid-handshake. Post-handshake server unwrap
* always applies peek. */
bufferUnderflow =
this.getUseClientMode() &&
Comment thread
rlm2002 marked this conversation as resolved.
peekTlsRecordHeader(in, inRemaining);
}
if (bufferUnderflow) {
status = SSLEngineResult.Status.BUFFER_UNDERFLOW;
}
else {
ret = DoHandshake(false);
}
}
else {
/* TLS-only: peek at the record header and
* return BUFFER_UNDERFLOW before calling into
* JNI when the full record is not yet present.
* Without this, native wolfSSL consumes partial
* record bytes via the I/O callback, violating
* the JSSE contract that BUFFER_UNDERFLOW must
* report bytesConsumed() == 0. DTLS still
* relies on the native WANT_READ path. */
boolean bufferUnderflow = false;
if (inRemaining > 0 && (this.ssl.dtls() == 0)) {
synchronized (netDataLock) {
int pos = in.position();
if (inRemaining < TLS_RECORD_HEADER_LEN) {
/* Not enough for TLS record header */
bufferUnderflow = true;
} else {
/* Peek at record length from header
* bytes 3-4 (big-endian) */
int recLen =
((in.get(pos + TLS_RECORD_LEN_HI_OFF)
& 0xFF) << 8) |
(in.get(pos + TLS_RECORD_LEN_LO_OFF)
& 0xFF);
if (inRemaining <
TLS_RECORD_HEADER_LEN + recLen) {
bufferUnderflow = true;
}
}
}
if (!this.contPartialRecord) {
bufferUnderflow =
peekTlsRecordHeader(in, inRemaining);
}

/* Serve stashed data from previous
Expand Down Expand Up @@ -1697,7 +1747,8 @@ else if (ret < 0 &&
}
}
else if (!this.handshakeFinished && (ret == 0) &&
(err == 0 || err == WolfSSL.SSL_ERROR_ZERO_RETURN)) {
(err == 0 || err == WolfSSL.SSL_ERROR_ZERO_RETURN)
&& !bufferUnderflow) {

boolean gotCloseNotify = false;
synchronized (ioLock) {
Expand Down Expand Up @@ -2650,6 +2701,11 @@ protected synchronized int internalRecvCb(ByteBuffer toRead, int sz) {
if (this.ssl.dtls() == 1 && this.handshakeFinished) {
this.nativeWantsToRead = 1;
}
/* Buffer exhausted mid-record: skip the header peek on the
* next unwrap() to pass continuation bytes to DoHandshake. */
if (this.netData != null && this.netData.remaining() == 0) {
this.contPartialRecord = true;
}
return WolfSSL.WOLFSSL_CBIO_ERR_WANT_READ;
}

Expand All @@ -2676,6 +2732,10 @@ protected synchronized int internalRecvCb(ByteBuffer toRead, int sz) {
toRead.position(toReadPos);
}

/* Reset to false when all requested bytes were delivered;
* set to true when the buffer was exhausted mid-record. */
this.contPartialRecord = (max < sz);

return max;
}
}
Expand Down
Loading
Loading