Skip to content

Commit 84f292a

Browse files
committed
fix(table): validate modified-UTF-8 continuation bytes in split decoder
read_java_utf now rejects lead bytes whose 2-/3-byte forms are not followed by proper 10xxxxxx continuation bytes, matching the sibling decoder in pk_vector_source. The legitimate 0xC0 0x80 encoding of NUL still decodes to '\0'. Also add Split.deserialize to the Python stub.
1 parent ce2eae0 commit 84f292a

1 file changed

Lines changed: 22 additions & 5 deletions

File tree

crates/paimon/src/table/source.rs

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1010,15 +1010,23 @@ fn read_java_utf(cur: &mut &[u8]) -> crate::Result<String> {
10101010
while i < bytes.len() {
10111011
let b = bytes[i];
10121012
let (unit, adv) = if b & 0x80 == 0 {
1013+
// 1-byte: 0xxxxxxx (includes raw 0x00)
10131014
(b as u16, 1)
10141015
} else if b & 0xE0 == 0xC0 {
1015-
let b2 = *bytes.get(i + 1).ok_or_else(utf_err)?;
1016-
((((b & 0x1F) as u16) << 6) | (b2 & 0x3F) as u16, 2)
1016+
// 2-byte: 110xxxxx 10xxxxxx (0xC0 0x80 is Java's encoding of '\0')
1017+
if i + 1 >= bytes.len() || bytes[i + 1] & 0xC0 != 0x80 {
1018+
return Err(utf_err());
1019+
}
1020+
((((b & 0x1F) as u16) << 6) | (bytes[i + 1] & 0x3F) as u16, 2)
10171021
} else if b & 0xF0 == 0xE0 {
1018-
let b2 = *bytes.get(i + 1).ok_or_else(utf_err)?;
1019-
let b3 = *bytes.get(i + 2).ok_or_else(utf_err)?;
1022+
// 3-byte: 1110xxxx 10xxxxxx 10xxxxxx
1023+
if i + 2 >= bytes.len() || bytes[i + 1] & 0xC0 != 0x80 || bytes[i + 2] & 0xC0 != 0x80 {
1024+
return Err(utf_err());
1025+
}
10201026
(
1021-
(((b & 0x0F) as u16) << 12) | (((b2 & 0x3F) as u16) << 6) | (b3 & 0x3F) as u16,
1027+
(((b & 0x0F) as u16) << 12)
1028+
| (((bytes[i + 1] & 0x3F) as u16) << 6)
1029+
| (bytes[i + 2] & 0x3F) as u16,
10221030
3,
10231031
)
10241032
} else {
@@ -1687,6 +1695,15 @@ mod tests {
16871695
assert!(read_java_utf(&mut cur).is_err());
16881696
}
16891697

1698+
#[test]
1699+
fn java_utf_rejects_malformed_continuation() {
1700+
// len=2, lead 0xC0 (2-byte form) followed by 0x41 ('A'), which is NOT a
1701+
// 10xxxxxx continuation byte. Must error, not silently decode to "\u{1}".
1702+
let bytes = [0x00u8, 0x02, 0xC0, 0x41];
1703+
let mut cur = bytes.as_slice();
1704+
assert!(read_java_utf(&mut cur).is_err());
1705+
}
1706+
16901707
#[test]
16911708
fn deletion_list_round_trips() {
16921709
for list in [

0 commit comments

Comments
 (0)