From 33099b5130b34d7f8cc227f78d968a858a35080a Mon Sep 17 00:00:00 2001
From: Wannaphong Phatthiyaphaibun <wannaphong@yahoo.com>
Date: Sun, 26 Nov 2023 19:24:10 +0700
Subject: [PATCH 1/2] Update match non-Thai tokens

---
 src/tokenizer/newmm.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tokenizer/newmm.rs b/src/tokenizer/newmm.rs
index 8d3f2213..ed5f1099 100644
--- a/src/tokenizer/newmm.rs
+++ b/src/tokenizer/newmm.rs
@@ -48,6 +48,7 @@ const NON_THAI_READABLE_PATTERN: &[&str; 5] = &[
     r"(?x)^[๐-๙]+([,\.][๐-๙]+)*",
     r"(?x)^[\ \t]+",
     r"(?x)^\r?\n",
+    r"(?x)^[\u0E00-\u0E7F  \t\r\n]",
 ];
 
 lazy_static! {

From 7ea79087ff32780a3e532c01ddd0295273ee1146 Mon Sep 17 00:00:00 2001
From: Wannaphong Phatthiyaphaibun <wannaphong@yahoo.com>
Date: Sun, 26 Nov 2023 19:30:40 +0700
Subject: [PATCH 2/2] Fixed size

---
 src/tokenizer/newmm.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tokenizer/newmm.rs b/src/tokenizer/newmm.rs
index ed5f1099..7909cc7c 100644
--- a/src/tokenizer/newmm.rs
+++ b/src/tokenizer/newmm.rs
@@ -42,7 +42,7 @@ const TEXT_SCAN_END: usize = TEXT_SCAN_POINT + TEXT_SCAN_RIGHT;
 
 type CharacterIndex = usize;
 
-const NON_THAI_READABLE_PATTERN: &[&str; 5] = &[
+const NON_THAI_READABLE_PATTERN: &[&str; 6] = &[
     r"(?x)^[-a-zA-Z]+",
     r"(?x)^[0-9]+([,\.][0-9]+)*",
     r"(?x)^[๐-๙]+([,\.][๐-๙]+)*",