From b375c2fe618cecfe76f45d2e09a08f47aa91723f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:20:30 +0000 Subject: [PATCH 1/3] Initial plan From fd1885cf74b19ae8ebbb93868301048278df36ab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:48:47 +0000 Subject: [PATCH 2/3] fix: resolve newmm exponential BFS path explosion (issue #893) - Add visited set to _bfs_paths_graph to prevent re-exploring already-visited nodes, reducing worst-case BFS from O(2^n) to O(V+E) - Clear ambiguity graph dict after each commit point in _onecut to prevent unbounded accumulation - Remove unnecessary graph edge insertion in the no-candidate (elif) branch - Use Trie.prefixes(text, begin_pos) with start offset to avoid creating large string copies on every dictionary lookup call - Use re.Pattern.match(text, pos) to avoid string slices in regex matches - Add optional start parameter to Trie.prefixes() (backward-compatible) - Add test_newmm_ambiguous_performance regression test for issue #893 - Update segment() docstring and CHANGELOG.md Co-authored-by: bact <128572+bact@users.noreply.github.com> --- CHANGELOG.md | 11 +++++++++++ pythainlp/tokenize/newmm.py | 30 +++++++++++++++++++----------- pythainlp/util/trie.py | 18 +++++++++++------- tests/core/test_tokenize.py | 16 ++++++++++++++++ 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e82e09d6..3ffb65ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,17 @@ The minimum requirement is now Python 3.9. - `pythainlp.is_read_only_mode()` helper function; use `PYTHAINLP_READ_ONLY=1` to prevent all write operations +### Fixed + +- `newmm` tokenization engine: fix exponential-time BFS path explosion when + tokenizing text with many ambiguous breaking points (e.g., repeated words + like "ด้านหน้า" that can be split multiple ways). The internal + `_bfs_paths_graph` function now uses a visited set, reducing worst-case + complexity from exponential to O(V + E). The ambiguity graph is now also + cleared after each commit point, preventing unbounded accumulation for long + inputs. The `Trie.prefixes()` method accepts an optional ``start`` offset to + avoid creating large string copies on each call. (#893) + ### Changed - Lazy load dictionaries to reduce memory usage (#1186) diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 76d6d3479..b7db9acb0 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -63,13 +63,17 @@ def _bfs_paths_graph( graph: defaultdict, start: int, goal: int ) -> Generator[list[int], None, None]: + # visited set prevents re-exploring nodes already reached via a shorter + # path, converting worst-case BFS from exponential to O(V + E). + visited: set[int] = {start} queue = [(start, [start])] while queue: (vertex, path) = queue.pop(0) for pos in graph[vertex]: if pos == goal: yield path + [pos] - else: + elif pos not in visited: + visited.add(pos) queue.append((pos, path + [pos])) @@ -89,7 +93,7 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: end_pos = 0 while pos_list[0] < len_text: begin_pos = heappop(pos_list) - for word in custom_dict.prefixes(text[begin_pos:]): + for word in custom_dict.prefixes(text, begin_pos): end_pos_candidate = begin_pos + len(word) if end_pos_candidate in valid_poss: graph[begin_pos].append(end_pos_candidate) @@ -107,20 +111,20 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: _bfs_paths_graph(graph, end_pos, pos_list[0]) ) graph_size = 0 + graph.clear() for pos in end_pos_candidates[1:]: yield text[end_pos:pos] end_pos = pos elif len_pos_list == 0: # no candidate, deal with non-dictionary word - m = _PAT_NONTHAI.match(text[begin_pos:]) + m = _PAT_NONTHAI.match(text, begin_pos) if m: # non-Thai token, skip to the end - end_pos = begin_pos + m.end() + end_pos = m.end() else: # Thai token, find minimum skip for pos in range(begin_pos + 1, len_text): if pos in valid_poss: - prefix = text[pos:] words = [ word - for word in custom_dict.prefixes(prefix) + for word in custom_dict.prefixes(text, pos) if ( (pos + len(word) in valid_poss) and not _PAT_THAI_TWOCHARS.match(word) @@ -131,14 +135,14 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: break # is a non-Thai token - if _PAT_NONTHAI.match(prefix): + if _PAT_NONTHAI.match(text, pos): end_pos = pos break else: end_pos = len_text - graph[begin_pos].append(end_pos) - graph_size = graph_size + 1 + graph_size = 0 + graph.clear() yield text[begin_pos:end_pos] heappush(pos_list, end_pos) @@ -155,13 +159,17 @@ def segment( A custom dictionary can be supplied. + For very long texts (hundreds of kilobytes or more), consider using + ``safe_mode=True`` to enable chunk-based processing and reduce memory use. + :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ defaults to word_dict_trie() :type custom_dict: Trie, optional - :param safe_mode: reduce chance for long processing time for long text\ - with many ambiguous breaking points, defaults to False + :param safe_mode: use chunk-based processing to reduce memory use and + processing time for long text with many ambiguous breaking points, + defaults to False :type safe_mode: bool, optional :return: list of tokens :rtype: list[str] diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index 07f2ddfc8..8a4982708 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -105,22 +105,26 @@ def remove(self, word: str) -> None: break del parent.children[ch] # remove from parent dict - def prefixes(self, text: str) -> list[str]: + def prefixes(self, text: str, start: int = 0) -> list[str]: """List all possible words from first sequence of characters in a word. - :param str text: a word - :return: a list of possible words - :rtype: List[str] + :param str text: text to search for prefixes + :param int start: starting position in text, defaults to 0 + :return: a list of possible words starting at ``start`` + :rtype: list[str] """ res = [] cur = self.root - for i, ch in enumerate(text): - node = cur.children.get(ch) + i = start + n = len(text) + while i < n: + node = cur.children.get(text[i]) if not node: break if node.end: - res.append(text[: i + 1]) + res.append(text[start : i + 1]) cur = node + i += 1 return res def __contains__(self, key: str) -> bool: diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index 35ea68889..a48617340 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -2,6 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import time import unittest from pythainlp.tokenize import ( @@ -583,6 +584,21 @@ def test_newmm_dangertext(self): word_tokenize(DANGER_TEXT_3, engine="newmm-safe"), list ) + def test_newmm_ambiguous_performance(self): + # Regression test for issue #893: newmm BFS path explosion. + # Repeated ambiguous words (e.g. "ด้านหน้า" which can split as + # "ด้าน"+"หน้า" or stay whole) used to cause exponential BFS blowup. + # This test verifies that tokenizing 1,000 repetitions completes + # quickly (well under 1 second). + text = "ด้านหน้า" * 1000 + t = time.perf_counter() + result = word_tokenize(text, engine="newmm") + elapsed = time.perf_counter() - t + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Should complete in well under 1 second after the BFS fix. + self.assertLess(elapsed, 5.0) + def test_tcc(self): assert_segment_handles_none_and_empty(self, tcc.segment) self.assertEqual( From 98511de8970e1cf84b798b6368e738f163d8bd0a Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 9 Mar 2026 21:10:16 +0700 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ffb65ea5..7453e8b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,17 +39,6 @@ The minimum requirement is now Python 3.9. - `pythainlp.is_read_only_mode()` helper function; use `PYTHAINLP_READ_ONLY=1` to prevent all write operations -### Fixed - -- `newmm` tokenization engine: fix exponential-time BFS path explosion when - tokenizing text with many ambiguous breaking points (e.g., repeated words - like "ด้านหน้า" that can be split multiple ways). The internal - `_bfs_paths_graph` function now uses a visited set, reducing worst-case - complexity from exponential to O(V + E). The ambiguity graph is now also - cleared after each commit point, preventing unbounded accumulation for long - inputs. The `Trie.prefixes()` method accepts an optional ``start`` offset to - avoid creating large string copies on each call. (#893) - ### Changed - Lazy load dictionaries to reduce memory usage (#1186) @@ -91,8 +80,11 @@ The minimum requirement is now Python 3.9. - Kho Khon alphabet issue in `tltk` transliteration (#1187) - Suppress Gensim duplicate-word warnings when loading word2vec binary files (#1316) -- `db.json` is no longer created on import; it is created lazily only +- `db.json` is no longer created on import; created lazily only when a corpus is first downloaded (#1317) +- Fix exponential-time explosion in "newmm" tokenization + engine when tokenizing text with many ambiguous + breaking points (#1319) ## [5.2.0] - 2025-12-20