From ee86a713ba71e62a2f787346197c92f8001a2143 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:22:48 +0000 Subject: [PATCH 1/4] Initial plan From 2136cdd42b6ce8875b2c4c9c030fc752dfec4d3e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:26:21 +0000 Subject: [PATCH 2/4] Refactor featurize() to separate list and dict logic, removing Union type and cast() calls Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tokenize/han_solo.py | 109 +++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 06491947a..2e2d56bf4 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -10,7 +10,7 @@ import threading from importlib.resources import as_file, files -from typing import Optional, Union, cast +from typing import Optional try: import pycrfsuite @@ -58,11 +58,87 @@ def pad(self, sentence: str, padder: str = "#") -> str: def featurize( self, sentence: str, padding: bool = True, indiv_char: bool = True, return_type: str = "list" + ) -> dict[str, list]: + if return_type == "list": + return self._featurize_list(sentence, padding, indiv_char) + else: + return self._featurize_dict(sentence, padding, indiv_char) + + def _featurize_list( + self, sentence: str, padding: bool, indiv_char: bool + ) -> dict[str, list]: + if padding: + sentence = self.pad(sentence) + all_features: list[list[str]] = [] + all_labels: list[str] = [] + skip_next = False + for current_position in range( + self.radius, len(sentence) - self.radius + 1 + ): + if skip_next: + skip_next = False + continue + features: list[str] = [] + cut = 0 + char = sentence[current_position] + if char == self.delimiter: + cut = 1 + skip_next = True + counter = 0 + chars_left = "" + chars_right = "" + abs_index_left = current_position # left start at -1 + abs_index_right = current_position - 1 # right start at 0 + while counter < self.radius: + abs_index_left -= ( + 1 # สมมุติตำแหน่งที่ 0 จะได้ -1, -2, -3, -4, -5 (radius = 5) + ) + char_left = sentence[abs_index_left] + while char_left == self.delimiter: + abs_index_left -= 1 + char_left = sentence[abs_index_left] + relative_index_left = -counter - 1 + # เก็บตัวหนังสือ + chars_left = char_left + chars_left + # ใส่ลง feature + if indiv_char: + left_key = "|".join([str(relative_index_left), char_left]) + features.append(left_key) + + abs_index_right += ( + 1 # สมมุติคือตำแหน่งที่ 0 จะได้ 0, 1, 2, 3, 4 (radius = 5) + ) + char_right = sentence[abs_index_right] + while char_right == self.delimiter: + abs_index_right += 1 + char_right = sentence[abs_index_right] + relative_index_right = counter + chars_right += char_right + if indiv_char: + right_key = "|".join( + [str(relative_index_right), char_right] + ) + features.append(right_key) + + counter += 1 + + chars = chars_left + chars_right + for i in range(0, len(chars) - self.N + 1): + ngram = chars[i : i + self.N] + ngram_key = "|".join([str(i - self.radius), ngram]) + features.append(ngram_key) + all_features.append(features) + all_labels.append(str(cut)) + + return {"X": all_features, "Y": all_labels} + + def _featurize_dict( + self, sentence: str, padding: bool, indiv_char: bool ) -> dict[str, list]: if padding: sentence = self.pad(sentence) - all_features = [] - all_labels = [] + all_features: list[dict[str, int]] = [] + all_labels: list[int] = [] skip_next = False for current_position in range( self.radius, len(sentence) - self.radius + 1 @@ -70,11 +146,7 @@ def featurize( if skip_next: skip_next = False continue - features: Union[dict[str, int], list[str]] - if return_type == "list": - features = [] - else: - features = {} + features: dict[str, int] = {} cut = 0 char = sentence[current_position] if char == self.delimiter: @@ -83,7 +155,6 @@ def featurize( counter = 0 chars_left = "" chars_right = "" - chars = "" abs_index_left = current_position # left start at -1 abs_index_right = current_position - 1 # right start at 0 while counter < self.radius: @@ -100,10 +171,7 @@ def featurize( # ใส่ลง feature if indiv_char: left_key = "|".join([str(relative_index_left), char_left]) - if return_type == "dict": - cast(dict[str, int], features)[left_key] = 1 - else: - cast(list[str], features).append(left_key) + features[left_key] = 1 abs_index_right += ( 1 # สมมุติคือตำแหน่งที่ 0 จะได้ 0, 1, 2, 3, 4 (radius = 5) @@ -118,10 +186,7 @@ def featurize( right_key = "|".join( [str(relative_index_right), char_right] ) - if return_type == "dict": - cast(dict[str, int], features)[right_key] = 1 - else: - cast(list[str], features).append(right_key) + features[right_key] = 1 counter += 1 @@ -129,15 +194,9 @@ def featurize( for i in range(0, len(chars) - self.N + 1): ngram = chars[i : i + self.N] ngram_key = "|".join([str(i - self.radius), ngram]) - if return_type == "dict": - cast(dict[str, int], features)[ngram_key] = 1 - else: - cast(list[str], features).append(ngram_key) + features[ngram_key] = 1 all_features.append(features) - if return_type == "list": - all_labels.append(str(cut)) - else: - all_labels.append(cut) + all_labels.append(cut) return {"X": all_features, "Y": all_labels} From 150ce69d0c9c4c06a513f9ddd918e427a02695e7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:29:41 +0000 Subject: [PATCH 3/4] Refactor to avoid code duplication while maintaining type safety Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tokenize/han_solo.py | 93 +++++----------------------------- 1 file changed, 14 insertions(+), 79 deletions(-) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 2e2d56bf4..752398202 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -58,19 +58,11 @@ def pad(self, sentence: str, padder: str = "#") -> str: def featurize( self, sentence: str, padding: bool = True, indiv_char: bool = True, return_type: str = "list" - ) -> dict[str, list]: - if return_type == "list": - return self._featurize_list(sentence, padding, indiv_char) - else: - return self._featurize_dict(sentence, padding, indiv_char) - - def _featurize_list( - self, sentence: str, padding: bool, indiv_char: bool ) -> dict[str, list]: if padding: sentence = self.pad(sentence) - all_features: list[list[str]] = [] - all_labels: list[str] = [] + all_features_list: list[list[str]] = [] + all_labels_int: list[int] = [] skip_next = False for current_position in range( self.radius, len(sentence) - self.radius + 1 @@ -127,76 +119,19 @@ def _featurize_list( ngram = chars[i : i + self.N] ngram_key = "|".join([str(i - self.radius), ngram]) features.append(ngram_key) - all_features.append(features) - all_labels.append(str(cut)) + all_features_list.append(features) + all_labels_int.append(cut) - return {"X": all_features, "Y": all_labels} - - def _featurize_dict( - self, sentence: str, padding: bool, indiv_char: bool - ) -> dict[str, list]: - if padding: - sentence = self.pad(sentence) - all_features: list[dict[str, int]] = [] - all_labels: list[int] = [] - skip_next = False - for current_position in range( - self.radius, len(sentence) - self.radius + 1 - ): - if skip_next: - skip_next = False - continue - features: dict[str, int] = {} - cut = 0 - char = sentence[current_position] - if char == self.delimiter: - cut = 1 - skip_next = True - counter = 0 - chars_left = "" - chars_right = "" - abs_index_left = current_position # left start at -1 - abs_index_right = current_position - 1 # right start at 0 - while counter < self.radius: - abs_index_left -= ( - 1 # สมมุติตำแหน่งที่ 0 จะได้ -1, -2, -3, -4, -5 (radius = 5) - ) - char_left = sentence[abs_index_left] - while char_left == self.delimiter: - abs_index_left -= 1 - char_left = sentence[abs_index_left] - relative_index_left = -counter - 1 - # เก็บตัวหนังสือ - chars_left = char_left + chars_left - # ใส่ลง feature - if indiv_char: - left_key = "|".join([str(relative_index_left), char_left]) - features[left_key] = 1 - - abs_index_right += ( - 1 # สมมุติคือตำแหน่งที่ 0 จะได้ 0, 1, 2, 3, 4 (radius = 5) - ) - char_right = sentence[abs_index_right] - while char_right == self.delimiter: - abs_index_right += 1 - char_right = sentence[abs_index_right] - relative_index_right = counter - chars_right += char_right - if indiv_char: - right_key = "|".join( - [str(relative_index_right), char_right] - ) - features[right_key] = 1 - - counter += 1 - - chars = chars_left + chars_right - for i in range(0, len(chars) - self.N + 1): - ngram = chars[i : i + self.N] - ngram_key = "|".join([str(i - self.radius), ngram]) - features[ngram_key] = 1 - all_features.append(features) - all_labels.append(cut) + # Convert to the requested return type + if return_type == "list": + all_features: list[list[str]] = all_features_list + all_labels: list[str] = [str(label) for label in all_labels_int] + else: + all_features: list[dict[str, int]] = [ + {key: 1 for key in feature_list} + for feature_list in all_features_list + ] + all_labels: list[int] = all_labels_int return {"X": all_features, "Y": all_labels} From 64bfc99ef08f7226b19297aa0f963b8f18aebbf9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 01:30:47 +0000 Subject: [PATCH 4/4] Remove variable shadowing for clearer code Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tokenize/han_solo.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 752398202..686c81ee2 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -124,16 +124,18 @@ def featurize( # Convert to the requested return type if return_type == "list": - all_features: list[list[str]] = all_features_list - all_labels: list[str] = [str(label) for label in all_labels_int] + return { + "X": all_features_list, + "Y": [str(label) for label in all_labels_int] + } else: - all_features: list[dict[str, int]] = [ - {key: 1 for key in feature_list} - for feature_list in all_features_list - ] - all_labels: list[int] = all_labels_int - - return {"X": all_features, "Y": all_labels} + return { + "X": [ + {key: 1 for key in feature_list} + for feature_list in all_features_list + ], + "Y": all_labels_int + } _to_feature = Featurizer()