Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[flake8]
max-line-length = 120
extend-ignore =
# Whitespace before ':' — conflicts with Black slice formatting
E203,
# Module level import not at top of file — needed for optional/lazy imports
E402,
# Line too long — ruff already handles style; E501 in ruff extend-ignore too
E501,
# Line break before binary operator — Black compatibility
W503,
# Redefinition of unused name — intentional lazy-import pattern
F811,
63 changes: 63 additions & 0 deletions .github/workflows/mypy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-License-Identifier: Apache-2.0

name: Type check

on:
push:
branches:
- dev
paths-ignore:
- '**.cff'
- '**.json'
- '**.md'
- '**.rst'
- '**.txt'
- 'docs/**'
pull_request:
branches:
- dev
paths-ignore:
- '**.cff'
- '**.json'
- '**.md'
- '**.rst'
- '**.txt'
- 'docs/**'

# Avoid duplicate runs for the same source branch and repository.
# For pull_request events, uses the source repo name from
# github.event.pull_request.head.repo.full_name; otherwise uses github.repository.
# For push events, uses the branch name from github.ref_name.
# For pull_request events, uses the source branch name from github.head_ref.
# This ensures events for the same repo and branch share the same group,
# and avoids cross-fork collisions when branch names are reused.
concurrency:
group: >-
${{ github.workflow }}-${{
github.event.pull_request.head.repo.full_name || github.repository
}}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true

jobs:
mypy:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v6

- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.9"
cache: "pip"

- name: Install mypy
run: |
pip install .
pip install mypy

- name: Run mypy
run: mypy pythainlp
17 changes: 17 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,10 @@ dev = [
"build>=1.0.0",
"bump-my-version>=1.2.6",
"coverage>=7.10.7",
"flake8>=7.0.0",
"flake8-type-checking>=3.2.0",
"mypy>=1.19.1",
"pylint>=4.0.0",
"ruff>=0.14.14",
"tox>=4.30.3",
]
Expand Down Expand Up @@ -460,10 +463,13 @@ module = [
"gensim.*",
"huggingface_hub.*",
"icu.*",
"importlib_resources",
"importlib_resources.*",
"khamyo.*",
"khanaa.*",
"multiel.*",
"nltk.*",
"nlpo3.*",
"numpy.*",
"onnxruntime.*",
"oskut.*",
Expand Down Expand Up @@ -494,3 +500,14 @@ module = [
"yaml.*",
]
ignore_missing_imports = true

[tool.pylint.main]
disable = [
"import-error",
"no-name-in-module",
"too-few-public-methods",
"too-many-arguments",
"too-many-locals",
"too-many-branches",
"too-many-statements",
]
2 changes: 1 addition & 1 deletion pythainlp/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
__version__ = "5.2.0" # type: ignore
__version__ = "5.2.0"

thai_consonants: str = (
"กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars
Expand Down
6 changes: 4 additions & 2 deletions pythainlp/augment/wordnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@

import itertools
from collections import OrderedDict
from typing import Callable, Optional
from typing import TYPE_CHECKING, Callable, Optional

from nltk.corpus import wordnet as wn
from nltk.corpus.reader.wordnet import Synset

if TYPE_CHECKING:
from nltk.corpus.reader.wordnet import Synset

from pythainlp.corpus import wordnet
from pythainlp.tag import pos_tag
Expand Down
43 changes: 28 additions & 15 deletions pythainlp/benchmarks/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@
This module provides pure Python implementations of common evaluation
metrics (BLEU, ROUGE) that handle Thai text tokenization automatically.
"""

from __future__ import annotations

import math
from collections import Counter
from typing import Union
from typing import Optional, Union, cast


def _get_ngrams(tokens: list[str], n: int) -> list[tuple[str, ...]]:
Expand Down Expand Up @@ -82,15 +83,15 @@ def bleu_score(
lowercase: bool = False,
max_ngram: int = 4,
smooth: bool = True,
) -> dict[str, float]:
) -> dict[str, Union[float, list[float]]]:
"""
Calculate BLEU score for Thai text with automatic tokenization.

This is a pure Python implementation of BLEU (Bilingual Evaluation
Understudy) metric that automatically tokenizes Thai text using
PyThaiNLP before calculating the score.

:param list[str] | list[list[str]] references: reference translations.
:param Union[list[str], list[list[str]]] references: reference translations.
Can be:
- A list of strings (one reference per hypothesis)
- A list of lists of strings (multiple references per hypothesis)
Expand All @@ -103,9 +104,11 @@ def bleu_score(
:param bool smooth: whether to use smoothing for zero counts
(default: True)

:return: dictionary with 'bleu', 'precisions', 'bp', 'length_ratio',
'hyp_length', and 'ref_length'
:rtype: dict[str, float]
:return: dictionary with ``'bleu'``, ``'precisions'``, ``'bp'``,
``'length_ratio'``, ``'hyp_length'``, and ``'ref_length'``.
The ``'precisions'`` value is a ``list[float]``; all other values
are ``float``.
:rtype: dict[str, Union[float, list[float]]]

:Example:
::
Expand All @@ -131,7 +134,9 @@ def bleu_score(

# Normalize references format
if references and isinstance(references[0], str):
refs_normalized: list[list[str]] = [[ref] for ref in references]
refs_normalized: list[list[str]] = [
[ref] for ref in cast("list[str]", references)
]
else:
refs_normalized = references # type: ignore[assignment]

Expand Down Expand Up @@ -231,7 +236,7 @@ def rouge_score(
reference: str,
hypothesis: str,
tokenize: str = "newmm",
rouge_types: list[str] | None = None,
rouge_types: Optional[list[str]] = None,
) -> dict[str, tuple[float, float, float]]:
"""
Calculate ROUGE scores for Thai text with automatic tokenization.
Expand All @@ -249,7 +254,7 @@ def rouge_score(
:param str hypothesis: hypothesis text to evaluate
:param str tokenize: tokenization engine to use (default: "newmm").
See :func:`pythainlp.tokenize.word_tokenize` for available engines.
:param list[str] | None rouge_types: list of ROUGE types to calculate.
:param Optional[list[str]] rouge_types: list of ROUGE types to calculate.
Default is ["rouge1", "rouge2", "rougeL"]

:return: dictionary mapping ROUGE type to (precision, recall, fmeasure)
Expand All @@ -273,8 +278,12 @@ def rouge_score(
rouge_types = ["rouge1", "rouge2", "rougeL"]

# Tokenize texts
ref_tokens = word_tokenize(reference, engine=tokenize, keep_whitespace=False)
hyp_tokens = word_tokenize(hypothesis, engine=tokenize, keep_whitespace=False)
ref_tokens = word_tokenize(
reference, engine=tokenize, keep_whitespace=False
)
hyp_tokens = word_tokenize(
hypothesis, engine=tokenize, keep_whitespace=False
)

result: dict[str, tuple[float, float, float]] = {}

Expand Down Expand Up @@ -364,8 +373,12 @@ def word_error_rate(
from pythainlp.tokenize import word_tokenize

# Tokenize texts
ref_tokens = word_tokenize(reference, engine=tokenize, keep_whitespace=False)
hyp_tokens = word_tokenize(hypothesis, engine=tokenize, keep_whitespace=False)
ref_tokens = word_tokenize(
reference, engine=tokenize, keep_whitespace=False
)
hyp_tokens = word_tokenize(
hypothesis, engine=tokenize, keep_whitespace=False
)

# Calculate edit distance using dynamic programming
r = len(ref_tokens)
Expand Down Expand Up @@ -393,7 +406,7 @@ def word_error_rate(

# Calculate WER
if r == 0:
return 0.0 if h == 0 else float('inf')
return 0.0 if h == 0 else float("inf")

return d[r][h] / r

Expand Down Expand Up @@ -465,6 +478,6 @@ def character_error_rate(

# Calculate CER
if r == 0:
return 0.0 if h == 0 else float('inf')
return 0.0 if h == 0 else float("inf")

return d[r][h] / r
10 changes: 7 additions & 3 deletions pythainlp/benchmarks/word_tokenization.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ def _f1(precision: float, recall: float) -> float:
return 2 * precision * recall / (precision + recall)


def _flatten_result(my_dict: dict, sep: str = ":") -> dict[str, Union[int, str]]:
def _flatten_result(
my_dict: dict, sep: str = ":"
) -> dict[str, Union[int, str]]:
"""Flatten two-dimension dictionary.

Use keys in the first dimension as a prefix for keys in the second dimension.
Expand Down Expand Up @@ -89,7 +91,7 @@ def benchmark(ref_samples: list[str], samples: list[str]) -> "pd.DataFrame":
flat_stats["expected"] = r
flat_stats["actual"] = s
results.append(flat_stats)
except:
except Exception:
reason = """
[Error]
Reason: %s
Expand Down Expand Up @@ -133,7 +135,9 @@ def preprocessing(txt: str, remove_space: bool = True) -> str:
return txt


def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, dict[str, Union[int, str]]]:
def compute_stats(
ref_sample: str, raw_sample: str
) -> dict[str, dict[str, Union[int, str]]]:
"""Compute statistics for tokenization quality

These statistics include:
Expand Down
29 changes: 19 additions & 10 deletions pythainlp/braille/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from __future__ import annotations

import re
from typing import Union, cast

from pythainlp.tokenize import word_tokenize
from pythainlp.util import Trie
Expand Down Expand Up @@ -138,11 +139,16 @@
_v1: list[str] = ["เ-tอ", "เ-ีtย", "เ-ืtอ", "-ัtว", "เ-tา", "เ-tาะ"]

# Create trie for efficient pattern matching
char_trie: Trie = Trie(list(thai_braille_mapping_dict.keys()) + _v1 + [" ", "<N>"])
char_trie: Trie = Trie(
list(thai_braille_mapping_dict.keys()) + _v1 + [" ", "<N>"]
)

# Build vowel replacement patterns
_vowel_patterns: list[str] = [
i.replace("-", "([ก-ฮ])").replace("t", "([่้๊๋])") + ",\\1" + i.replace("t", "") + "\\2"
i.replace("-", "([ก-ฮ])").replace("t", "([่้๊๋])")
+ ",\\1"
+ i.replace("t", "")
+ "\\2"
for i in _v1
]
_vowel_patterns += [
Expand Down Expand Up @@ -241,18 +247,19 @@ class Braille:
Converts dot number patterns to Unicode braille characters.
"""

def __init__(self, data: list[list[str]] | list[str] | str) -> None:
def __init__(self, data: Union[list[list[str]], list[str], str]) -> None:
"""Initialize Braille converter.

:param data: Braille dot patterns as list or string
:type data: list[list[str]] | list[str] | str
:type data: Union[list[list[str]], list[str], str]
"""
self.inputdata: list[list[str]] | list[str] | str = data
self.inputdata: Union[list[list[str]], list[str], str] = data
if isinstance(data, list):
if len(data) > 1:
self.data: list[list[str]] | list[str] = [""] * len(data)
for i in range(len(data)):
self.data[i] = sorted(list(data[i]))
nested_data: list[list[str]] = [[] for _ in range(len(data))]
for i, item in enumerate(data):
nested_data[i] = sorted(item)
self.data: Union[list[list[str]], list[str]] = nested_data
elif len(data) == 1:
self.data = sorted(list(data[0]))
else:
Expand Down Expand Up @@ -540,7 +547,7 @@ def tobraille(self) -> str:
result += self.db[pattern_str]
return result
else:
pattern_str = "".join(self.data)
pattern_str = "".join(cast("list[str]", self.data))
return self.db.get(pattern_str, "")

def printbraille(self) -> str:
Expand Down Expand Up @@ -571,6 +578,8 @@ def printbraille(self) -> str:
mirrored_patterns.reverse()
return "".join(mirrored_patterns)
else:
mirrored = "".join(mirror_map[dot] for dot in self.data)
mirrored = "".join(
mirror_map[dot] for dot in cast("list[str]", self.data)
)
mirrored_sorted = "".join(sorted(mirrored))
return self.db[mirrored_sorted]
1 change: 1 addition & 0 deletions pythainlp/chat/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from pythainlp.generate.wangchanglm import WangChanGLM


class ChatBotModel:
history: list[tuple[str, str]]
model: "WangChanGLM"
Expand Down
2 changes: 1 addition & 1 deletion pythainlp/corpus/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import sys
import tarfile
import zipfile
from http.client import HTTPMessage, HTTPResponse
from importlib.resources import files
from typing import TYPE_CHECKING

Expand All @@ -21,6 +20,7 @@
from pythainlp.tools.path import is_offline_mode

if TYPE_CHECKING:
from http.client import HTTPMessage, HTTPResponse
from typing import Any, Optional

_CHECK_MODE: Optional[str] = os.getenv("PYTHAINLP_READ_MODE")
Expand Down
Loading
Loading