diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 000000000..a96f860c2 --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2026-present PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +# Bandit is a tool designed to find common security issues in Python code. +# https://github.com/pycqa/bandit + +name: Bandit + +on: + push: + branches: + - dev + - main + paths-ignore: + - '**.cff' + - '**.json' + - '**.md' + - '**.rst' + - '**.txt' + - 'docs/**' + pull_request: + branches: + - dev + - main + paths-ignore: + - '**.cff' + - '**.json' + - '**.md' + - '**.rst' + - '**.txt' + - 'docs/**' + +# Avoid duplicate runs for the same source branch and repository. +# For pull_request events, uses the source repo name from +# github.event.pull_request.head.repo.full_name; otherwise uses github.repository. +# For push events, uses the branch name from github.ref_name. +# For pull_request events, uses the source branch name from github.head_ref. +# This ensures events for the same repo and branch share the same group, +# and avoids cross-fork collisions when branch names are reused. +concurrency: + group: >- + ${{ github.workflow }}-${{ + github.event.pull_request.head.repo.full_name || github.repository + }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + bandit: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + cache: "pip" + + - name: Install bandit + run: pip install "bandit>=1.9.4" + + - name: Run bandit + run: bandit -r pythainlp -c pyproject.toml diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index e035dbf68..a843b96a8 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -25,7 +25,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Generate CITATION.cff from codemeta.json uses: caltechlibrary/codemeta2cff@99ce10831c3037a067c3d9551d455638e0b438ac #v0.3.3 @@ -37,7 +37,7 @@ jobs: printf '\n```' >> $GITHUB_STEP_SUMMARY - name: Validate CITATION.cff - uses: dieghernan/cff-validator@114aae53e1850c3757733beb60036941900e3dc3 #v4 + uses: dieghernan/cff-validator@54c8e737314b948f25c0e5524a9d8ae3f39b207b #v5.0.1 - name: Commit and push updated CITATION.cff uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c613ade92..fe14eed7e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -62,11 +62,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@v4.36.2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4 + uses: github/codeql-action/autobuild@v4.36.2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@v4.36.2 diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 1609af5cf..3126a3b05 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 749c17e54..edbb627c6 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 with: @@ -38,7 +38,7 @@ jobs: cd docs && make html cd .. - name: Deploy documentation - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@v4.1.0 with: personal_token: ${{ secrets.PERSONAL_TOKEN }} publish_dir: ./docs/_build/html diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 383158175..bc0d47183 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Ruff uses: astral-sh/ruff-action@v3 with: diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml index d7dd798e9..1dfdbef0b 100644 --- a/.github/workflows/markdownlint.yml +++ b/.github/workflows/markdownlint.yml @@ -24,11 +24,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.3 with: fetch-depth: 0 - - uses: DavidAnson/markdownlint-cli2-action@v23.1.0 + - uses: DavidAnson/markdownlint-cli2-action@v23.2.0 with: globs: | **/*.md diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index ce8f1c65f..d174913cb 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -46,7 +46,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/publish-versioned-docs.yml b/.github/workflows/publish-versioned-docs.yml index 5d2be64d4..0aea53fe4 100644 --- a/.github/workflows/publish-versioned-docs.yml +++ b/.github/workflows/publish-versioned-docs.yml @@ -31,19 +31,19 @@ jobs: run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - name: Checkout pythainlp (for scripts) - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 with: path: pythainlp - name: Checkout dev-docs repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 with: repository: PyThaiNLP/dev-docs token: ${{ secrets.PERSONAL_TOKEN }} path: dev-docs - name: Checkout docs repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 with: repository: PyThaiNLP/docs token: ${{ secrets.PERSONAL_TOKEN }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 95ca9c901..cba969cb9 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -52,7 +52,7 @@ jobs: build: ${{ steps.check_build_trigger.outputs.build }} steps: - name: Checkout source code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 with: ref: ${{ github.event.pull_request.head.sha }} - id: check_build_trigger @@ -71,7 +71,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index d18a6d22d..98f7a6472 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -21,7 +21,7 @@ jobs: python-version: ["3.12"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f8d6c9770..9566cebb0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v10 + uses: actions/stale@v10.3.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 16ecc2d34..7987dd370 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -82,7 +82,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/docs/api/transliterate.rst b/docs/api/transliterate.rst index e47a54568..e2a8036c7 100644 --- a/docs/api/transliterate.rst +++ b/docs/api/transliterate.rst @@ -17,6 +17,9 @@ Modules The `transliterate` function serves as a versatile transliteration tool, offering a range of transliteration engines to choose from. It provides flexibility and customization for your transliteration needs. +.. autofunction:: transliterate_wiktionary + :noindex: + .. autofunction:: pronunciate :noindex: @@ -30,6 +33,9 @@ Modules The `puan` function offers a unique transliteration feature known as "Puan." It provides a specialized transliteration method for Thai text and is an additional option for rendering Thai text into English characters. +.. autofunction:: get_word_dict + :noindex: + .. autoclass:: pythainlp.transliterate.wunsen.WunsenTransliterate :members: diff --git a/pyproject.toml b/pyproject.toml index 308fda4ae..fd8463d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2016-present PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -71,6 +71,7 @@ dependencies = [ ## 1) Development ######################################## dev = [ + "bandit>=1.9.4", "black>=25.11.0", "build>=1.0.0", "bump-my-version>=1.2.6", @@ -311,6 +312,11 @@ include = [ "README.md", ] +[tool.bandit] +# Skip tests that produce known false positives or are accepted risks. +# B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. +skips = ["B110"] + [tool.bumpversion] current_version = "5.3.4" commit = true diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index 2825ebe46..3a3134e93 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -29,10 +29,10 @@ def __init__(self) -> None: ) self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( - _MODEL_NAME + _MODEL_NAME # nosec B615 ) self.model_for_masked_lm: AutoModelForMaskedLM = ( - AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) + AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) # nosec B615 ) self.model: Pipeline = pipeline( "fill-mask", @@ -55,7 +55,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 # NOSONAR output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index f593b3a95..1ccda2803 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -29,7 +29,7 @@ def __init__(self) -> None: self.target_tokenizer: type[CamembertTokenizer] = CamembertTokenizer self.tokenizer: CamembertTokenizer = ( CamembertTokenizer.from_pretrained( - self.model_name, revision="main" + self.model_name, revision="main" # nosec B615 ) ) self.tokenizer.additional_special_tokens = [ diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 9da5930bb..7e33483c3 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -71,7 +71,7 @@ def get_corpus_db(url: str) -> Optional[_ResponseWrapper]: try: req = Request(url, headers={"User-Agent": _USER_AGENT}) # SSL certificate verification is enabled by default - with urlopen(req, timeout=10) as response: + with urlopen(req, timeout=10) as response: # nosec B310 corpus_db = _ResponseWrapper(response) except HTTPError as http_err: print(f"HTTP error occurred: {http_err}") @@ -349,7 +349,7 @@ def _download(url: str, dst: str) -> int: req = Request(url, headers={"User-Agent": _USER_AGENT}) # SSL certificate verification is enabled by default - with urlopen(req, timeout=10) as response: + with urlopen(req, timeout=10) as response: # nosec B310 file_size = int(response.info().get("Content-Length", -1)) with open(get_full_data_path(dst), "wb") as f: pbar = None @@ -383,7 +383,7 @@ def _check_hash(dst: str, md5: str) -> None: with open(get_full_data_path(dst), "rb") as f: content = f.read() # MD5 is insecure but sufficient here - file_md5 = hashlib.md5(content).hexdigest() # noqa: S324 + file_md5 = hashlib.md5(content).hexdigest() # noqa: S324 # nosec B324 if md5 != file_md5: raise ValueError("Hash does not match expected.") @@ -484,7 +484,7 @@ def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: f"Symlink {member.name} points outside extraction directory: {member.linkname}" ) - tar.extractall(path=path) + tar.extractall(path=path) # nosec B202 def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: @@ -539,7 +539,7 @@ def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: f"Symlink {member} points outside extraction directory: {link_target}" ) - zip_file.extractall(path=path) + zip_file.extractall(path=path) # nosec B202 def _version2int(v: str) -> int: @@ -853,12 +853,17 @@ def make_safe_directory_name(name: str) -> str: return safe_name -def get_hf_hub(repo_id: str, filename: str = "") -> str: +def get_hf_hub( + repo_id: str, filename: str = "", revision: Optional[str] = None +) -> str: """HuggingFace Hub in :mod:`pythainlp` data directory. :param str repo_id: repo_id :param str filename: filename (optional, default is empty string). If empty, downloads entire snapshot. + :param Optional[str] revision: a git revision id, which can be a branch + name, a tag, or a commit hash (optional, default is ``None``). + Pin to a full commit hash for reproducible and secure downloads. :return: path :rtype: str """ @@ -876,10 +881,15 @@ def get_hf_hub(repo_id: str, filename: str = "") -> str: root_project = safe_path_join(hf_root, name_dir) if filename: output_path = hf_hub_download( - repo_id=repo_id, filename=filename, local_dir=root_project + repo_id=repo_id, + filename=filename, + local_dir=root_project, + revision=revision, ) else: output_path = snapshot_download( - repo_id=repo_id, local_dir=root_project + repo_id=repo_id, + local_dir=root_project, + revision=revision, ) return str(output_path) diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 391fc7d47..214c8b1d2 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -82,7 +82,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.word) # noqa: S311 + start_seq = random.choice(self.word) # noqa: S311 # nosec B311 # NOSONAR rand_text = start_seq.lower() self._word_prob = { i: self.counts[i] / self.n @@ -108,10 +108,10 @@ def _next_word( N = len(word_list) for _ in range(N): # Non-cryptographic use, pseudo-random generator is acceptable here - w = random.choice(word_list) # noqa: S311 + w = random.choice(word_list) # noqa: S311 # nosec B311 # NOSONAR if duplicate is False: while w in words: - w = random.choice(word_list) # noqa: S311 + w = random.choice(word_list) # noqa: S311 # nosec B311 # NOSONAR words.append(w) if output_str: @@ -185,7 +185,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.words) # noqa: S311 + start_seq = random.choice(self.words) # noqa: S311 # nosec B311 # NOSONAR late_word = start_seq list_word = [] list_word.append(start_seq) @@ -204,7 +204,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 # NOSONAR late_word = items[-1] list_word.append(late_word) @@ -288,7 +288,7 @@ def gen_sentence( late_word: Union[str, tuple[str, str]] if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.bi_keys) # noqa: S311 + start_seq = random.choice(self.bi_keys) # noqa: S311 # nosec B311 # NOSONAR late_word = start_seq list_word: list[Union[str, tuple[str, str]]] = [] list_word.append(start_seq) @@ -307,7 +307,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 # NOSONAR late_word = items[1:] list_word.append(late_word) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 08492c597..b61dda4c6 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -97,7 +97,7 @@ "emb_sz": 400, "n_hid": 1550, "n_layers": 4, - "pad_token": 1, + "pad_token": 1, # nosec B105 "qrnn": False, "tie_weights": True, "out_bias": True, @@ -150,7 +150,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(list(thwiki_itos)) # noqa: S311 + start_seq = random.choice(list(thwiki_itos)) # noqa: S311 # nosec B311 # NOSONAR predicted_text: str = learn.predict( start_seq, N, temperature=0.8, min_p=prob, sep="-*-" ) diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index c0fc69306..7d18f60eb 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -47,6 +47,7 @@ def load_model( torch_dtype: Optional["torch.dtype"] = None, offload_folder: str = "./", low_cpu_mem_usage: bool = True, + revision: Optional[str] = None, ) -> None: """Load model @@ -57,6 +58,8 @@ def load_model( :param Optional[torch.dtype] torch_dtype: torch_dtype :param str offload_folder: offload folder :param bool low_cpu_mem_usage: low cpu mem usage + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ import pandas as pd from transformers import AutoModelForCausalLM, AutoTokenizer @@ -72,9 +75,10 @@ def load_model( torch_dtype=torch_dtype, offload_folder=offload_folder, low_cpu_mem_usage=low_cpu_mem_usage, + revision=revision, ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(self.model_path) + AutoTokenizer.from_pretrained(self.model_path, revision=revision) ) self.df: "pd.DataFrame" = pd.DataFrame( self.tokenizer.vocab.items(), columns=["text", "idx"] diff --git a/pythainlp/lm/qwen3.py b/pythainlp/lm/qwen3.py index 82d8ac32b..fd6a49be4 100644 --- a/pythainlp/lm/qwen3.py +++ b/pythainlp/lm/qwen3.py @@ -31,6 +31,7 @@ def load_model( device: str = "cuda", torch_dtype: Optional["torch.dtype"] = None, low_cpu_mem_usage: bool = True, + revision: Optional[str] = None, ) -> None: """Load Qwen3 model. @@ -38,6 +39,8 @@ def load_model( :param str device: device (cpu, cuda or other) :param Optional[torch.dtype] torch_dtype: torch data type (e.g., torch.float16, torch.bfloat16) :param bool low_cpu_mem_usage: low cpu mem usage + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. :Example: @@ -74,7 +77,9 @@ def load_model( self.model_path = model_path try: - self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_path, revision=revision + ) except OSError as exc: raise RuntimeError( f"Failed to load tokenizer from '{self.model_path}'. " @@ -87,6 +92,7 @@ def load_model( device_map=device, torch_dtype=torch_dtype, low_cpu_mem_usage=low_cpu_mem_usage, + revision=revision, ) except OSError as exc: # Clean up tokenizer on failure diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 4d0390722..18a8433d6 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -26,7 +26,9 @@ class Parse: def __init__( - self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-head" + self, + model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-head", + revision: Optional[str] = None, ) -> None: from transformers import ( AutoConfig, @@ -39,9 +41,13 @@ def __init__( if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-head" - self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( + model, revision=revision + ) self.model: AutoModelForQuestionAnswering = ( - AutoModelForQuestionAnswering.from_pretrained(model) + AutoModelForQuestionAnswering.from_pretrained( + model, revision=revision + ) ) x = AutoModelForTokenClassification.from_pretrained if os.path.isdir(model): @@ -50,14 +56,14 @@ def __init__( x(safe_path_join(model, "tagger")), ) else: - c = AutoConfig.from_pretrained( - cached_file(model, "deprel/config.json") + c = AutoConfig.from_pretrained( # nosec B615 + cached_file(model, "deprel/config.json", revision=revision), ) - d = x(cached_file(model, "deprel/pytorch_model.bin"), config=c) - s = AutoConfig.from_pretrained( - cached_file(model, "tagger/config.json") + d = x(cached_file(model, "deprel/pytorch_model.bin", revision=revision), config=c) + s = AutoConfig.from_pretrained( # nosec B615 + cached_file(model, "tagger/config.json", revision=revision), ) - t = x(cached_file(model, "tagger/pytorch_model.bin"), config=s) + t = x(cached_file(model, "tagger/pytorch_model.bin", revision=revision), config=s) self.deprel: TokenClassificationPipeline = TokenClassificationPipeline( model=d, tokenizer=self.tokenizer, aggregation_strategy="simple" ) diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index 3e28d7412..2d6705f4c 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -20,12 +20,17 @@ class Parse: def __init__( self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-goeswith", + revision: Optional[str] = None, ) -> None: if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-goeswith" - self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( + model, revision=revision + ) self.model: AutoModelForTokenClassification = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def __call__( diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index d0a368ec5..603b5af8c 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -6,7 +6,7 @@ import random import re import warnings -from typing import TYPE_CHECKING, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast if TYPE_CHECKING: from collections.abc import Callable @@ -29,7 +29,7 @@ _model_name: str = "clicknext/phayathaibert" _tokenizer: "CamembertTokenizer" = CamembertTokenizer.from_pretrained( - _model_name + _model_name # nosec B615 ) @@ -120,7 +120,7 @@ def rm_useless_spaces(self, text: str) -> str: """ return re.sub(" {2,}", " ", text) - def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S107 + def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S107 # nosec B107 """Replace spaces with _ :param str text: text to replace spaces :return: text where all spaces replaced with _ @@ -227,10 +227,10 @@ def __init__(self) -> None: ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(_model_name) + AutoTokenizer.from_pretrained(_model_name) # nosec B615 ) self.model_for_masked_lm: "AutoModelForMaskedLM" = ( - AutoModelForMaskedLM.from_pretrained(_model_name) + AutoModelForMaskedLM.from_pretrained(_model_name) # nosec B615 ) self.model: "Pipeline" = pipeline( # transformers.Pipeline "fill-mask", @@ -253,7 +253,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 # NOSONAR output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] @@ -317,7 +317,11 @@ def augment( class PartOfSpeechTagger: - def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: + def __init__( + self, + model: str = "lunarlist/pos_thai_phayathai", + revision: Optional[str] = None, + ) -> None: # Load model directly from transformers import ( AutoModelForTokenClassification, @@ -325,10 +329,12 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: "AutoModelForTokenClassification" = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def get_tag( @@ -364,17 +370,23 @@ def get_tag( class NamedEntityTagger: - def __init__(self, model: str = "Pavarissy/phayathaibert-thainer") -> None: + def __init__( + self, + model: str = "Pavarissy/phayathaibert-thainer", + revision: Optional[str] = None, + ) -> None: from transformers import ( AutoModelForTokenClassification, AutoTokenizer, ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: "AutoModelForTokenClassification" = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def get_ner( diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 5ac7003f1..8062894b8 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -27,7 +27,7 @@ use_cuda: bool = torch.cuda.is_available() device: torch.device = torch.device("cuda" if use_cuda else "cpu") tokenizer: "PreTrainedTokenizer" = AutoTokenizer.from_pretrained( - "airesearch/wangchanberta-base-att-spm-uncased" + "airesearch/wangchanberta-base-att-spm-uncased" # nosec B615 ) @@ -36,7 +36,7 @@ def __init__(self) -> None: super().__init__() self.bert: BertForTokenClassification = ( BertForTokenClassification.from_pretrained( - "bookpanda/wangchanberta-base-att-spm-uncased-tagging" + "bookpanda/wangchanberta-base-att-spm-uncased-tagging" # nosec B615 ) ) @@ -97,7 +97,7 @@ def evaluate_one_text(model: BertModel, sentence: str) -> list[str]: mlm_model: "AutoModelForMaskedLM" = AutoModelForMaskedLM.from_pretrained( - "bookpanda/wangchanberta-base-att-spm-uncased-masking" + "bookpanda/wangchanberta-base-att-spm-uncased-masking" # nosec B615 ) if use_cuda: mlm_model = mlm_model.to(device=device) diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index 0a5a843d5..6d376105c 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -5,6 +5,8 @@ from __future__ import annotations +from typing import Optional + from pythainlp.summarize import CPE_KMUTT_THAI_SENTENCE_SUM @@ -18,6 +20,7 @@ def __init__( max_length: int = 100, skip_special_tokens: bool = True, pretrained_mt5_model_name: str = "", + revision: Optional[str] = None, ) -> None: """Initialize mT5 Summarizer. @@ -34,6 +37,8 @@ def __init__( output. Default is True. :param str pretrained_mt5_model_name: Name of pretrained model. If empty (default), uses google/mt5-{model_size}. + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ from transformers import MT5ForConditionalGeneration, T5Tokenizer @@ -52,9 +57,13 @@ def __init__( model_name = pretrained_mt5_model_name self.model_name: str = model_name self.model: MT5ForConditionalGeneration = ( - MT5ForConditionalGeneration.from_pretrained(model_name) + MT5ForConditionalGeneration.from_pretrained( + model_name, revision=revision + ) + ) + self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained( + model_name, revision=revision ) - self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(model_name) self.num_beams: int = num_beams self.no_repeat_ngram_size: int = no_repeat_ngram_size self.min_length: int = min_length diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index 5c56f7867..dd708c84a 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import Optional + def pos_tag( words: list[str], engine: str = "perceptron", corpus: str = "orchid" @@ -168,6 +170,7 @@ def pos_tag_transformers( sentence: str, engine: str = "bert", corpus: str = "blackboard", + revision: Optional[str] = None, ) -> list[list[tuple[str, str]]]: """Marks sentences with part-of-speech (POS) tags. @@ -189,6 +192,8 @@ def pos_tag_transformers( `_ \ treebanks, natively use Universal POS tags \ (support wangchanberta and mdeberta engine) + :param Optional[str] revision: a git revision id (branch, tag, or commit + hash) for the model. Pin to a full commit hash for secure downloads. :return: a list of lists of tuples (word, POS tag) :rtype: list[list[tuple[str, str]]] @@ -228,12 +233,16 @@ def pos_tag_transformers( if corpus == "blackboard" and engine in _blackboard_support_engine.keys(): base_model = _blackboard_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained(base_model) - tokenizer = AutoTokenizer.from_pretrained(base_model) + model = AutoModelForTokenClassification.from_pretrained( + base_model, revision=revision + ) + tokenizer = AutoTokenizer.from_pretrained(base_model, revision=revision) elif corpus == "pud" and engine in _pud_support_engine.keys(): base_model = _pud_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained(base_model) - tokenizer = AutoTokenizer.from_pretrained(base_model) + model = AutoModelForTokenClassification.from_pretrained( + base_model, revision=revision + ) + tokenizer = AutoTokenizer.from_pretrained(base_model, revision=revision) else: raise ValueError( f"pos_tag_transformers not support {engine} engine or {corpus} corpus." diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index a1a822ea0..3a96470c1 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -56,7 +56,7 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: is_span_beginning = pos >= match.start() token = segments[segment_idx] if is_span_beginning: - connected_token = "" + connected_token = "" # nosec B105 while pos < match.end() and segment_idx < len(segments): connected_token += segments[segment_idx] pos += len(segments[segment_idx]) diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index d974dcd06..c8580e60e 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -135,7 +135,8 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: 'ภาษาไทยปรากฏครั้งแรกในกุทธศักราช 1727' """ num_misspells = math.floor(len(sentence) * ratio) - positions = random.sample(range(len(sentence)), k=num_misspells) + # Non-cryptographic use, pseudo-random generator is acceptable here + positions = random.sample(range(len(sentence)), k=num_misspells) # noqa: S311 # nosec B311 # NOSONAR # convert strings to array of characters misspelled = list(sentence) @@ -145,7 +146,7 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: continue # Non-cryptographic use, pseudo-random generator is acceptable here - candidate = random.choice(potential_candidates) # noqa: S311 + candidate = random.choice(potential_candidates) # noqa: S311 # nosec B311 # NOSONAR misspelled[pos] = candidate diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index ce87a4c8b..ce641f4f6 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -30,12 +30,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "alirezamsh/small100", + revision: Optional[str] = None, ) -> None: from transformers import M2M100ForConditionalGeneration self.pretrained: str = pretrained self.model: "M2M100ForConditionalGeneration" = ( - M2M100ForConditionalGeneration.from_pretrained(self.pretrained) + M2M100ForConditionalGeneration.from_pretrained( + self.pretrained, revision=revision + ) ) self.tgt_lang: Optional[str] = None if use_gpu: diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 156b35709..a980f1948 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -43,14 +43,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Helsinki-NLP/opus-mt-th-fr", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_thfr: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_thfr: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_thfr = self.model_thfr.cuda() diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index e40cbd1a4..cabd77b0a 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -149,11 +149,11 @@ def __init__( vocab_file: str, spm_file: str, tgt_lang: Optional[str] = None, - bos_token: str = "", # noqa: S107 - eos_token: str = "", # noqa: S107 - sep_token: str = "", # noqa: S107 - pad_token: str = "", # noqa: S107 - unk_token: str = "", # noqa: S107 + bos_token: str = "", # noqa: S107 # nosec B107 + eos_token: str = "", # noqa: S107 # nosec B107 + sep_token: str = "", # noqa: S107 # nosec B107 + pad_token: str = "", # noqa: S107 # nosec B107 + unk_token: str = "", # noqa: S107 # nosec B107 language_codes: str = "m2m100", sp_model_kwargs: Optional[dict[str, str]] = None, num_madeup_words: int = 8, diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 54a035454..534dee00a 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -33,14 +33,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Lalita/marianmt-th-zh_cn", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_thzh: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_thzh: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_thzh = self.model_thzh.cuda() @@ -108,14 +109,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Lalita/marianmt-zh_cn-th", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_zhth: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_zhth: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_zhth = self.model_zhth.cuda() diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index f0b21d8c0..3c9f1fb4c 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -4,13 +4,19 @@ """Transliteration.""" __all__: list[str] = [ + "get_word_dict", "pronunciate", "puan", "romanize", "transliterate", + "transliterate_wiktionary", "pronunciate_pali", ] from pythainlp.transliterate.core import pronunciate, romanize, transliterate from pythainlp.transliterate.pali import pronunciate_pali from pythainlp.transliterate.spoonerism import puan +from pythainlp.transliterate.wiktionary import ( + get_word_dict, + transliterate_wiktionary, +) diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 52ab6a28e..369fec1a0 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -49,7 +49,7 @@ def __init__(self) -> None: f" CLI: thainlp data get {_MODEL_NAME}" ) - loader = torch.load(self.__model_filename, map_location=device) + loader = torch.load(self.__model_filename, map_location=device) # nosec B614 INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] @@ -419,7 +419,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 # NOSONAR if teacher_force and target_seq is not None: decoder_input = target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index f1438198d..a2df19022 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -58,7 +58,7 @@ def __init__(self) -> None: f" CLI: thainlp data get {_MODEL_NAME}" ) - loader = torch.load(self.__model_filename, map_location=device) + loader = torch.load(self.__model_filename, map_location=device) # nosec B614 INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] @@ -438,7 +438,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 # NOSONAR decoder_input = ( target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/transliterate/wiktionary.py b/pythainlp/transliterate/wiktionary.py new file mode 100644 index 000000000..122e42321 --- /dev/null +++ b/pythainlp/transliterate/wiktionary.py @@ -0,0 +1,448 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Thai pronunciation transliteration from Wiktionary th-pron module. +Source code: https://en.wiktionary.org/wiki/Module:th-pron +""" + +from __future__ import annotations + +import re +import unicodedata +from typing import cast + +_THAI_RANGE: str = r"[ก-๛̄]" + +_SYSTEMS: dict[str, int] = { + "paiboon": 0, + "royin": 1, + "ipa": 2, +} + +_INITIAL: dict[str, dict[str, list[str] | str]] = { + "ก": {"seq": ["g", "k", "k"], "class": "mid"}, + "จ": {"seq": ["j", "ch", "t͡ɕ"], "class": "mid"}, + "ด": {"seq": ["d", "d", "d"], "class": "mid"}, + "ฎ": {"seq": ["d", "d", "d"], "class": "mid"}, + "ฏ": {"seq": ["dt", "t", "t"], "class": "mid"}, + "ต": {"seq": ["dt", "t", "t"], "class": "mid"}, + "บ": {"seq": ["b", "b", "b"], "class": "mid"}, + "ป": {"seq": ["bp", "p", "p"], "class": "mid"}, + "อ": {"seq": ["", "@", "ʔ"], "class": "mid"}, + "ง": {"seq": ["ng", "$ng", "ŋ"], "class": "low"}, + "ณ": {"seq": ["n", "n", "n"], "class": "low"}, + "น": {"seq": ["n", "n", "n"], "class": "low"}, + "ม": {"seq": ["m", "m", "m"], "class": "low"}, + "ญ": {"seq": ["y", "y", "j"], "class": "low"}, + "ย": {"seq": ["y", "y", "j"], "class": "low"}, + "ร": {"seq": ["r", "r", "r"], "class": "low"}, + "ล": {"seq": ["l", "l", "l"], "class": "low"}, + "ฬ": {"seq": ["l", "l", "l"], "class": "low"}, + "ว": {"seq": ["w", "w", "w"], "class": "low"}, + "ค": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ฅ": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ฆ": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ข": {"seq": ["k", "kh", "kʰ"], "class": "high"}, + "ฃ": {"seq": ["k", "kh", "kʰ"], "class": "high"}, + "ช": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "low"}, + "ฌ": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "low"}, + "ฉ": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "high"}, + "ฑ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ฒ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ท": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ธ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ฐ": {"seq": ["t", "th", "tʰ"], "class": "high"}, + "ถ": {"seq": ["t", "th", "tʰ"], "class": "high"}, + "พ": {"seq": ["p", "ph", "pʰ"], "class": "low"}, + "ภ": {"seq": ["p", "ph", "pʰ"], "class": "low"}, + "ผ": {"seq": ["p", "ph", "pʰ"], "class": "high"}, + "ฟ": {"seq": ["f", "f", "f"], "class": "low"}, + "ฝ": {"seq": ["f", "f", "f"], "class": "high"}, + "ซ": {"seq": ["s", "s", "s"], "class": "low"}, + "ศ": {"seq": ["s", "s", "s"], "class": "high"}, + "ษ": {"seq": ["s", "s", "s"], "class": "high"}, + "ส": {"seq": ["s", "s", "s"], "class": "high"}, + "ฮ": {"seq": ["h", "h", "h"], "class": "low"}, + "ห": {"seq": ["h", "h", "h"], "class": "high"}, + "หง": {"seq": ["ng", "$ng", "ŋ"], "class": "high"}, + "หน": {"seq": ["n", "n", "n"], "class": "high"}, + "หม": {"seq": ["m", "m", "m"], "class": "high"}, + "หญ": {"seq": ["y", "y", "j"], "class": "high"}, + "หย": {"seq": ["y", "y", "j"], "class": "high"}, + "หร": {"seq": ["r", "r", "r"], "class": "high"}, + "หล": {"seq": ["l", "l", "l"], "class": "high"}, + "หว": {"seq": ["w", "w", "w"], "class": "high"}, + "…": {"seq": ["…", "…", "…"], "class": ""}, + "": {"seq": ["", "", ""], "class": ""}, +} + +_VOWEL: dict[str, dict[str, list[str]]] = { + "open": { + "ะ": ["a", "a", "a"], + "": ["a", "a", "a"], + "ิ": ["i", "i", "i"], + "ึ": ["ʉ", "ue", "ɯ"], + "ุ": ["u", "u", "u"], + "เะ": ["e", "e", "eʔ"], + "แะ": ["ɛ", "ae", "ɛʔ"], + "โะ": ["o", "o", "oʔ"], + "เาะ": ["ɔ", "o", "ɔʔ"], + "็": ["ɔ", "o", "ɔ"], + "เิ": ["ə", "oe", "ɤ"], + "เอะ": ["ə", "oe", "ɤʔ"], + "า": ["aa", "a", "aː"], + "ี": ["ii", "i", "iː"], + "ู": ["uu", "u", "uː"], + "ือ": ["ʉʉ", "ue", "ɯː"], + "เ": ["ee", "e", "eː"], + "แ": ["ɛɛ", "ae", "ɛː"], + "โ": ["oo", "o", "oː"], + "อ": ["ɔɔ", "o", "ɔː"], + "ร": ["ɔɔn", "on", "ɔːn"], + "เอ": ["əə", "oe", "ɤː"], + "เียะ": ["ia", "ia", "ia̯ʔ"], + "เือะ": ["ʉa", "uea", "ɯa̯ʔ"], + "ัวะ": ["ua", "ua", "ua̯ʔ"], + "เีย": ["iia", "ia", "ia̯"], + "เือ": ["ʉʉa", "uea", "ɯa̯"], + "ัว": ["uua", "ua", "ua̯"], + "ิว": ["iu", "io", "iw"], + "ีว": ["iiu", "io", "iːw"], + "เ็ว": ["eo", "eo", "ew"], + "แ็ว": ["ɛo", "aeo", "ɛw"], + "เา": ["ao", "ao", "aw"], + "เว": ["eeo", "eo", "eːw"], + "แว": ["ɛɛo", "aeo", "ɛːw"], + "าว": ["aao", "ao", "aːw"], + "เอว": ["əəo", "oeu", "ɤːw"], + "โว": ["oow", "ou", "oːw"], + "เียว": ["iao", "iao", "ia̯w"], + "ัย": ["ai", "ai", "aj"], + "ใ": ["ai", "ai", "aj"], + "ไ": ["ai", "ai", "aj"], + "ไย": ["ai", "ai", "aj"], + "ึย": ["ʉi", "uei", "ɯj"], + "็อย": ["ɔi", "oi", "ɔj"], + "เิ็ย": ["əi", "oei", "ɤj"], + "ุย": ["ui", "ui", "uj"], + "าย": ["aai", "ai", "aːj"], + "อย": ["ɔɔi", "oi", "ɔːj"], + "โย": ["ooi", "oi", "oːj"], + "เย": ["əəi", "oei", "ɤːj"], + "ูย": ["uui", "ui", "uːj"], + "วย": ["uai", "uai", "ua̯j"], + "เือย": ["ʉai", "ueai", "ɯa̯j"], + "ำ": ["am", "am", "am"], + }, + "closed": { + "ั": ["a", "a", "a"], + "รร": ["a", "a", "a"], + "ิ": ["i", "i", "i"], + "ึ": ["ʉ", "ue", "ɯ"], + "ุ": ["u", "u", "u"], + "เ": ["ee", "e", "eː"], + "เ็": ["e", "e", "e"], + "แ็": ["ɛ", "ae", "ɛ"], + "แ": ["ɛɛ", "ae", "ɛː"], + "": ["o", "o", "o"], + "็อ": ["ɔ", "o", "ɔ"], + "เิ็": ["ə", "oe", "ɤ"], + "า": ["aa", "a", "aː"], + "ี": ["ii", "i", "iː"], + "ื": ["ʉʉ", "ue", "ɯː"], + "ู": ["uu", "u", "uː"], + "โ": ["oo", "o", "oː"], + "อ": ["ɔɔ", "o", "ɔː"], + "เิ": ["əə", "oe", "ɤː"], + "เอ": ["əə", "oe", "ɤː"], + "เีย": ["iia", "ia", "ia̯"], + "เือ": ["ʉʉa", "uea", "ɯa̯"], + "ว": ["uua", "ua", "ua̯"], + "ไ": ["ai", "ai", "aj"], + "เา": ["ao", "ao", "aw"], + "็อย": ["ɔi", "oi", "ɔj"], + }, +} + +_UNROM_LONG: dict[str, bool] = { + "เีย": True, + "เือ": True, + "ัว": True, + "ว": True, + "เือย": True, + "วาย": True, + "เอว": True, + "เียว": True, +} + +_LIVE_EXC: dict[str, bool] = { + "ัย": True, + "ใ": True, + "ไ": True, + "ไย": True, + "ุย": True, + "วย": True, + "็อย": True, + "เิ็ย": True, + "เา": True, + "ิว": True, + "เ็ว": True, + "แ็ว": True, + "ำ": True, +} + +_CODA: dict[str, list[str]] = { + "ก": ["k", "k", "k̚"], + "ข": ["k", "k", "k̚"], + "ฃ": ["k", "k", "k̚"], + "ค": ["k", "k", "k̚"], + "ฅ": ["k", "k", "k̚"], + "ฆ": ["k", "k", "k̚"], + "จ": ["t", "t", "t̚"], + "ฉ": ["t", "t", "t̚"], + "ช": ["ch", "ch", "t͡ɕʰ"], + "ซ": ["s", "s", "s"], + "ฌ": ["t", "t", "t̚"], + "ฎ": ["t", "t", "t̚"], + "ฏ": ["t", "t", "t̚"], + "ฐ": ["t", "t", "t̚"], + "ฑ": ["t", "t", "t̚"], + "ฒ": ["t", "t", "t̚"], + "ด": ["t", "t", "t̚"], + "ต": ["t", "t", "t̚"], + "ถ": ["t", "t", "t̚"], + "ท": ["t", "t", "t̚"], + "ธ": ["t", "t", "t̚"], + "ศ": ["t", "t", "t̚"], + "ษ": ["t", "t", "t̚"], + "ส": ["s", "s", "s"], + "บ": ["p", "p", "p̚"], + "ป": ["p", "p", "p̚"], + "ผ": ["p", "p", "p̚"], + "ฝ": ["p", "p", "p̚"], + "พ": ["p", "p", "p̚"], + "ฟ": ["f", "f", "f"], + "ภ": ["p", "p", "p̚"], + "ง": ["ng", "ng$", "ŋ"], + "ญ": ["n", "n", "n"], + "ณ": ["n", "n", "n"], + "น": ["n", "n", "n"], + "ร": ["n", "n", "n"], + "ล": ["l", "l", "l"], + "ฬ": ["n", "n", "n"], + "ม": ["m", "m", "m"], + "ฯ": ["ʔ", "ʔ", "ʔ"], +} + +_TONE_FROM_MARK: dict[str, dict[str, str]] = { + "่": {"high": "low", "mid": "low", "low": "falling"}, + "้": {"high": "falling", "mid": "falling", "low": "high"}, + "๊": {"high": "high", "mid": "high", "low": "high"}, + "๋": {"high": "rising", "mid": "rising", "low": "rising"}, + "̄": {"high": "mid", "mid": "mid", "low": "mid"}, +} + +_TONE_NO_MARK: dict[str, dict[str, str]] = { + "dead-short": {"high": "low", "mid": "low", "low": "high"}, + "dead-long": {"high": "low", "mid": "low", "low": "falling"}, + "live": {"high": "rising", "mid": "mid", "low": "mid"}, +} + +_TONE_ROM_MARKS: dict[str, str] = { + "high": "́", + "mid": "", + "low": "̀", + "rising": "̌", + "falling": "̂", +} + +_TONE_LEVELS: dict[str, str] = { + "high": "˦˥", + "mid": "˧", + "low": "˨˩", + "rising": "˩˩˦", + "falling": "˥˩", +} + +_SYMBOLS: dict[str, str] = { + "๐": "0", + "๑": "1", + "๒": "2", + "๓": "3", + "๔": "4", + "๕": "5", + "๖": "6", + "๗": "7", + "๘": "8", + "๙": "9", +} + +_MGVC_PATTERN = re.compile( + r"^([รลว]?)([ิึุ็ีืัำู]?[าอรยว]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) +_FULL_PATTERN = re.compile( + r"^([เแโใไ]?)(หฺ[ก-รลว-ฮ])(ฺ?[รลว]?)([ิึุ็ีืัู]?็?[่้๊๋̄]?[าอรยวำ]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?[คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) +_PARTIAL_PATTERN = re.compile( + r"^([เแโใไ]?)([ก-รลว-ฮ])(ฺ?[รลว]?)([ิึุ็ีืัู]?็?[่้๊๋̄]?[าอรยวำ]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?[คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) + + +def _c2_decomp(c2_char: str, seq_idx: int) -> str: + return "".join(_CODA.get(char, ["", "", ""])[seq_idx] for char in c2_char) + + +def th_pron_transliterate(text: str, mode: str = "ipa") -> str: + """Transliterate Thai text using Wiktionary th-pron logic. + + :param str text: Thai text input (single word or text fragment). + :param str mode: Output mode: ``paiboon``, ``royin``, or ``ipa``. + Unsupported modes return the input text unchanged. + + :return: Transliterated text. + :rtype: str + + :Example: + + >>> th_pron_transliterate("แมว", mode="royin") + 'maeo' + """ + seq_idx = _SYSTEMS.get(mode) + if seq_idx is None: + return text + + def process_word(match_word: re.Match[str]) -> str: + word = match_word.group(0) + + if re.search(r"[่้๊๋̄].?[่้๊๋̄]", word): + return word + + def syllable(match: re.Match[str]) -> str: + v1, c1, g, v2, c2 = match.groups() + + tmark_match = re.search(r"[่้๊๋̄]", v2) + tmark = tmark_match.group(0) if tmark_match else None + v2 = re.sub(r"[่้๊๋̄]", "", v2) + + if re.match(r"^ห.$", c1): + mgvc_match = _MGVC_PATTERN.match(c1[1] + g + v2 + c2) + if mgvc_match: + g_new, v2_new, c2_new = mgvc_match.groups() + c1, g, v2, c2 = "ห", g_new, v2_new, c2_new + if g and v2 != "ย": + c1, g = c1 + g, "" + + if g == "ล" and not (v2 + c2): + c2 = g + g = "" + + openness = "closed" if c2 != "" else "open" + + if (v1 + g + v2) in _VOWEL[openness]: + orig_v = v1 + g + v2 + v = _VOWEL[openness][orig_v][seq_idx] + g = "" + else: + orig_v = v1 + v2 + v_lookup = _VOWEL[openness].get(v1 + v2) + v = v_lookup[seq_idx] if v_lookup else (v1 + v2) + g_clean = g.replace("ฺ", "") + g_lookup = _INITIAL.get(g_clean, _INITIAL[""]) + g = cast(list[str], g_lookup["seq"])[seq_idx] + + c1_clean = c1.replace("ฺ", "") + if c1_clean in _INITIAL: + ini = cast(list[str], _INITIAL[c1_clean]["seq"])[seq_idx] + cls = cast(str, _INITIAL[c1_clean]["class"]) + else: + return match.group(0) + + length = ( + "long" + if re.search(r"([aiʉueɛoɔə])\1", v) + or "ː" in v + or orig_v in _UNROM_LONG + else "short" + ) + life = ( + "live" + if re.search(r"[มญณนรลฬง]", c2) + or (orig_v.endswith("ย") and v.endswith("i")) + or (c2 == "" and length == "long") + or _LIVE_EXC.get(orig_v) + else "dead" + ) + + if c2 in _CODA: + c2 = _CODA[c2][seq_idx] + else: + c2 = _c2_decomp(c2, seq_idx) + + tone_dict = ( + _TONE_FROM_MARK.get(tmark) + if tmark + else _TONE_NO_MARK.get(f"{life}-{length}", _TONE_NO_MARK.get(life)) + ) + tone = tone_dict.get(cls) if tone_dict else None + + if mode == "paiboon": + v = re.sub( + r"^([^aiʉueɛoɔə]*)([aiʉueɛoɔə])", + f"\\g<1>\\g<2>{_TONE_ROM_MARKS.get(tone, '')}", + v, + ) + elif mode == "ipa": + c2 = c2 + _TONE_LEVELS.get(tone, "") + + return ini + g + v + c2 + + word = _FULL_PATTERN.sub(syllable, word) + word = _PARTIAL_PATTERN.sub(syllable, word) + return word + + text = re.sub(f"{_THAI_RANGE}+", lambda m: process_word(m), text) + + text = re.sub(r"[๐-๙]", lambda m: _SYMBOLS.get(m.group(0), m.group(0)), text) + + if mode == "royin": + text = re.sub(r"^@", "", text) + text = re.sub(r"([\s\W])@", r"\1", text) + text = text.replace("@", "-") + text = re.sub(r"^\$ng", "ng", text) + text = re.sub(r"([\s\W])\$ng", r"\1ng", text) + text = re.sub(r"([aeiou])\$ng", r"\1-ng", text) + text = text.replace("$ng", "ng") + text = re.sub(r"ng\$([^\w\s])", r"ng\1", text) + text = re.sub(r"ng\$", "ng", text) + + if mode == "ipa": + text = re.sub(r"[ \-–]", ".", text) + text = re.sub(r"([aiɯu])([˥-˩]+)$", r"\1ʔ\2", text) + + return unicodedata.normalize("NFC", text) + + +def get_word_dict(word: str) -> dict[str, str]: + """Return Wiktionary transliteration outputs in all supported systems. + + :param str word: Thai input word. + :return: ``dict[str, str]`` with ``word``, ``paiboon``, ``royin``, and ``ipa``. + :rtype: dict[str, str] + + :Example: + + >>> get_word_dict("แมว") + {'word': 'แมว', 'paiboon': 'mɛɛo', 'royin': 'maeo', 'ipa': 'mɛːw˧'} + """ + return { + "word": word, + "paiboon": th_pron_transliterate(word, mode="paiboon"), + "royin": th_pron_transliterate(word, mode="royin"), + "ipa": th_pron_transliterate(word, mode="ipa"), + } + + +def transliterate_wiktionary(text: str, mode: str = "ipa") -> str: + """Backward-compatible alias for :func:`th_pron_transliterate`.""" + return th_pron_transliterate(text, mode=mode) diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index cb1474381..f8745e52a 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -28,7 +28,7 @@ def _get_tokenizer() -> CamembertTokenizer: from transformers import CamembertTokenizer _tokenizer = CamembertTokenizer.from_pretrained( - f"airesearch/{_model_name}", revision="main" + f"airesearch/{_model_name}", revision="main" # nosec B615 ) if _model_name == "wangchanberta-base-att-spm-uncased": _tokenizer.additional_special_tokens = [ @@ -164,21 +164,27 @@ class NamedEntityRecognition: model: PreTrainedModel def __init__( - self, model: str = "pythainlp/thainer-corpus-v2-base-model" + self, + model: str = "pythainlp/thainer-corpus-v2-base-model", + revision: Optional[str] = None, ) -> None: """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand :param str model: The model that use wangchanberta pretrained. + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ from transformers import AutoModelForTokenClassification, AutoTokenizer self.tokenizer: PreTrainedTokenizerBase = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: PreTrainedModel = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def _fix_span_error( diff --git a/tests/core/__init__.py b/tests/core/__init__.py index b4d67630f..66ca8a714 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -24,6 +24,7 @@ "tests.core.test_tokenize", "tests.core.test_tools", "tests.core.test_transliterate", + "tests.core.test_transliterate_wiktionary", "tests.core.test_util", ] diff --git a/tests/core/test_transliterate_wiktionary.py b/tests/core/test_transliterate_wiktionary.py new file mode 100644 index 000000000..d7c19d2a9 --- /dev/null +++ b/tests/core/test_transliterate_wiktionary.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from pythainlp.transliterate import get_word_dict +from pythainlp.transliterate.wiktionary import transliterate_wiktionary + + +class TransliterateThPronTestCase(unittest.TestCase): + def test_transliterate_wiktionary(self): + self.assertEqual( + transliterate_wiktionary("แมว", mode="royin"), "maeo" + ) + self.assertEqual( + transliterate_wiktionary("คน", mode="paiboon"), "kon" + ) + self.assertEqual( + transliterate_wiktionary("คน", mode="ipa"), "kʰon˧") + + def test_transliterate_wiktionary_returns_input_for_unknown_mode(self): + self.assertEqual(transliterate_wiktionary("แมว", mode="unknown"), "แมว") + + def test_thai_digits_conversion(self): + self.assertEqual(transliterate_wiktionary("๑๒๓", mode="royin"), "123") + + def test_get_word_dict(self): + self.assertEqual( + get_word_dict("ปฺระ-ชา"), + { + "word": "ปฺระ-ชา", + "paiboon": "bprà-chaa", + "royin": "pra-cha", + "ipa": "pra˨˩.t͡ɕʰaː˧", + }, + ) + + def test_transliterate_wiktionary_paiboon_cases(self): + # test cases from https://w.wiki/RifX + test_cases = [ + ('เฮิ็้ย', 'hə́i'), + ('เฉิ็ย', 'chə̌i'), + ('เงิ็น', 'ngən'), + ('เกดส', 'gèets'), # เกตส์ (Gates) + ('มันส', 'mans'), # มันส์ + ('ไอ๊ส', 'áis'), # ไอซ์ (ice) + ('เซ็กส', 'séks'), # เซ็กส์ (sex) + ('เอ็๊กส', 'éks'), # เอกซ์ & เอ็กซ์ & เอ๊กซ์ (ex) + ('เฮ้าส', 'háos'), # เฮาส์ & เฮ้าส์ (house) + ('เม้าส', 'máos'), # เมาส์ & เม้าส์ (mouse) + ('ทฺรำ-เป็ด', 'tram-bpèt'), # ทรัมเป็ต + ('ห็อย', 'hɔ̌i'), + ('หฺม็อย', 'mɔ̌i'), + ('หฺมั่น-โถว', 'màn-tǒow'), + ('เด๊ด-สะ-มอ-เร่', 'déet-sà-mɔɔ-rêe'), + ('เห', 'hěe'), + ('แคฺล', 'klɛɛ'), + ('แคล', 'kɛɛl'), + ('เพฺล', 'plee'), + ('เพล', 'peel'), + ('เปฺล', 'bplee'), + ('เปล', 'bpeel'), + ('เบล', 'beel'), + ('เซล', 'seel'), + ('โพล', 'pool'), + ('รา-ชา-ทิ-ราด', 'raa-chaa-tí-râat'), + ('ขฺวน-ขฺวาย', 'kwǒn-kwǎai'), # ขวนขวาย Only the word ขวน read as kwǒn instead of kǔuan. + ('ข่วน', 'kùuan'), + ('หอน', 'hɔ̌ɔn'), + ('โหน', 'hǒon'), # ห้อยโหน homograph issue + ('สะ-โหฺน', 'sà-nǒo'), # โสน homograph issue + ('แหน', 'hɛ̌ɛn'), # หวงแหน homograph issue + ('แหฺน', 'nɛ̌ɛ'), # จอกแหน homograph issue + ('แถ็ว', 'tɛ̌o'), # แถว + ('ซวง', 'suuang'), + ('น้ำ', 'nám'), + ('หฺมาย', 'mǎai'), + ('แห็่ง', 'hɛ̀ng'), + ('หน', 'hǒn'), + ('เหด-สุด-วิ-ไส', 'hèet-sùt-wí-sǎi'), + ('ไหฺย่', 'yài'), + ('หก', 'hòk'), + ('หอย', 'hɔ̌ɔi'), + ('กับ', 'gàp'), + ('ธรรม', 'tam'), + ('ปฺระ-ชา', 'bprà-chaa'), + ('นะ-คอน', 'ná-kɔɔn'), + ('บาด', 'bàat'), + ('บ้า', 'bâa'), + ('แข็ง', 'kɛ̌ng'), + ('แกะ', 'gɛ̀'), + ('แดง', 'dɛɛng'), + ('แปฺล', 'bplɛɛ'), + ('ผฺล็อง', 'plɔ̌ng'), + ('เกาะ', 'gɔ̀'), + ('นอน', 'nɔɔn'), + ('พ่อ', 'pɔ̂ɔ'), + ('เห็ด', 'hèt'), + ('เล็่น', 'lên'), + ('เตะ', 'dtè'), + ('เพฺลง', 'pleeng'), + ('เท-วี', 'tee-wii'), + ('เยอะ', 'yə́'), + ('เดิน', 'dəən'), + ('เผฺลอ', 'plə̌ə'), + ('ตก', 'dtòk'), + ('โต๊ะ', 'dtó'), + ('โชค', 'chôok'), + ('โม-โห', 'moo-hǒo'), + ('คิด', 'kít'), + ('มิ-ถุน', 'mí-tǔn'), + ('หิ-มะ', 'hì-má'), + ('อีก', 'ìik'), + ('จี้', 'jîi'), + ('ลึก', 'lʉ́k'), + ('รึ', 'rʉ́'), + ('กฺลืน', 'glʉʉn'), + ('ชื่อ', 'chʉ̂ʉ'), + ('คุก', 'kúk'), + ('จุ-ฬา', 'jù-laa'), + ('ลูก', 'lûuk'), + ('ปู', 'bpuu'), + ('เดี๊ยะ', 'día'), + ('เปาะ-เปี๊ยะ', 'bpɔ̀-bpía'), + ('ปอ-เปี๊ยะ', 'bpɔɔ-bpía'), + ('เปฺรี๊ยะ', 'bpría'), + ('เตียง', 'dtiiang'), + ('เมีย', 'miia'), + ('เอือะ', 'ʉ̀a'), + ('เรื่อง', 'rʉ̂ʉang'), + ('เรือ', 'rʉʉa'), + ('ผฺลัวะ', 'plùa'), + ('นวด', 'nûuat'), + ('ตัว', 'dtuua'), + ('ไม่', 'mâi'), + ('ใส่', 'sài'), + ('วัย', 'wai'), + ('ไทย', 'tai'), + ('ไม้', 'mái'), + ('หาย', 'hǎai'), + ('ผฺล็อย', 'plɔ̌i'), + ('ซอย', 'sɔɔi'), + ('เลย', 'ləəi'), + ('โดย', 'dooi'), + ('ทุย', 'tui'), + ('เหฺนื่อย', 'nʉ̀ai'), + ('สวย', 'sǔai'), + ('เรา', 'rao'), + ('ขาว', 'kǎao'), + ('แมว', 'mɛɛo'), + ('เกอว', 'gəəo'), + ('เร็ว', 'reo'), + ('เอว', 'eeo'), + ('หิว', 'hǐu'), + ('เขียว', 'kǐao'), + ('ทำ', 'tam'), + ] + + for thai_text, expected_translit in test_cases: + with self.subTest(thai_text=thai_text): + self.assertEqual( + transliterate_wiktionary(thai_text, mode="paiboon"), + expected_translit + ) + +if __name__ == '__main__': + unittest.main()