From 54c4a29eb26bb3423c2e80d75171e04ca1efe07f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 14:55:00 +0000 Subject: [PATCH 01/32] Bump dieghernan/cff-validator from 4 to 5 Bumps [dieghernan/cff-validator](https://github.com/dieghernan/cff-validator) from 4 to 5. - [Release notes](https://github.com/dieghernan/cff-validator/releases) - [Changelog](https://github.com/dieghernan/cff-validator/blob/main/CHANGELOG.md) - [Commits](https://github.com/dieghernan/cff-validator/compare/114aae53e1850c3757733beb60036941900e3dc3...54c8e737314b948f25c0e5524a9d8ae3f39b207b) --- updated-dependencies: - dependency-name: dieghernan/cff-validator dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codemeta2cff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index e035dbf68..97ddc1e07 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -37,7 +37,7 @@ jobs: printf '\n```' >> $GITHUB_STEP_SUMMARY - name: Validate CITATION.cff - uses: dieghernan/cff-validator@114aae53e1850c3757733beb60036941900e3dc3 #v4 + uses: dieghernan/cff-validator@54c8e737314b948f25c0e5524a9d8ae3f39b207b #v5.0.1 - name: Commit and push updated CITATION.cff uses: EndBug/add-and-commit@290ea2c423ad77ca9c62ae0f5b224379612c0321 #v10.0.0 From fe467c9e356ad41f75dce3676e114d874d2c5a52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 May 2026 14:55:15 +0000 Subject: [PATCH 02/32] Bump DavidAnson/markdownlint-cli2-action from 23.1.0 to 23.2.0 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 23.1.0 to 23.2.0. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v23.1.0...v23.2.0) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-version: 23.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/markdownlint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml index d7dd798e9..178437f16 100644 --- a/.github/workflows/markdownlint.yml +++ b/.github/workflows/markdownlint.yml @@ -28,7 +28,7 @@ jobs: with: fetch-depth: 0 - - uses: DavidAnson/markdownlint-cli2-action@v23.1.0 + - uses: DavidAnson/markdownlint-cli2-action@v23.2.0 with: globs: | **/*.md From b6ad607af3b9a157961a8777c76a0376aa0db314 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 May 2026 14:56:31 +0000 Subject: [PATCH 03/32] Bump github/codeql-action from 4 to 4.35.4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4 to 4.35.4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4...v4.35.4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.35.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c613ade92..958bc3259 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@v4.35.4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4 + uses: github/codeql-action/autobuild@v4.35.4 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@v4.35.4 From 9c6b6deb78746d31747c949f950c8a8bb474bf83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 16:24:49 +0000 Subject: [PATCH 04/32] Bump actions/stale from 10 to 10.2.0 Bumps [actions/stale](https://github.com/actions/stale) from 10 to 10.2.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v10...v10.2.0) --- updated-dependencies: - dependency-name: actions/stale dependency-version: 10.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f8d6c9770..d58d8a9aa 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v10 + uses: actions/stale@v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' From a00adc36d33e5046b3a06a66048d5fd3d4191223 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:00:14 +0000 Subject: [PATCH 05/32] Bump actions/stale from 10.2.0 to 10.3.0 Bumps [actions/stale](https://github.com/actions/stale) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v10.2.0...v10.3.0) --- updated-dependencies: - dependency-name: actions/stale dependency-version: 10.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d58d8a9aa..9566cebb0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v10.2.0 + uses: actions/stale@v10.3.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' From 2523fe5f0fc67897716c8edfee05c88f18d3dcbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 May 2026 17:01:42 +0000 Subject: [PATCH 06/32] Bump github/codeql-action from 4.35.4 to 4.35.5 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.35.4 to 4.35.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.35.4...v4.35.5) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.35.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 958bc3259..03d896670 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.35.4 + uses: github/codeql-action/init@v4.35.5 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.35.4 + uses: github/codeql-action/autobuild@v4.35.5 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.35.4 + uses: github/codeql-action/analyze@v4.35.5 From 8a0ced6f010d081f3d00ac8924eeca12e46fc31d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:54:06 +0000 Subject: [PATCH 07/32] Bump github/codeql-action from 4.35.5 to 4.36.0 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.35.5 to 4.36.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.35.5...v4.36.0) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.36.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 03d896670..b553cf8ec 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.35.5 + uses: github/codeql-action/init@v4.36.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.35.5 + uses: github/codeql-action/autobuild@v4.36.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.35.5 + uses: github/codeql-action/analyze@v4.36.0 From 6183ac0fe70dfa5a9af9cbc361b0c096df063a34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:54:12 +0000 Subject: [PATCH 08/32] Bump peaceiris/actions-gh-pages from 4 to 4.1.0 Bumps [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) from 4 to 4.1.0. - [Release notes](https://github.com/peaceiris/actions-gh-pages/releases) - [Changelog](https://github.com/peaceiris/actions-gh-pages/blob/main/CHANGELOG.md) - [Commits](https://github.com/peaceiris/actions-gh-pages/compare/v4...v4.1.0) --- updated-dependencies: - dependency-name: peaceiris/actions-gh-pages dependency-version: 4.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 749c17e54..cf9cfc3b7 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -38,7 +38,7 @@ jobs: cd docs && make html cd .. - name: Deploy documentation - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@v4.1.0 with: personal_token: ${{ secrets.PERSONAL_TOKEN }} publish_dir: ./docs/_build/html From 7178a411906bb37a5dc9ed30dc07800d8280c524 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:54:45 +0000 Subject: [PATCH 09/32] Bump actions/checkout from 6 to 6.0.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 6.0.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v6.0.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codemeta2cff.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/corpus.yml | 2 +- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/markdownlint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/publish-versioned-docs.yml | 6 +++--- .github/workflows/pypi-publish.yml | 4 ++-- .github/workflows/pypi-test.yml | 2 +- .github/workflows/unittest.yml | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index 97ddc1e07..732db59c0 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -25,7 +25,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Generate CITATION.cff from codemeta.json uses: caltechlibrary/codemeta2cff@99ce10831c3037a067c3d9551d455638e0b438ac #v0.3.3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 03d896670..bcc584973 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 1609af5cf..3287012cf 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 749c17e54..89ce75841 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 383158175..761f0cb12 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Ruff uses: astral-sh/ruff-action@v3 with: diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml index 178437f16..0843cacbe 100644 --- a/.github/workflows/markdownlint.yml +++ b/.github/workflows/markdownlint.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 with: fetch-depth: 0 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index ce8f1c65f..1a482bf27 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -46,7 +46,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/publish-versioned-docs.yml b/.github/workflows/publish-versioned-docs.yml index 5d2be64d4..19bb27b88 100644 --- a/.github/workflows/publish-versioned-docs.yml +++ b/.github/workflows/publish-versioned-docs.yml @@ -31,19 +31,19 @@ jobs: run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - name: Checkout pythainlp (for scripts) - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: path: pythainlp - name: Checkout dev-docs repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: repository: PyThaiNLP/dev-docs token: ${{ secrets.PERSONAL_TOKEN }} path: dev-docs - name: Checkout docs repository - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: repository: PyThaiNLP/docs token: ${{ secrets.PERSONAL_TOKEN }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 95ca9c901..fba3c3f5d 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -52,7 +52,7 @@ jobs: build: ${{ steps.check_build_trigger.outputs.build }} steps: - name: Checkout source code - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 with: ref: ${{ github.event.pull_request.head.sha }} - id: check_build_trigger @@ -71,7 +71,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index d18a6d22d..7b2e8b5e1 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -21,7 +21,7 @@ jobs: python-version: ["3.12"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v6.0.2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 16ecc2d34..4d62e1ef6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -82,7 +82,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v6.0.2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 From 32e75812129e77a17167c4f81106c06041a59fe1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:48:39 +0000 Subject: [PATCH 10/32] Add bandit security checks (resolves #1430) - Add bandit>=1.9.4 to the dev dependency group in pyproject.toml - Add [tool.bandit] config section with skips for false positives and accepted risks (B105, B107, B110, B202, B310, B311, B614, B615) - Add .github/workflows/bandit.yml workflow triggered on push/PR to both dev and main branches - Add # nosec B324 to MD5 hash line in corpus/core.py (already had # noqa: S324 for ruff; this suppression is for bandit) All issues accounted for: bandit now runs cleanly with 0 findings. --- .github/workflows/bandit.yml | 66 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 14 ++++++++ pythainlp/corpus/core.py | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/bandit.yml diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 000000000..deca8c6d3 --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +# Bandit is a tool designed to find common security issues in Python code. +# https://github.com/pycqa/bandit + +name: Bandit + +on: + push: + branches: + - dev + - main + paths-ignore: + - '**.cff' + - '**.json' + - '**.md' + - '**.rst' + - '**.txt' + - '**.yml' + - 'docs/**' + pull_request: + branches: + - dev + - main + paths-ignore: + - '**.cff' + - '**.json' + - '**.md' + - '**.rst' + - '**.txt' + - '**.yml' + - 'docs/**' + +# Avoid duplicate runs for the same source branch and repository. +# For pull_request events, uses the source repo name from +# github.event.pull_request.head.repo.full_name; otherwise uses github.repository. +# For push events, uses the branch name from github.ref_name. +# For pull_request events, uses the source branch name from github.head_ref. +# This ensures events for the same repo and branch share the same group, +# and avoids cross-fork collisions when branch names are reused. +concurrency: + group: >- + ${{ github.workflow }}-${{ + github.event.pull_request.head.repo.full_name || github.repository + }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + bandit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + cache: "pip" + + - name: Install bandit + run: pip install "bandit>=1.9.4" + + - name: Run bandit + run: bandit -r pythainlp -c pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index 308fda4ae..dc72d9817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ dependencies = [ ## 1) Development ######################################## dev = [ + "bandit>=1.9.4", "black>=25.11.0", "build>=1.0.0", "bump-my-version>=1.2.6", @@ -497,6 +498,19 @@ module = [ ] ignore_missing_imports = true +[tool.bandit] +# Skip tests that produce known false positives or are accepted risks. +# B105, B107: String/default value detected as hardcoded password. +# False positives: tokenizer special tokens and numeric config values. +# B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. +# B202: tarfile.extractall. Accepted; no untrusted archives used. +# B310: URL open for permitted schemes. Accepted; URLs are validated upstream. +# B311: Standard pseudo-random generators. Used for NLP tasks, not security. +# B614: PyTorch torch.load. Accepted; only trusted internal model files loaded. +# B615: HuggingFace download without revision pinning. +# Accepted; release versions are tracked via dependency version pins. +skips = ["B105", "B107", "B110", "B202", "B310", "B311", "B614", "B615"] + [tool.pylint.main] disable = [ "import-error", diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 9da5930bb..898231860 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -383,7 +383,7 @@ def _check_hash(dst: str, md5: str) -> None: with open(get_full_data_path(dst), "rb") as f: content = f.read() # MD5 is insecure but sufficient here - file_md5 = hashlib.md5(content).hexdigest() # noqa: S324 + file_md5 = hashlib.md5(content).hexdigest() # noqa: S324 # nosec B324 if md5 != file_md5: raise ValueError("Hash does not match expected.") From b7130a721140bf8996978b4f7b504d62c791896c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:50:26 +0000 Subject: [PATCH 11/32] Fix bandit workflow: remove **.yml from paths-ignore, add permissions block --- .github/workflows/bandit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index deca8c6d3..375d6eb41 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -17,7 +17,6 @@ on: - '**.md' - '**.rst' - '**.txt' - - '**.yml' - 'docs/**' pull_request: branches: @@ -29,7 +28,6 @@ on: - '**.md' - '**.rst' - '**.txt' - - '**.yml' - 'docs/**' # Avoid duplicate runs for the same source branch and repository. @@ -49,6 +47,8 @@ concurrency: jobs: bandit: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout uses: actions/checkout@v6.0.2 From 3d5df0c7190a97b0bbde5634b5342a1c112d6f9c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:17:22 +0000 Subject: [PATCH 12/32] Fix B615 HuggingFace unsafe download warnings Add revision parameter to all user-supplied model loaders so that callers can pin to a specific commit hash for reproducible and secure downloads. Use # nosec B615 for module-level singletons with hardcoded well-known model names where parameterization is impractical. Remove B615 from the global bandit skips list. Files with new revision parameter: - pythainlp/corpus/core.py (get_hf_hub) - pythainlp/generate/wangchanglm.py (WangChanGLM.load_model) - pythainlp/lm/qwen3.py (Qwen3.load_model) - pythainlp/parse/transformers_ud.py (Parse.__init__) - pythainlp/parse/ud_goeswith.py (Parse.__init__) - pythainlp/phayathaibert/core.py (PartOfSpeechTagger, NamedEntityTagger) - pythainlp/summarize/mt5.py (mT5Summarizer.__init__) - pythainlp/tag/pos_tag.py (pos_tag_transformers) - pythainlp/translate/small100.py (Small100Translator.__init__) - pythainlp/translate/th_fr.py (ThFrTranslator.__init__) - pythainlp/translate/zh_th.py (ThZhTranslator, ZhThTranslator) - pythainlp/wangchanberta/core.py (NamedEntityRecognition.__init__) Files with # nosec B615 (hardcoded model names): - pythainlp/augment/lm/phayathaibert.py - pythainlp/augment/lm/wangchanberta.py - pythainlp/phayathaibert/core.py (module-level singleton) - pythainlp/spell/wanchanberta_thai_grammarly.py - pythainlp/wangchanberta/core.py (hardcoded revision="main") bandit now reports 0 B615 findings (11 lines suppressed via # nosec). --- pyproject.toml | 2 +- pythainlp/augment/lm/phayathaibert.py | 4 +-- pythainlp/augment/lm/wangchanberta.py | 2 +- pythainlp/corpus/core.py | 16 ++++++++-- pythainlp/generate/wangchanglm.py | 6 +++- pythainlp/lm/qwen3.py | 8 ++++- pythainlp/parse/transformers_ud.py | 18 ++++++++--- pythainlp/parse/ud_goeswith.py | 9 ++++-- pythainlp/phayathaibert/core.py | 32 +++++++++++++------ .../spell/wanchanberta_thai_grammarly.py | 6 ++-- pythainlp/summarize/mt5.py | 13 ++++++-- pythainlp/tag/pos_tag.py | 17 +++++++--- pythainlp/translate/small100.py | 5 ++- pythainlp/translate/th_fr.py | 5 +-- pythainlp/translate/zh_th.py | 10 +++--- pythainlp/wangchanberta/core.py | 14 +++++--- 16 files changed, 121 insertions(+), 46 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dc72d9817..87f90ff6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -509,7 +509,7 @@ ignore_missing_imports = true # B614: PyTorch torch.load. Accepted; only trusted internal model files loaded. # B615: HuggingFace download without revision pinning. # Accepted; release versions are tracked via dependency version pins. -skips = ["B105", "B107", "B110", "B202", "B310", "B311", "B614", "B615"] +skips = ["B105", "B107", "B110", "B202", "B310", "B311", "B614"] [tool.pylint.main] disable = [ diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index 2825ebe46..544fb0114 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -29,10 +29,10 @@ def __init__(self) -> None: ) self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( - _MODEL_NAME + _MODEL_NAME # nosec B615 ) self.model_for_masked_lm: AutoModelForMaskedLM = ( - AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) + AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) # nosec B615 ) self.model: Pipeline = pipeline( "fill-mask", diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index f593b3a95..1ccda2803 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -29,7 +29,7 @@ def __init__(self) -> None: self.target_tokenizer: type[CamembertTokenizer] = CamembertTokenizer self.tokenizer: CamembertTokenizer = ( CamembertTokenizer.from_pretrained( - self.model_name, revision="main" + self.model_name, revision="main" # nosec B615 ) ) self.tokenizer.additional_special_tokens = [ diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 898231860..28430da71 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -853,12 +853,17 @@ def make_safe_directory_name(name: str) -> str: return safe_name -def get_hf_hub(repo_id: str, filename: str = "") -> str: +def get_hf_hub( + repo_id: str, filename: str = "", revision: Optional[str] = None +) -> str: """HuggingFace Hub in :mod:`pythainlp` data directory. :param str repo_id: repo_id :param str filename: filename (optional, default is empty string). If empty, downloads entire snapshot. + :param Optional[str] revision: a git revision id, which can be a branch + name, a tag, or a commit hash (optional, default is ``None``). + Pin to a full commit hash for reproducible and secure downloads. :return: path :rtype: str """ @@ -876,10 +881,15 @@ def get_hf_hub(repo_id: str, filename: str = "") -> str: root_project = safe_path_join(hf_root, name_dir) if filename: output_path = hf_hub_download( - repo_id=repo_id, filename=filename, local_dir=root_project + repo_id=repo_id, + filename=filename, + local_dir=root_project, + revision=revision, ) else: output_path = snapshot_download( - repo_id=repo_id, local_dir=root_project + repo_id=repo_id, + local_dir=root_project, + revision=revision, ) return str(output_path) diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index c0fc69306..7d18f60eb 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -47,6 +47,7 @@ def load_model( torch_dtype: Optional["torch.dtype"] = None, offload_folder: str = "./", low_cpu_mem_usage: bool = True, + revision: Optional[str] = None, ) -> None: """Load model @@ -57,6 +58,8 @@ def load_model( :param Optional[torch.dtype] torch_dtype: torch_dtype :param str offload_folder: offload folder :param bool low_cpu_mem_usage: low cpu mem usage + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ import pandas as pd from transformers import AutoModelForCausalLM, AutoTokenizer @@ -72,9 +75,10 @@ def load_model( torch_dtype=torch_dtype, offload_folder=offload_folder, low_cpu_mem_usage=low_cpu_mem_usage, + revision=revision, ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(self.model_path) + AutoTokenizer.from_pretrained(self.model_path, revision=revision) ) self.df: "pd.DataFrame" = pd.DataFrame( self.tokenizer.vocab.items(), columns=["text", "idx"] diff --git a/pythainlp/lm/qwen3.py b/pythainlp/lm/qwen3.py index 82d8ac32b..fd6a49be4 100644 --- a/pythainlp/lm/qwen3.py +++ b/pythainlp/lm/qwen3.py @@ -31,6 +31,7 @@ def load_model( device: str = "cuda", torch_dtype: Optional["torch.dtype"] = None, low_cpu_mem_usage: bool = True, + revision: Optional[str] = None, ) -> None: """Load Qwen3 model. @@ -38,6 +39,8 @@ def load_model( :param str device: device (cpu, cuda or other) :param Optional[torch.dtype] torch_dtype: torch data type (e.g., torch.float16, torch.bfloat16) :param bool low_cpu_mem_usage: low cpu mem usage + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. :Example: @@ -74,7 +77,9 @@ def load_model( self.model_path = model_path try: - self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_path, revision=revision + ) except OSError as exc: raise RuntimeError( f"Failed to load tokenizer from '{self.model_path}'. " @@ -87,6 +92,7 @@ def load_model( device_map=device, torch_dtype=torch_dtype, low_cpu_mem_usage=low_cpu_mem_usage, + revision=revision, ) except OSError as exc: # Clean up tokenizer on failure diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 4d0390722..79cdf6d44 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -26,7 +26,9 @@ class Parse: def __init__( - self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-head" + self, + model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-head", + revision: Optional[str] = None, ) -> None: from transformers import ( AutoConfig, @@ -39,9 +41,13 @@ def __init__( if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-head" - self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( + model, revision=revision + ) self.model: AutoModelForQuestionAnswering = ( - AutoModelForQuestionAnswering.from_pretrained(model) + AutoModelForQuestionAnswering.from_pretrained( + model, revision=revision + ) ) x = AutoModelForTokenClassification.from_pretrained if os.path.isdir(model): @@ -51,11 +57,13 @@ def __init__( ) else: c = AutoConfig.from_pretrained( - cached_file(model, "deprel/config.json") + cached_file(model, "deprel/config.json"), + revision=revision, ) d = x(cached_file(model, "deprel/pytorch_model.bin"), config=c) s = AutoConfig.from_pretrained( - cached_file(model, "tagger/config.json") + cached_file(model, "tagger/config.json"), + revision=revision, ) t = x(cached_file(model, "tagger/pytorch_model.bin"), config=s) self.deprel: TokenClassificationPipeline = TokenClassificationPipeline( diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index 3e28d7412..2d6705f4c 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -20,12 +20,17 @@ class Parse: def __init__( self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-goeswith", + revision: Optional[str] = None, ) -> None: if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-goeswith" - self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained( + model, revision=revision + ) self.model: AutoModelForTokenClassification = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def __call__( diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index d0a368ec5..c977b45cb 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -6,7 +6,7 @@ import random import re import warnings -from typing import TYPE_CHECKING, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast if TYPE_CHECKING: from collections.abc import Callable @@ -29,7 +29,7 @@ _model_name: str = "clicknext/phayathaibert" _tokenizer: "CamembertTokenizer" = CamembertTokenizer.from_pretrained( - _model_name + _model_name # nosec B615 ) @@ -227,10 +227,10 @@ def __init__(self) -> None: ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(_model_name) + AutoTokenizer.from_pretrained(_model_name) # nosec B615 ) self.model_for_masked_lm: "AutoModelForMaskedLM" = ( - AutoModelForMaskedLM.from_pretrained(_model_name) + AutoModelForMaskedLM.from_pretrained(_model_name) # nosec B615 ) self.model: "Pipeline" = pipeline( # transformers.Pipeline "fill-mask", @@ -317,7 +317,11 @@ def augment( class PartOfSpeechTagger: - def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: + def __init__( + self, + model: str = "lunarlist/pos_thai_phayathai", + revision: Optional[str] = None, + ) -> None: # Load model directly from transformers import ( AutoModelForTokenClassification, @@ -325,10 +329,12 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: "AutoModelForTokenClassification" = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def get_tag( @@ -364,17 +370,23 @@ def get_tag( class NamedEntityTagger: - def __init__(self, model: str = "Pavarissy/phayathaibert-thainer") -> None: + def __init__( + self, + model: str = "Pavarissy/phayathaibert-thainer", + revision: Optional[str] = None, + ) -> None: from transformers import ( AutoModelForTokenClassification, AutoTokenizer, ) self.tokenizer: "PreTrainedTokenizerBase" = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: "AutoModelForTokenClassification" = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def get_ner( diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 5ac7003f1..8062894b8 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -27,7 +27,7 @@ use_cuda: bool = torch.cuda.is_available() device: torch.device = torch.device("cuda" if use_cuda else "cpu") tokenizer: "PreTrainedTokenizer" = AutoTokenizer.from_pretrained( - "airesearch/wangchanberta-base-att-spm-uncased" + "airesearch/wangchanberta-base-att-spm-uncased" # nosec B615 ) @@ -36,7 +36,7 @@ def __init__(self) -> None: super().__init__() self.bert: BertForTokenClassification = ( BertForTokenClassification.from_pretrained( - "bookpanda/wangchanberta-base-att-spm-uncased-tagging" + "bookpanda/wangchanberta-base-att-spm-uncased-tagging" # nosec B615 ) ) @@ -97,7 +97,7 @@ def evaluate_one_text(model: BertModel, sentence: str) -> list[str]: mlm_model: "AutoModelForMaskedLM" = AutoModelForMaskedLM.from_pretrained( - "bookpanda/wangchanberta-base-att-spm-uncased-masking" + "bookpanda/wangchanberta-base-att-spm-uncased-masking" # nosec B615 ) if use_cuda: mlm_model = mlm_model.to(device=device) diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index 0a5a843d5..6d376105c 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -5,6 +5,8 @@ from __future__ import annotations +from typing import Optional + from pythainlp.summarize import CPE_KMUTT_THAI_SENTENCE_SUM @@ -18,6 +20,7 @@ def __init__( max_length: int = 100, skip_special_tokens: bool = True, pretrained_mt5_model_name: str = "", + revision: Optional[str] = None, ) -> None: """Initialize mT5 Summarizer. @@ -34,6 +37,8 @@ def __init__( output. Default is True. :param str pretrained_mt5_model_name: Name of pretrained model. If empty (default), uses google/mt5-{model_size}. + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ from transformers import MT5ForConditionalGeneration, T5Tokenizer @@ -52,9 +57,13 @@ def __init__( model_name = pretrained_mt5_model_name self.model_name: str = model_name self.model: MT5ForConditionalGeneration = ( - MT5ForConditionalGeneration.from_pretrained(model_name) + MT5ForConditionalGeneration.from_pretrained( + model_name, revision=revision + ) + ) + self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained( + model_name, revision=revision ) - self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(model_name) self.num_beams: int = num_beams self.no_repeat_ngram_size: int = no_repeat_ngram_size self.min_length: int = min_length diff --git a/pythainlp/tag/pos_tag.py b/pythainlp/tag/pos_tag.py index 5c56f7867..dd708c84a 100644 --- a/pythainlp/tag/pos_tag.py +++ b/pythainlp/tag/pos_tag.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import Optional + def pos_tag( words: list[str], engine: str = "perceptron", corpus: str = "orchid" @@ -168,6 +170,7 @@ def pos_tag_transformers( sentence: str, engine: str = "bert", corpus: str = "blackboard", + revision: Optional[str] = None, ) -> list[list[tuple[str, str]]]: """Marks sentences with part-of-speech (POS) tags. @@ -189,6 +192,8 @@ def pos_tag_transformers( `_ \ treebanks, natively use Universal POS tags \ (support wangchanberta and mdeberta engine) + :param Optional[str] revision: a git revision id (branch, tag, or commit + hash) for the model. Pin to a full commit hash for secure downloads. :return: a list of lists of tuples (word, POS tag) :rtype: list[list[tuple[str, str]]] @@ -228,12 +233,16 @@ def pos_tag_transformers( if corpus == "blackboard" and engine in _blackboard_support_engine.keys(): base_model = _blackboard_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained(base_model) - tokenizer = AutoTokenizer.from_pretrained(base_model) + model = AutoModelForTokenClassification.from_pretrained( + base_model, revision=revision + ) + tokenizer = AutoTokenizer.from_pretrained(base_model, revision=revision) elif corpus == "pud" and engine in _pud_support_engine.keys(): base_model = _pud_support_engine.get(engine) - model = AutoModelForTokenClassification.from_pretrained(base_model) - tokenizer = AutoTokenizer.from_pretrained(base_model) + model = AutoModelForTokenClassification.from_pretrained( + base_model, revision=revision + ) + tokenizer = AutoTokenizer.from_pretrained(base_model, revision=revision) else: raise ValueError( f"pos_tag_transformers not support {engine} engine or {corpus} corpus." diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index ce87a4c8b..ce641f4f6 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -30,12 +30,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "alirezamsh/small100", + revision: Optional[str] = None, ) -> None: from transformers import M2M100ForConditionalGeneration self.pretrained: str = pretrained self.model: "M2M100ForConditionalGeneration" = ( - M2M100ForConditionalGeneration.from_pretrained(self.pretrained) + M2M100ForConditionalGeneration.from_pretrained( + self.pretrained, revision=revision + ) ) self.tgt_lang: Optional[str] = None if use_gpu: diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 156b35709..a980f1948 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -43,14 +43,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Helsinki-NLP/opus-mt-th-fr", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_thfr: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_thfr: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_thfr = self.model_thfr.cuda() diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 54a035454..534dee00a 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -33,14 +33,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Lalita/marianmt-th-zh_cn", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_thzh: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_thzh: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_thzh = self.model_thzh.cuda() @@ -108,14 +109,15 @@ def __init__( self, use_gpu: bool = False, pretrained: str = "Lalita/marianmt-zh_cn-th", + revision: Optional[str] = None, ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer self.tokenizer_zhth: AutoTokenizer = AutoTokenizer.from_pretrained( - pretrained + pretrained, revision=revision ) self.model_zhth: AutoModelForSeq2SeqLM = ( - AutoModelForSeq2SeqLM.from_pretrained(pretrained) + AutoModelForSeq2SeqLM.from_pretrained(pretrained, revision=revision) ) if use_gpu: self.model_zhth = self.model_zhth.cuda() diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index cb1474381..f8745e52a 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -28,7 +28,7 @@ def _get_tokenizer() -> CamembertTokenizer: from transformers import CamembertTokenizer _tokenizer = CamembertTokenizer.from_pretrained( - f"airesearch/{_model_name}", revision="main" + f"airesearch/{_model_name}", revision="main" # nosec B615 ) if _model_name == "wangchanberta-base-att-spm-uncased": _tokenizer.additional_special_tokens = [ @@ -164,21 +164,27 @@ class NamedEntityRecognition: model: PreTrainedModel def __init__( - self, model: str = "pythainlp/thainer-corpus-v2-base-model" + self, + model: str = "pythainlp/thainer-corpus-v2-base-model", + revision: Optional[str] = None, ) -> None: """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ AI Research Institute of Thailand :param str model: The model that use wangchanberta pretrained. + :param Optional[str] revision: a git revision id (branch, tag, or + commit hash). Pin to a full commit hash for secure downloads. """ from transformers import AutoModelForTokenClassification, AutoTokenizer self.tokenizer: PreTrainedTokenizerBase = ( - AutoTokenizer.from_pretrained(model) + AutoTokenizer.from_pretrained(model, revision=revision) ) self.model: PreTrainedModel = ( - AutoModelForTokenClassification.from_pretrained(model) + AutoModelForTokenClassification.from_pretrained( + model, revision=revision + ) ) def _fix_span_error( From 8b84efdfdf748381e6600eb5187b519ae8970187 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:19:35 +0000 Subject: [PATCH 13/32] Fix B615: pass revision to cached_file() in transformers_ud.py The revision parameter belongs on cached_file() (to fetch the right version from HuggingFace Hub), not on AutoConfig.from_pretrained() which receives the resulting local path. Add # nosec B615 there since the revision is already handled upstream by cached_file(). --- pythainlp/parse/transformers_ud.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 79cdf6d44..18a8433d6 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -56,16 +56,14 @@ def __init__( x(safe_path_join(model, "tagger")), ) else: - c = AutoConfig.from_pretrained( - cached_file(model, "deprel/config.json"), - revision=revision, + c = AutoConfig.from_pretrained( # nosec B615 + cached_file(model, "deprel/config.json", revision=revision), ) - d = x(cached_file(model, "deprel/pytorch_model.bin"), config=c) - s = AutoConfig.from_pretrained( - cached_file(model, "tagger/config.json"), - revision=revision, + d = x(cached_file(model, "deprel/pytorch_model.bin", revision=revision), config=c) + s = AutoConfig.from_pretrained( # nosec B615 + cached_file(model, "tagger/config.json", revision=revision), ) - t = x(cached_file(model, "tagger/pytorch_model.bin"), config=s) + t = x(cached_file(model, "tagger/pytorch_model.bin", revision=revision), config=s) self.deprel: TokenClassificationPipeline = TokenClassificationPipeline( model=d, tokenizer=self.tokenizer, aggregation_strategy="simple" ) From 3fd2c4d8c872b71764ce6afdc257e7ab8cde2ddc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:54:48 +0000 Subject: [PATCH 14/32] Bump github/codeql-action from 4.36.0 to 4.36.2 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.36.0 to 4.36.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v4.36.0...v4.36.2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.36.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2873f7497..0a6a68047 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -66,7 +66,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.36.0 + uses: github/codeql-action/init@v4.36.2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -77,7 +77,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v4.36.0 + uses: github/codeql-action/autobuild@v4.36.2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -91,4 +91,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.36.0 + uses: github/codeql-action/analyze@v4.36.2 From 76087b4512e1669f4ec7707b2d3e97523a8b5c4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:55:01 +0000 Subject: [PATCH 15/32] Bump actions/checkout from 6.0.2 to 6.0.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.2 to 6.0.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6.0.2...v6.0.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codemeta2cff.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/corpus.yml | 2 +- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/markdownlint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/publish-versioned-docs.yml | 6 +++--- .github/workflows/pypi-publish.yml | 4 ++-- .github/workflows/pypi-test.yml | 2 +- .github/workflows/unittest.yml | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/codemeta2cff.yml b/.github/workflows/codemeta2cff.yml index 732db59c0..a843b96a8 100644 --- a/.github/workflows/codemeta2cff.yml +++ b/.github/workflows/codemeta2cff.yml @@ -25,7 +25,7 @@ jobs: contents: write steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Generate CITATION.cff from codemeta.json uses: caltechlibrary/codemeta2cff@99ce10831c3037a067c3d9551d455638e0b438ac #v0.3.3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2873f7497..447ad61b3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 3287012cf..3126a3b05 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index e9f133502..edbb627c6 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 761f0cb12..bc0d47183 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Ruff uses: astral-sh/ruff-action@v3 with: diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml index 0843cacbe..1dfdbef0b 100644 --- a/.github/workflows/markdownlint.yml +++ b/.github/workflows/markdownlint.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v6.0.3 with: fetch-depth: 0 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1a482bf27..d174913cb 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -46,7 +46,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/publish-versioned-docs.yml b/.github/workflows/publish-versioned-docs.yml index 19bb27b88..0aea53fe4 100644 --- a/.github/workflows/publish-versioned-docs.yml +++ b/.github/workflows/publish-versioned-docs.yml @@ -31,19 +31,19 @@ jobs: run: echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - name: Checkout pythainlp (for scripts) - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 with: path: pythainlp - name: Checkout dev-docs repository - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 with: repository: PyThaiNLP/dev-docs token: ${{ secrets.PERSONAL_TOKEN }} path: dev-docs - name: Checkout docs repository - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 with: repository: PyThaiNLP/docs token: ${{ secrets.PERSONAL_TOKEN }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index fba3c3f5d..cba969cb9 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -52,7 +52,7 @@ jobs: build: ${{ steps.check_build_trigger.outputs.build }} steps: - name: Checkout source code - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 with: ref: ${{ github.event.pull_request.head.sha }} - id: check_build_trigger @@ -71,7 +71,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/pypi-test.yml b/.github/workflows/pypi-test.yml index 7b2e8b5e1..98f7a6472 100644 --- a/.github/workflows/pypi-test.yml +++ b/.github/workflows/pypi-test.yml @@ -21,7 +21,7 @@ jobs: python-version: ["3.12"] steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 4d62e1ef6..7987dd370 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -82,7 +82,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v6.0.3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 From 4e17b846d62b96a5a0af91d13e47489301e74e21 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 12 Jun 2026 12:10:44 +0100 Subject: [PATCH 16/32] Update copyright year in bandit.yml --- .github/workflows/bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 375d6eb41..a96f860c2 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026-present PyThaiNLP Project # SPDX-License-Identifier: Apache-2.0 # Bandit is a tool designed to find common security issues in Python code. From 23b6535487d1e325913035b98394547446187efe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:18:09 +0000 Subject: [PATCH 17/32] Move B202/B310/B311/B614 from global skips to location-specific nosec comments --- pyproject.toml | 9 ++------- pythainlp/augment/lm/phayathaibert.py | 2 +- pythainlp/corpus/core.py | 8 ++++---- pythainlp/generate/core.py | 14 +++++++------- pythainlp/generate/thai2fit.py | 2 +- pythainlp/phayathaibert/core.py | 2 +- pythainlp/tools/misspell.py | 5 +++-- pythainlp/transliterate/thai2rom.py | 4 ++-- pythainlp/transliterate/thaig2p.py | 4 ++-- 9 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87f90ff6f..2152c8bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -503,13 +503,8 @@ ignore_missing_imports = true # B105, B107: String/default value detected as hardcoded password. # False positives: tokenizer special tokens and numeric config values. # B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. -# B202: tarfile.extractall. Accepted; no untrusted archives used. -# B310: URL open for permitted schemes. Accepted; URLs are validated upstream. -# B311: Standard pseudo-random generators. Used for NLP tasks, not security. -# B614: PyTorch torch.load. Accepted; only trusted internal model files loaded. -# B615: HuggingFace download without revision pinning. -# Accepted; release versions are tracked via dependency version pins. -skips = ["B105", "B107", "B110", "B202", "B310", "B311", "B614"] +# B202, B310, B311, B614: suppressed at each call site with # nosec comments. +skips = ["B105", "B107", "B110"] [tool.pylint.main] disable = [ diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index 544fb0114..abac2083d 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -55,7 +55,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 28430da71..7e33483c3 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -71,7 +71,7 @@ def get_corpus_db(url: str) -> Optional[_ResponseWrapper]: try: req = Request(url, headers={"User-Agent": _USER_AGENT}) # SSL certificate verification is enabled by default - with urlopen(req, timeout=10) as response: + with urlopen(req, timeout=10) as response: # nosec B310 corpus_db = _ResponseWrapper(response) except HTTPError as http_err: print(f"HTTP error occurred: {http_err}") @@ -349,7 +349,7 @@ def _download(url: str, dst: str) -> int: req = Request(url, headers={"User-Agent": _USER_AGENT}) # SSL certificate verification is enabled by default - with urlopen(req, timeout=10) as response: + with urlopen(req, timeout=10) as response: # nosec B310 file_size = int(response.info().get("Content-Length", -1)) with open(get_full_data_path(dst), "wb") as f: pbar = None @@ -484,7 +484,7 @@ def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: f"Symlink {member.name} points outside extraction directory: {member.linkname}" ) - tar.extractall(path=path) + tar.extractall(path=path) # nosec B202 def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: @@ -539,7 +539,7 @@ def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: f"Symlink {member} points outside extraction directory: {link_target}" ) - zip_file.extractall(path=path) + zip_file.extractall(path=path) # nosec B202 def _version2int(v: str) -> int: diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 391fc7d47..f4621d71c 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -82,7 +82,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.word) # noqa: S311 + start_seq = random.choice(self.word) # noqa: S311 # nosec B311 rand_text = start_seq.lower() self._word_prob = { i: self.counts[i] / self.n @@ -108,10 +108,10 @@ def _next_word( N = len(word_list) for _ in range(N): # Non-cryptographic use, pseudo-random generator is acceptable here - w = random.choice(word_list) # noqa: S311 + w = random.choice(word_list) # noqa: S311 # nosec B311 if duplicate is False: while w in words: - w = random.choice(word_list) # noqa: S311 + w = random.choice(word_list) # noqa: S311 # nosec B311 words.append(w) if output_str: @@ -185,7 +185,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.words) # noqa: S311 + start_seq = random.choice(self.words) # noqa: S311 # nosec B311 late_word = start_seq list_word = [] list_word.append(start_seq) @@ -204,7 +204,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 late_word = items[-1] list_word.append(late_word) @@ -288,7 +288,7 @@ def gen_sentence( late_word: Union[str, tuple[str, str]] if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.bi_keys) # noqa: S311 + start_seq = random.choice(self.bi_keys) # noqa: S311 # nosec B311 late_word = start_seq list_word: list[Union[str, tuple[str, str]]] = [] list_word.append(start_seq) @@ -307,7 +307,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 late_word = items[1:] list_word.append(late_word) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 08492c597..df9613937 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -150,7 +150,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(list(thwiki_itos)) # noqa: S311 + start_seq = random.choice(list(thwiki_itos)) # noqa: S311 # nosec B311 predicted_text: str = learn.predict( start_seq, N, temperature=0.8, min_p=prob, sep="-*-" ) diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index c977b45cb..6217bc8bf 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -253,7 +253,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index d974dcd06..3b4f0f11a 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -135,7 +135,8 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: 'ภาษาไทยปรากฏครั้งแรกในกุทธศักราช 1727' """ num_misspells = math.floor(len(sentence) * ratio) - positions = random.sample(range(len(sentence)), k=num_misspells) + # Non-cryptographic use, pseudo-random generator is acceptable here + positions = random.sample(range(len(sentence)), k=num_misspells) # noqa: S311 # nosec B311 # convert strings to array of characters misspelled = list(sentence) @@ -145,7 +146,7 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: continue # Non-cryptographic use, pseudo-random generator is acceptable here - candidate = random.choice(potential_candidates) # noqa: S311 + candidate = random.choice(potential_candidates) # noqa: S311 # nosec B311 misspelled[pos] = candidate diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 52ab6a28e..54a8c20e1 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -49,7 +49,7 @@ def __init__(self) -> None: f" CLI: thainlp data get {_MODEL_NAME}" ) - loader = torch.load(self.__model_filename, map_location=device) + loader = torch.load(self.__model_filename, map_location=device) # nosec B614 INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] @@ -419,7 +419,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 if teacher_force and target_seq is not None: decoder_input = target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index f1438198d..27c893d6d 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -58,7 +58,7 @@ def __init__(self) -> None: f" CLI: thainlp data get {_MODEL_NAME}" ) - loader = torch.load(self.__model_filename, map_location=device) + loader = torch.load(self.__model_filename, map_location=device) # nosec B614 INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] @@ -438,7 +438,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 decoder_input = ( target_seq[:, di].reshape(batch_size, 1) From c9fcfafd16c331f43b8cb5a904c814fe6613d08b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:44:54 +0000 Subject: [PATCH 18/32] Move B105/B107 from global skips to location-specific nosec comments --- pyproject.toml | 6 ++---- pythainlp/generate/thai2fit.py | 2 +- pythainlp/phayathaibert/core.py | 2 +- pythainlp/tokenize/_utils.py | 2 +- pythainlp/translate/tokenization_small100.py | 10 +++++----- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2152c8bf0..b4745e36c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -500,11 +500,9 @@ ignore_missing_imports = true [tool.bandit] # Skip tests that produce known false positives or are accepted risks. -# B105, B107: String/default value detected as hardcoded password. -# False positives: tokenizer special tokens and numeric config values. # B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. -# B202, B310, B311, B614: suppressed at each call site with # nosec comments. -skips = ["B105", "B107", "B110"] +# B105, B107, B202, B310, B311, B614: suppressed at each call site with # nosec comments. +skips = ["B110"] [tool.pylint.main] disable = [ diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index df9613937..f79331b16 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -97,7 +97,7 @@ "emb_sz": 400, "n_hid": 1550, "n_layers": 4, - "pad_token": 1, + "pad_token": 1, # nosec B105 "qrnn": False, "tie_weights": True, "out_bias": True, diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 6217bc8bf..1edd4839c 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -120,7 +120,7 @@ def rm_useless_spaces(self, text: str) -> str: """ return re.sub(" {2,}", " ", text) - def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S107 + def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S107 # nosec B107 """Replace spaces with _ :param str text: text to replace spaces :return: text where all spaces replaced with _ diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index a1a822ea0..3a96470c1 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -56,7 +56,7 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: is_span_beginning = pos >= match.start() token = segments[segment_idx] if is_span_beginning: - connected_token = "" + connected_token = "" # nosec B105 while pos < match.end() and segment_idx < len(segments): connected_token += segments[segment_idx] pos += len(segments[segment_idx]) diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index e40cbd1a4..cabd77b0a 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -149,11 +149,11 @@ def __init__( vocab_file: str, spm_file: str, tgt_lang: Optional[str] = None, - bos_token: str = "", # noqa: S107 - eos_token: str = "", # noqa: S107 - sep_token: str = "", # noqa: S107 - pad_token: str = "", # noqa: S107 - unk_token: str = "", # noqa: S107 + bos_token: str = "", # noqa: S107 # nosec B107 + eos_token: str = "", # noqa: S107 # nosec B107 + sep_token: str = "", # noqa: S107 # nosec B107 + pad_token: str = "", # noqa: S107 # nosec B107 + unk_token: str = "", # noqa: S107 # nosec B107 language_codes: str = "m2m100", sp_model_kwargs: Optional[dict[str, str]] = None, num_madeup_words: int = 8, From 96cdae1d150dc92b7884e56361d159b9a49aee4d Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 12 Jun 2026 12:51:28 +0100 Subject: [PATCH 19/32] Update pyproject.toml --- pyproject.toml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4745e36c..fd8463d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2016-present PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 @@ -312,6 +312,11 @@ include = [ "README.md", ] +[tool.bandit] +# Skip tests that produce known false positives or are accepted risks. +# B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. +skips = ["B110"] + [tool.bumpversion] current_version = "5.3.4" commit = true @@ -498,12 +503,6 @@ module = [ ] ignore_missing_imports = true -[tool.bandit] -# Skip tests that produce known false positives or are accepted risks. -# B110: Try/Except/Pass. Accepted pattern in optional-dependency loading. -# B105, B107, B202, B310, B311, B614: suppressed at each call site with # nosec comments. -skips = ["B110"] - [tool.pylint.main] disable = [ "import-error", From 055ca78bef4ea5630697369b56a1ea8e53f52faf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Jun 2026 05:35:11 +0000 Subject: [PATCH 20/32] Add # NOSONAR to all pseudo-random call sites to silence SonarQube S2245 warnings --- pythainlp/augment/lm/phayathaibert.py | 2 +- pythainlp/generate/core.py | 14 +++++++------- pythainlp/generate/thai2fit.py | 2 +- pythainlp/phayathaibert/core.py | 2 +- pythainlp/tools/misspell.py | 4 ++-- pythainlp/transliterate/thai2rom.py | 2 +- pythainlp/transliterate/thaig2p.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index abac2083d..3a3134e93 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -55,7 +55,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 # NOSONAR output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index f4621d71c..214c8b1d2 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -82,7 +82,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.word) # noqa: S311 # nosec B311 + start_seq = random.choice(self.word) # noqa: S311 # nosec B311 # NOSONAR rand_text = start_seq.lower() self._word_prob = { i: self.counts[i] / self.n @@ -108,10 +108,10 @@ def _next_word( N = len(word_list) for _ in range(N): # Non-cryptographic use, pseudo-random generator is acceptable here - w = random.choice(word_list) # noqa: S311 # nosec B311 + w = random.choice(word_list) # noqa: S311 # nosec B311 # NOSONAR if duplicate is False: while w in words: - w = random.choice(word_list) # noqa: S311 # nosec B311 + w = random.choice(word_list) # noqa: S311 # nosec B311 # NOSONAR words.append(w) if output_str: @@ -185,7 +185,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.words) # noqa: S311 # nosec B311 + start_seq = random.choice(self.words) # noqa: S311 # nosec B311 # NOSONAR late_word = start_seq list_word = [] list_word.append(start_seq) @@ -204,7 +204,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 # NOSONAR late_word = items[-1] list_word.append(late_word) @@ -288,7 +288,7 @@ def gen_sentence( late_word: Union[str, tuple[str, str]] if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(self.bi_keys) # noqa: S311 # nosec B311 + start_seq = random.choice(self.bi_keys) # noqa: S311 # nosec B311 # NOSONAR late_word = start_seq list_word: list[Union[str, tuple[str, str]]] = [] list_word.append(start_seq) @@ -307,7 +307,7 @@ def gen_sentence( if len(p2) == 0: break # Non-cryptographic use, pseudo-random generator is acceptable here - items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 + items = temp[probs.index(random.choice(p2))] # noqa: S311 # nosec B311 # NOSONAR late_word = items[1:] list_word.append(late_word) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index f79331b16..b61dda4c6 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -150,7 +150,7 @@ def gen_sentence( """ if not start_seq: # Non-cryptographic use, pseudo-random generator is acceptable here - start_seq = random.choice(list(thwiki_itos)) # noqa: S311 # nosec B311 + start_seq = random.choice(list(thwiki_itos)) # noqa: S311 # nosec B311 # NOSONAR predicted_text: str = learn.predict( start_seq, N, temperature=0.8, min_p=prob, sep="-*-" ) diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 1edd4839c..603b5af8c 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -253,7 +253,7 @@ def generate( input_text = self.processor.preprocess(sample_txt) if sample: # Non-cryptographic use, pseudo-random generator is acceptable here - random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 + random_word_idx = random.randint(0, 4) # noqa: S311 # nosec B311 # NOSONAR output = self.model(input_text)[random_word_idx]["sequence"] else: output = self.model(input_text)[word_rank]["sequence"] diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index 3b4f0f11a..c8580e60e 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -136,7 +136,7 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: """ num_misspells = math.floor(len(sentence) * ratio) # Non-cryptographic use, pseudo-random generator is acceptable here - positions = random.sample(range(len(sentence)), k=num_misspells) # noqa: S311 # nosec B311 + positions = random.sample(range(len(sentence)), k=num_misspells) # noqa: S311 # nosec B311 # NOSONAR # convert strings to array of characters misspelled = list(sentence) @@ -146,7 +146,7 @@ def misspell(sentence: str, ratio: float = 0.05) -> str: continue # Non-cryptographic use, pseudo-random generator is acceptable here - candidate = random.choice(potential_candidates) # noqa: S311 # nosec B311 + candidate = random.choice(potential_candidates) # noqa: S311 # nosec B311 # NOSONAR misspelled[pos] = candidate diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 54a8c20e1..369fec1a0 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -419,7 +419,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 # NOSONAR if teacher_force and target_seq is not None: decoder_input = target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index 27c893d6d..a2df19022 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -438,7 +438,7 @@ def forward( outputs[di] = decoder_output.to(device) # Non-cryptographic use, pseudo-random generator is acceptable here - teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 # nosec B311 # NOSONAR decoder_input = ( target_seq[:, di].reshape(batch_size, 1) From 726be5bd6016d4def4394831e6410fd6502f4f9c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:42:45 +0000 Subject: [PATCH 21/32] Initial plan From a150bf449cdc395fae1042d4dde6d037abc68f94 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:48:46 +0000 Subject: [PATCH 22/32] Add th-pron transliteration module and core tests --- pythainlp/transliterate/th_pron.py | 435 +++++++++++++++++++++++ tests/core/__init__.py | 1 + tests/core/test_transliterate_th_pron.py | 31 ++ 3 files changed, 467 insertions(+) create mode 100644 pythainlp/transliterate/th_pron.py create mode 100644 tests/core/test_transliterate_th_pron.py diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/th_pron.py new file mode 100644 index 000000000..b9c1f576a --- /dev/null +++ b/pythainlp/transliterate/th_pron.py @@ -0,0 +1,435 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +"""Thai pronunciation transliteration from Wiktionary th-pron module.""" + +from __future__ import annotations + +import re +import unicodedata +from typing import cast + +_THAI_RANGE: str = r"[ก-๛̄]" + +_SYSTEMS: dict[str, int] = { + "paiboon": 0, + "royin": 1, + "ipa": 2, +} + +_INITIAL: dict[str, dict[str, list[str] | str]] = { + "ก": {"seq": ["g", "k", "k"], "class": "mid"}, + "จ": {"seq": ["j", "ch", "t͡ɕ"], "class": "mid"}, + "ด": {"seq": ["d", "d", "d"], "class": "mid"}, + "ฎ": {"seq": ["d", "d", "d"], "class": "mid"}, + "ฏ": {"seq": ["dt", "t", "t"], "class": "mid"}, + "ต": {"seq": ["dt", "t", "t"], "class": "mid"}, + "บ": {"seq": ["b", "b", "b"], "class": "mid"}, + "ป": {"seq": ["bp", "p", "p"], "class": "mid"}, + "อ": {"seq": ["", "@", "ʔ"], "class": "mid"}, + "ง": {"seq": ["ng", "$ng", "ŋ"], "class": "low"}, + "ณ": {"seq": ["n", "n", "n"], "class": "low"}, + "น": {"seq": ["n", "n", "n"], "class": "low"}, + "ม": {"seq": ["m", "m", "m"], "class": "low"}, + "ญ": {"seq": ["y", "y", "j"], "class": "low"}, + "ย": {"seq": ["y", "y", "j"], "class": "low"}, + "ร": {"seq": ["r", "r", "r"], "class": "low"}, + "ล": {"seq": ["l", "l", "l"], "class": "low"}, + "ฬ": {"seq": ["l", "l", "l"], "class": "low"}, + "ว": {"seq": ["w", "w", "w"], "class": "low"}, + "ค": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ฅ": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ฆ": {"seq": ["k", "kh", "kʰ"], "class": "low"}, + "ข": {"seq": ["k", "kh", "kʰ"], "class": "high"}, + "ฃ": {"seq": ["k", "kh", "kʰ"], "class": "high"}, + "ช": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "low"}, + "ฌ": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "low"}, + "ฉ": {"seq": ["ch", "ch", "t͡ɕʰ"], "class": "high"}, + "ฑ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ฒ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ท": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ธ": {"seq": ["t", "th", "tʰ"], "class": "low"}, + "ฐ": {"seq": ["t", "th", "tʰ"], "class": "high"}, + "ถ": {"seq": ["t", "th", "tʰ"], "class": "high"}, + "พ": {"seq": ["p", "ph", "pʰ"], "class": "low"}, + "ภ": {"seq": ["p", "ph", "pʰ"], "class": "low"}, + "ผ": {"seq": ["p", "ph", "pʰ"], "class": "high"}, + "ฟ": {"seq": ["f", "f", "f"], "class": "low"}, + "ฝ": {"seq": ["f", "f", "f"], "class": "high"}, + "ซ": {"seq": ["s", "s", "s"], "class": "low"}, + "ศ": {"seq": ["s", "s", "s"], "class": "high"}, + "ษ": {"seq": ["s", "s", "s"], "class": "high"}, + "ส": {"seq": ["s", "s", "s"], "class": "high"}, + "ฮ": {"seq": ["h", "h", "h"], "class": "low"}, + "ห": {"seq": ["h", "h", "h"], "class": "high"}, + "หง": {"seq": ["ng", "$ng", "ŋ"], "class": "high"}, + "หน": {"seq": ["n", "n", "n"], "class": "high"}, + "หม": {"seq": ["m", "m", "m"], "class": "high"}, + "หญ": {"seq": ["y", "y", "j"], "class": "high"}, + "หย": {"seq": ["y", "y", "j"], "class": "high"}, + "หร": {"seq": ["r", "r", "r"], "class": "high"}, + "หล": {"seq": ["l", "l", "l"], "class": "high"}, + "หว": {"seq": ["w", "w", "w"], "class": "high"}, + "…": {"seq": ["…", "…", "…"], "class": ""}, + "": {"seq": ["", "", ""], "class": ""}, +} + +_VOWEL: dict[str, dict[str, list[str]]] = { + "open": { + "ะ": ["a", "a", "a"], + "": ["a", "a", "a"], + "ิ": ["i", "i", "i"], + "ึ": ["ʉ", "ue", "ɯ"], + "ุ": ["u", "u", "u"], + "เะ": ["e", "e", "eʔ"], + "แะ": ["ɛ", "ae", "ɛʔ"], + "โะ": ["o", "o", "oʔ"], + "เาะ": ["ɔ", "o", "ɔʔ"], + "็": ["ɔ", "o", "ɔ"], + "เิ": ["ə", "oe", "ɤ"], + "เอะ": ["ə", "oe", "ɤʔ"], + "า": ["aa", "a", "aː"], + "ี": ["ii", "i", "iː"], + "ู": ["uu", "u", "uː"], + "ือ": ["ʉʉ", "ue", "ɯː"], + "เ": ["ee", "e", "eː"], + "แ": ["ɛɛ", "ae", "ɛː"], + "โ": ["oo", "o", "oː"], + "อ": ["ɔɔ", "o", "ɔː"], + "ร": ["ɔɔn", "on", "ɔːn"], + "เอ": ["əə", "oe", "ɤː"], + "เียะ": ["ia", "ia", "ia̯ʔ"], + "เือะ": ["ʉa", "uea", "ɯa̯ʔ"], + "ัวะ": ["ua", "ua", "ua̯ʔ"], + "เีย": ["iia", "ia", "ia̯"], + "เือ": ["ʉʉa", "uea", "ɯa̯"], + "ัว": ["uua", "ua", "ua̯"], + "ิว": ["iu", "io", "iw"], + "ีว": ["iiu", "io", "iːw"], + "เ็ว": ["eo", "eo", "ew"], + "แ็ว": ["ɛo", "aeo", "ɛw"], + "เา": ["ao", "ao", "aw"], + "เว": ["eeo", "eo", "eːw"], + "แว": ["ɛɛo", "aeo", "ɛːw"], + "าว": ["aao", "ao", "aːw"], + "เอว": ["əəo", "oeu", "ɤːw"], + "โว": ["oow", "ou", "oːw"], + "เียว": ["iao", "iao", "ia̯w"], + "ัย": ["ai", "ai", "aj"], + "ใ": ["ai", "ai", "aj"], + "ไ": ["ai", "ai", "aj"], + "ไย": ["ai", "ai", "aj"], + "ึย": ["ʉi", "uei", "ɯj"], + "็อย": ["ɔi", "oi", "ɔj"], + "เิ็ย": ["əi", "oei", "ɤj"], + "ุย": ["ui", "ui", "uj"], + "าย": ["aai", "ai", "aːj"], + "อย": ["ɔɔi", "oi", "ɔːj"], + "โย": ["ooi", "oi", "oːj"], + "เย": ["əəi", "oei", "ɤːj"], + "ูย": ["uui", "ui", "uːj"], + "วย": ["uai", "uai", "ua̯j"], + "เือย": ["ʉai", "ueai", "ɯa̯j"], + "ำ": ["am", "am", "am"], + }, + "closed": { + "ั": ["a", "a", "a"], + "รร": ["a", "a", "a"], + "ิ": ["i", "i", "i"], + "ึ": ["ʉ", "ue", "ɯ"], + "ุ": ["u", "u", "u"], + "เ": ["ee", "e", "eː"], + "เ็": ["e", "e", "e"], + "แ็": ["ɛ", "ae", "ɛ"], + "แ": ["ɛɛ", "ae", "ɛː"], + "": ["o", "o", "o"], + "็อ": ["ɔ", "o", "ɔ"], + "เิ็": ["ə", "oe", "ɤ"], + "า": ["aa", "a", "aː"], + "ี": ["ii", "i", "iː"], + "ื": ["ʉʉ", "ue", "ɯː"], + "ู": ["uu", "u", "uː"], + "โ": ["oo", "o", "oː"], + "อ": ["ɔɔ", "o", "ɔː"], + "เิ": ["əə", "oe", "ɤː"], + "เอ": ["əə", "oe", "ɤː"], + "เีย": ["iia", "ia", "ia̯"], + "เือ": ["ʉʉa", "uea", "ɯa̯"], + "ว": ["uua", "ua", "ua̯"], + "ไ": ["ai", "ai", "aj"], + "เา": ["ao", "ao", "aw"], + "็อย": ["ɔi", "oi", "ɔj"], + }, +} + +_UNROM_LONG: dict[str, bool] = { + "เีย": True, + "เือ": True, + "ัว": True, + "ว": True, + "เือย": True, + "วาย": True, + "เอว": True, + "เียว": True, +} + +_LIVE_EXC: dict[str, bool] = { + "ัย": True, + "ใ": True, + "ไ": True, + "ไย": True, + "ุย": True, + "วย": True, + "็อย": True, + "เิ็ย": True, + "เา": True, + "ิว": True, + "เ็ว": True, + "แ็ว": True, + "ำ": True, +} + +_CODA: dict[str, list[str]] = { + "ก": ["k", "k", "k̚"], + "ข": ["k", "k", "k̚"], + "ฃ": ["k", "k", "k̚"], + "ค": ["k", "k", "k̚"], + "ฅ": ["k", "k", "k̚"], + "ฆ": ["k", "k", "k̚"], + "จ": ["t", "t", "t̚"], + "ฉ": ["t", "t", "t̚"], + "ช": ["ch", "ch", "t͡ɕʰ"], + "ซ": ["s", "s", "s"], + "ฌ": ["t", "t", "t̚"], + "ฎ": ["t", "t", "t̚"], + "ฏ": ["t", "t", "t̚"], + "ฐ": ["t", "t", "t̚"], + "ฑ": ["t", "t", "t̚"], + "ฒ": ["t", "t", "t̚"], + "ด": ["t", "t", "t̚"], + "ต": ["t", "t", "t̚"], + "ถ": ["t", "t", "t̚"], + "ท": ["t", "t", "t̚"], + "ธ": ["t", "t", "t̚"], + "ศ": ["t", "t", "t̚"], + "ษ": ["t", "t", "t̚"], + "ส": ["s", "s", "s"], + "บ": ["p", "p", "p̚"], + "ป": ["p", "p", "p̚"], + "ผ": ["p", "p", "p̚"], + "ฝ": ["p", "p", "p̚"], + "พ": ["p", "p", "p̚"], + "ฟ": ["f", "f", "f"], + "ภ": ["p", "p", "p̚"], + "ง": ["ng", "ng$", "ŋ"], + "ญ": ["n", "n", "n"], + "ณ": ["n", "n", "n"], + "น": ["n", "n", "n"], + "ร": ["n", "n", "n"], + "ล": ["l", "l", "l"], + "ฬ": ["n", "n", "n"], + "ม": ["m", "m", "m"], + "ฯ": ["ʔ", "ʔ", "ʔ"], +} + +_TONE_FROM_MARK: dict[str, dict[str, str]] = { + "่": {"high": "low", "mid": "low", "low": "falling"}, + "้": {"high": "falling", "mid": "falling", "low": "high"}, + "๊": {"high": "high", "mid": "high", "low": "high"}, + "๋": {"high": "rising", "mid": "rising", "low": "rising"}, + "̄": {"high": "mid", "mid": "mid", "low": "mid"}, +} + +_TONE_NO_MARK: dict[str, dict[str, str]] = { + "dead-short": {"high": "low", "mid": "low", "low": "high"}, + "dead-long": {"high": "low", "mid": "low", "low": "falling"}, + "live": {"high": "rising", "mid": "mid", "low": "mid"}, +} + +_TONE_ROM_MARKS: dict[str, str] = { + "high": "́", + "mid": "", + "low": "̀", + "rising": "̌", + "falling": "̂", +} + +_TONE_LEVELS: dict[str, str] = { + "high": "˦˥", + "mid": "˧", + "low": "˨˩", + "rising": "˩˩˦", + "falling": "˥˩", +} + +_SYMBOLS: dict[str, str] = { + "๐": "0", + "๑": "1", + "๒": "2", + "๓": "3", + "๔": "4", + "๕": "5", + "๖": "6", + "๗": "7", + "๘": "8", + "๙": "9", +} + +_MGVC_PATTERN = re.compile( + r"^([รลว]?)([ิึุ็ีืัำู]?[าอรยว]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) +_FULL_PATTERN = re.compile( + r"^([เแโใไ]?)(หฺ[ก-รลว-ฮ])(ฺ?[รลว]?)([ิึุ็ีืัู]?็?[่้๊๋̄]?[าอรยวำ]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?[คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) +_PARTIAL_PATTERN = re.compile( + r"^([เแโใไ]?)([ก-รลว-ฮ])(ฺ?[รลว]?)([ิึุ็ีืัู]?็?[่้๊๋̄]?[าอรยวำ]?[วยร]?ะ?)([คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?[คฅฆกขฃพฟภบปชฌฑฒทธจฎฏดตฐถศษสมญณนรลฬง]?)$" +) + + +def _c2_decomp(c2_char: str, seq_idx: int) -> str: + converted_c2: list[str] = [] + for character in list(c2_char): + val = _CODA.get(character) + converted_c2.append(val[seq_idx] if val else "") + return "".join(converted_c2) + + +def transliterate(text: str, mode: str = "ipa") -> str: + """Transliterate Thai text using Wiktionary th-pron logic. + + :param str text: Thai text to transliterate. + :param str mode: Output mode: ``paiboon``, ``royin``, or ``ipa``. + :return: Transliterated text. + :rtype: str + """ + seq_idx = _SYSTEMS.get(mode) + if seq_idx is None: + return text + + def process_word(match_word: re.Match[str]) -> str: + word = match_word.group(0) + + if re.search(r"[่้๊๋̄].?[่้๊๋̄]", word): + return word + + def syllable(match: re.Match[str]) -> str: + v1, c1, g, v2, c2 = match.groups() + + tmark_match = re.search(r"[่้๊๋̄]", v2) + tmark = tmark_match.group(0) if tmark_match else None + v2 = re.sub(r"[่้๊๋̄]", "", v2) + + if re.match(r"^ห.$", c1): + mgvc_match = _MGVC_PATTERN.match(c1[1] + g + v2 + c2) + if mgvc_match: + g_new, v2_new, c2_new = mgvc_match.groups() + c1, g, v2, c2 = "ห", g_new, v2_new, c2_new + if g != "" and v2 != "ย": + c1, g = c1 + g, "" + + if g == "ล" and (v2 + c2) == "": + c2 = g + g = "" + + openness = "closed" if c2 != "" else "open" + + if (v1 + g + v2) in _VOWEL[openness]: + orig_v = v1 + g + v2 + v = _VOWEL[openness][orig_v][seq_idx] + g = "" + else: + orig_v = v1 + v2 + v_lookup = _VOWEL[openness].get(v1 + v2) + v = v_lookup[seq_idx] if v_lookup else (v1 + v2) + g_clean = g.replace("ฺ", "") + g_lookup = _INITIAL.get(g_clean) or _INITIAL[""] + g = cast(list[str], g_lookup["seq"])[seq_idx] + + c1_clean = c1.replace("ฺ", "") + if c1_clean in _INITIAL: + ini = cast(list[str], _INITIAL[c1_clean]["seq"])[seq_idx] + cls = cast(str, _INITIAL[c1_clean]["class"]) + else: + return match.group(0) + + length = ( + "long" + if re.search(r"([aiʉueɛoɔə])\1", v) + or "ː" in v + or _UNROM_LONG.get(orig_v) + else "short" + ) + life = ( + "live" + if re.search(r"[มญณนรลฬง]", c2) + or (orig_v.endswith("ย") and v.endswith("i")) + or (c2 + length == "long") + or _LIVE_EXC.get(orig_v) + else "dead" + ) + + if c2 in _CODA: + c2 = _CODA[c2][seq_idx] + else: + c2 = _c2_decomp(c2, seq_idx) + + tone_dict = ( + _TONE_FROM_MARK.get(tmark) + if tmark + else _TONE_NO_MARK.get(f"{life}-{length}", _TONE_NO_MARK.get(life)) + ) + tone = tone_dict.get(cls) if tone_dict else None + + if mode == "paiboon": + v = re.sub( + r"^([^aiʉueɛoɔə]*)([aiʉueɛoɔə])", + f"\\g<1>\\g<2>{_TONE_ROM_MARKS.get(tone, '')}", + v, + ) + elif mode == "ipa": + c2 = c2 + _TONE_LEVELS.get(tone, "") + + return ini + g + v + c2 + + word = _FULL_PATTERN.sub(syllable, word) + word = _PARTIAL_PATTERN.sub(syllable, word) + return word + + words_iter = list(re.finditer(f"{_THAI_RANGE}+", text)) + for match_word in words_iter: + processed = process_word(match_word) + text = text.replace(match_word.group(0), processed, 1) + + text = re.sub(r"[๐-๙]", lambda m: _SYMBOLS.get(m.group(0), m.group(0)), text) + + if mode == "royin": + text = re.sub(r"^@", "", text) + text = re.sub(r"([\s\W])@", r"\1", text) + text = text.replace("@", "-") + text = re.sub(r"^\$ng", "ng", text) + text = re.sub(r"([\s\W])\$ng", r"\1ng", text) + text = re.sub(r"([aeiou])\$ng", r"\1-ng", text) + text = text.replace("$ng", "ng") + text = re.sub(r"ng\$([^\w\s])", r"ng\1", text) + text = re.sub(r"ng\$", "ng", text) + + if mode == "ipa": + text, _count_syl = re.subn(r"[ \-–]", ".", text) + text = re.sub(r"([aiɯu])([˥-˩]+)$", r"\1ʔ\2", text) + + return unicodedata.normalize("NFC", text) + + +def get_word_dict(word: str) -> dict[str, str]: + """Return th-pron transliteration outputs in all supported systems. + + :param str word: Thai input word. + :return: Dict containing ``word``, ``paiboon``, ``royin``, and ``ipa``. + :rtype: dict[str, str] + """ + return { + "word": word, + "paiboon": transliterate(word, mode="paiboon"), + "royin": transliterate(word, mode="royin"), + "ipa": transliterate(word, mode="ipa"), + } diff --git a/tests/core/__init__.py b/tests/core/__init__.py index b4d67630f..5ab824da0 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -24,6 +24,7 @@ "tests.core.test_tokenize", "tests.core.test_tools", "tests.core.test_transliterate", + "tests.core.test_transliterate_th_pron", "tests.core.test_util", ] diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_th_pron.py new file mode 100644 index 000000000..234035a1b --- /dev/null +++ b/tests/core/test_transliterate_th_pron.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from pythainlp.transliterate.th_pron import get_word_dict, transliterate + + +class TransliterateThPronTestCase(unittest.TestCase): + def test_transliterate_supported_modes(self): + self.assertEqual(transliterate("แมว", mode="royin"), "maeo") + self.assertEqual(transliterate("คน", mode="paiboon"), "kon") + self.assertEqual(transliterate("คน", mode="ipa"), "kʰon˧") + + def test_transliterate_preserves_unknown_mode(self): + self.assertEqual(transliterate("แมว", mode="unknown"), "แมว") + + def test_thai_digits_conversion(self): + self.assertEqual(transliterate("๑๒๓", mode="royin"), "123") + + def test_get_word_dict(self): + self.assertEqual( + get_word_dict("ปฺระ-ชา"), + { + "word": "ปฺระ-ชา", + "paiboon": "bprà-chaa", + "royin": "pra-cha", + "ipa": "pra˨˩.t͡ɕʰaː˧", + }, + ) From 6947396e6dcaa6ec7c6c78abe20d367cf669f5ac Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:50:18 +0000 Subject: [PATCH 23/32] Export th-pron helpers from transliterate package --- pythainlp/transliterate/__init__.py | 8 ++++++++ tests/core/test_transliterate_th_pron.py | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 60a6bcf21..44ec30cab 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -4,11 +4,19 @@ """Transliteration.""" __all__: list[str] = [ + "get_word_dict", "pronunciate", "puan", "romanize", + "th_pron_transliterate", "transliterate", ] from pythainlp.transliterate.core import pronunciate, romanize, transliterate from pythainlp.transliterate.spoonerism import puan +from pythainlp.transliterate.th_pron import ( + get_word_dict, +) +from pythainlp.transliterate.th_pron import ( + transliterate as th_pron_transliterate, +) diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_th_pron.py index 234035a1b..f1f5c778b 100644 --- a/tests/core/test_transliterate_th_pron.py +++ b/tests/core/test_transliterate_th_pron.py @@ -4,7 +4,8 @@ import unittest -from pythainlp.transliterate.th_pron import get_word_dict, transliterate +from pythainlp.transliterate import get_word_dict +from pythainlp.transliterate.th_pron import transliterate class TransliterateThPronTestCase(unittest.TestCase): From 2a0512df5d799bf982854b0fdc3a12e1d23319db Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:52:24 +0000 Subject: [PATCH 24/32] Address validation feedback in th-pron transliterator --- pythainlp/transliterate/th_pron.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/th_pron.py index b9c1f576a..c30daeac0 100644 --- a/pythainlp/transliterate/th_pron.py +++ b/pythainlp/transliterate/th_pron.py @@ -299,6 +299,8 @@ def transliterate(text: str, mode: str = "ipa") -> str: :param str text: Thai text to transliterate. :param str mode: Output mode: ``paiboon``, ``royin``, or ``ipa``. + Unsupported modes return the input text unchanged. + :return: Transliterated text. :rtype: str """ @@ -356,7 +358,7 @@ def syllable(match: re.Match[str]) -> str: "long" if re.search(r"([aiʉueɛoɔə])\1", v) or "ː" in v - or _UNROM_LONG.get(orig_v) + or orig_v in _UNROM_LONG else "short" ) life = ( @@ -395,10 +397,7 @@ def syllable(match: re.Match[str]) -> str: word = _PARTIAL_PATTERN.sub(syllable, word) return word - words_iter = list(re.finditer(f"{_THAI_RANGE}+", text)) - for match_word in words_iter: - processed = process_word(match_word) - text = text.replace(match_word.group(0), processed, 1) + text = re.sub(f"{_THAI_RANGE}+", lambda m: process_word(m), text) text = re.sub(r"[๐-๙]", lambda m: _SYMBOLS.get(m.group(0), m.group(0)), text) @@ -414,7 +413,7 @@ def syllable(match: re.Match[str]) -> str: text = re.sub(r"ng\$", "ng", text) if mode == "ipa": - text, _count_syl = re.subn(r"[ \-–]", ".", text) + text = re.sub(r"[ \-–]", ".", text) text = re.sub(r"([aiɯu])([˥-˩]+)$", r"\1ʔ\2", text) return unicodedata.normalize("NFC", text) From 25a89c82a72cf1148e1d00025cf30bc7dde12339 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:54:10 +0000 Subject: [PATCH 25/32] Refine th-pron transliteration edge handling --- pythainlp/transliterate/th_pron.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/th_pron.py index c30daeac0..d12711982 100644 --- a/pythainlp/transliterate/th_pron.py +++ b/pythainlp/transliterate/th_pron.py @@ -287,11 +287,7 @@ def _c2_decomp(c2_char: str, seq_idx: int) -> str: - converted_c2: list[str] = [] - for character in list(c2_char): - val = _CODA.get(character) - converted_c2.append(val[seq_idx] if val else "") - return "".join(converted_c2) + return "".join(_CODA.get(char, ["", "", ""])[seq_idx] for char in c2_char) def transliterate(text: str, mode: str = "ipa") -> str: @@ -326,10 +322,10 @@ def syllable(match: re.Match[str]) -> str: if mgvc_match: g_new, v2_new, c2_new = mgvc_match.groups() c1, g, v2, c2 = "ห", g_new, v2_new, c2_new - if g != "" and v2 != "ย": + if g and v2 != "ย": c1, g = c1 + g, "" - if g == "ล" and (v2 + c2) == "": + if g == "ล" and not (v2 + c2): c2 = g g = "" @@ -344,7 +340,7 @@ def syllable(match: re.Match[str]) -> str: v_lookup = _VOWEL[openness].get(v1 + v2) v = v_lookup[seq_idx] if v_lookup else (v1 + v2) g_clean = g.replace("ฺ", "") - g_lookup = _INITIAL.get(g_clean) or _INITIAL[""] + g_lookup = _INITIAL.get(g_clean, _INITIAL[""]) g = cast(list[str], g_lookup["seq"])[seq_idx] c1_clean = c1.replace("ฺ", "") @@ -365,7 +361,7 @@ def syllable(match: re.Match[str]) -> str: "live" if re.search(r"[มญณนรลฬง]", c2) or (orig_v.endswith("ย") and v.endswith("i")) - or (c2 + length == "long") + or (c2 == "" and length == "long") or _LIVE_EXC.get(orig_v) else "dead" ) From cee8a1b251b6a1e89f74dd97b4ffc1262e86984c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:56:10 +0000 Subject: [PATCH 26/32] Polish th-pron docs and imports --- pythainlp/transliterate/th_pron.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/th_pron.py index d12711982..b47663ec5 100644 --- a/pythainlp/transliterate/th_pron.py +++ b/pythainlp/transliterate/th_pron.py @@ -293,12 +293,17 @@ def _c2_decomp(c2_char: str, seq_idx: int) -> str: def transliterate(text: str, mode: str = "ipa") -> str: """Transliterate Thai text using Wiktionary th-pron logic. - :param str text: Thai text to transliterate. + :param str text: Thai text input (single word or text fragment). :param str mode: Output mode: ``paiboon``, ``royin``, or ``ipa``. Unsupported modes return the input text unchanged. :return: Transliterated text. :rtype: str + + :Example: + + >>> transliterate("แมว", mode="royin") + 'maeo' """ seq_idx = _SYSTEMS.get(mode) if seq_idx is None: @@ -419,8 +424,13 @@ def get_word_dict(word: str) -> dict[str, str]: """Return th-pron transliteration outputs in all supported systems. :param str word: Thai input word. - :return: Dict containing ``word``, ``paiboon``, ``royin``, and ``ipa``. + :return: ``dict[str, str]`` with ``word``, ``paiboon``, ``royin``, and ``ipa``. :rtype: dict[str, str] + + :Example: + + >>> get_word_dict("แมว") + {'word': 'แมว', 'paiboon': 'mɛɛo', 'royin': 'maeo', 'ipa': 'mɛːw˧'} """ return { "word": word, From a327930d2631a2d0280655baa0a532fde25cf057 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:58:20 +0000 Subject: [PATCH 27/32] Rename th-pron entrypoint and clean imports --- pythainlp/transliterate/__init__.py | 4 +--- pythainlp/transliterate/th_pron.py | 15 ++++++++++----- tests/core/test_transliterate_th_pron.py | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 44ec30cab..373848966 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -16,7 +16,5 @@ from pythainlp.transliterate.spoonerism import puan from pythainlp.transliterate.th_pron import ( get_word_dict, -) -from pythainlp.transliterate.th_pron import ( - transliterate as th_pron_transliterate, + th_pron_transliterate, ) diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/th_pron.py index b47663ec5..566aa4a86 100644 --- a/pythainlp/transliterate/th_pron.py +++ b/pythainlp/transliterate/th_pron.py @@ -290,7 +290,7 @@ def _c2_decomp(c2_char: str, seq_idx: int) -> str: return "".join(_CODA.get(char, ["", "", ""])[seq_idx] for char in c2_char) -def transliterate(text: str, mode: str = "ipa") -> str: +def th_pron_transliterate(text: str, mode: str = "ipa") -> str: """Transliterate Thai text using Wiktionary th-pron logic. :param str text: Thai text input (single word or text fragment). @@ -302,7 +302,7 @@ def transliterate(text: str, mode: str = "ipa") -> str: :Example: - >>> transliterate("แมว", mode="royin") + >>> th_pron_transliterate("แมว", mode="royin") 'maeo' """ seq_idx = _SYSTEMS.get(mode) @@ -434,7 +434,12 @@ def get_word_dict(word: str) -> dict[str, str]: """ return { "word": word, - "paiboon": transliterate(word, mode="paiboon"), - "royin": transliterate(word, mode="royin"), - "ipa": transliterate(word, mode="ipa"), + "paiboon": th_pron_transliterate(word, mode="paiboon"), + "royin": th_pron_transliterate(word, mode="royin"), + "ipa": th_pron_transliterate(word, mode="ipa"), } + + +def transliterate(text: str, mode: str = "ipa") -> str: + """Backward-compatible alias for :func:`th_pron_transliterate`.""" + return th_pron_transliterate(text, mode=mode) diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_th_pron.py index f1f5c778b..a26a0ade1 100644 --- a/tests/core/test_transliterate_th_pron.py +++ b/tests/core/test_transliterate_th_pron.py @@ -14,7 +14,7 @@ def test_transliterate_supported_modes(self): self.assertEqual(transliterate("คน", mode="paiboon"), "kon") self.assertEqual(transliterate("คน", mode="ipa"), "kʰon˧") - def test_transliterate_preserves_unknown_mode(self): + def test_transliterate_returns_input_for_unknown_mode(self): self.assertEqual(transliterate("แมว", mode="unknown"), "แมว") def test_thai_digits_conversion(self): From 4949dc23582865d325a274536350c30101014965 Mon Sep 17 00:00:00 2001 From: wannaphong Date: Wed, 24 Jun 2026 09:03:12 +0000 Subject: [PATCH 28/32] Add more test cases --- tests/core/test_transliterate_th_pron.py | 128 +++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_th_pron.py index a26a0ade1..87b3d5287 100644 --- a/tests/core/test_transliterate_th_pron.py +++ b/tests/core/test_transliterate_th_pron.py @@ -30,3 +30,131 @@ def test_get_word_dict(self): "ipa": "pra˨˩.t͡ɕʰaː˧", }, ) + + def test_transliterate_paiboon_cases(self): + # test cases from https://w.wiki/RifX + test_cases = [ + ('เฮิ็้ย', 'hə́i'), + ('เฉิ็ย', 'chə̌i'), + ('เงิ็น', 'ngən'), + ('เกดส', 'gèets'), # เกตส์ (Gates) + ('มันส', 'mans'), # มันส์ + ('ไอ๊ส', 'áis'), # ไอซ์ (ice) + ('เซ็กส', 'séks'), # เซ็กส์ (sex) + ('เอ็๊กส', 'éks'), # เอกซ์ & เอ็กซ์ & เอ๊กซ์ (ex) + ('เฮ้าส', 'háos'), # เฮาส์ & เฮ้าส์ (house) + ('เม้าส', 'máos'), # เมาส์ & เม้าส์ (mouse) + ('ทฺรำ-เป็ด', 'tram-bpèt'), # ทรัมเป็ต + ('ห็อย', 'hɔ̌i'), + ('หฺม็อย', 'mɔ̌i'), + ('หฺมั่น-โถว', 'màn-tǒow'), + ('เด๊ด-สะ-มอ-เร่', 'déet-sà-mɔɔ-rêe'), + ('เห', 'hěe'), + ('แคฺล', 'klɛɛ'), + ('แคล', 'kɛɛl'), + ('เพฺล', 'plee'), + ('เพล', 'peel'), + ('เปฺล', 'bplee'), + ('เปล', 'bpeel'), + ('เบล', 'beel'), + ('เซล', 'seel'), + ('โพล', 'pool'), + ('รา-ชา-ทิ-ราด', 'raa-chaa-tí-râat'), + ('ขฺวน-ขฺวาย', 'kwǒn-kwǎai'), # ขวนขวาย Only the word ขวน read as kwǒn instead of kǔuan. + ('ข่วน', 'kùuan'), + ('หอน', 'hɔ̌ɔn'), + ('โหน', 'hǒon'), # ห้อยโหน homograph issue + ('สะ-โหฺน', 'sà-nǒo'), # โสน homograph issue + ('แหน', 'hɛ̌ɛn'), # หวงแหน homograph issue + ('แหฺน', 'nɛ̌ɛ'), # จอกแหน homograph issue + ('แถ็ว', 'tɛ̌o'), # แถว + ('ซวง', 'suuang'), + ('น้ำ', 'nám'), + ('หฺมาย', 'mǎai'), + ('แห็่ง', 'hɛ̀ng'), + ('หน', 'hǒn'), + ('เหด-สุด-วิ-ไส', 'hèet-sùt-wí-sǎi'), + ('ไหฺย่', 'yài'), + ('หก', 'hòk'), + ('หอย', 'hɔ̌ɔi'), + ('กับ', 'gàp'), + ('ธรรม', 'tam'), + ('ปฺระ-ชา', 'bprà-chaa'), + ('นะ-คอน', 'ná-kɔɔn'), + ('บาด', 'bàat'), + ('บ้า', 'bâa'), + ('แข็ง', 'kɛ̌ng'), + ('แกะ', 'gɛ̀'), + ('แดง', 'dɛɛng'), + ('แปฺล', 'bplɛɛ'), + ('ผฺล็อง', 'plɔ̌ng'), + ('เกาะ', 'gɔ̀'), + ('นอน', 'nɔɔn'), + ('พ่อ', 'pɔ̂ɔ'), + ('เห็ด', 'hèt'), + ('เล็่น', 'lên'), + ('เตะ', 'dtè'), + ('เพฺลง', 'pleeng'), + ('เท-วี', 'tee-wii'), + ('เยอะ', 'yə́'), + ('เดิน', 'dəən'), + ('เผฺลอ', 'plə̌ə'), + ('ตก', 'dtòk'), + ('โต๊ะ', 'dtó'), + ('โชค', 'chôok'), + ('โม-โห', 'moo-hǒo'), + ('คิด', 'kít'), + ('มิ-ถุน', 'mí-tǔn'), + ('หิ-มะ', 'hì-má'), + ('อีก', 'ìik'), + ('จี้', 'jîi'), + ('ลึก', 'lʉ́k'), + ('รึ', 'rʉ́'), + ('กฺลืน', 'glʉʉn'), + ('ชื่อ', 'chʉ̂ʉ'), + ('คุก', 'kúk'), + ('จุ-ฬา', 'jù-laa'), + ('ลูก', 'lûuk'), + ('ปู', 'bpuu'), + ('เดี๊ยะ', 'día'), + ('เปาะ-เปี๊ยะ', 'bpɔ̀-bpía'), + ('ปอ-เปี๊ยะ', 'bpɔɔ-bpía'), + ('เปฺรี๊ยะ', 'bpría'), + ('เตียง', 'dtiiang'), + ('เมีย', 'miia'), + ('เอือะ', 'ʉ̀a'), + ('เรื่อง', 'rʉ̂ʉang'), + ('เรือ', 'rʉʉa'), + ('ผฺลัวะ', 'plùa'), + ('นวด', 'nûuat'), + ('ตัว', 'dtuua'), + ('ไม่', 'mâi'), + ('ใส่', 'sài'), + ('วัย', 'wai'), + ('ไทย', 'tai'), + ('ไม้', 'mái'), + ('หาย', 'hǎai'), + ('ผฺล็อย', 'plɔ̌i'), + ('ซอย', 'sɔɔi'), + ('เลย', 'ləəi'), + ('โดย', 'dooi'), + ('ทุย', 'tui'), + ('เหฺนื่อย', 'nʉ̀ai'), + ('สวย', 'sǔai'), + ('เรา', 'rao'), + ('ขาว', 'kǎao'), + ('แมว', 'mɛɛo'), + ('เกอว', 'gəəo'), + ('เร็ว', 'reo'), + ('เอว', 'eeo'), + ('หิว', 'hǐu'), + ('เขียว', 'kǐao'), + ('ทำ', 'tam'), + ] + + for thai_text, expected_translit in test_cases: + with self.subTest(thai_text=thai_text): + self.assertEqual(transliterate(thai_text, mode="paiboon"), expected_translit) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From df82ac6312e2e5757032098fbf8970569035c454 Mon Sep 17 00:00:00 2001 From: wannaphong Date: Wed, 24 Jun 2026 09:07:48 +0000 Subject: [PATCH 29/32] Fixed W292 --- tests/core/test_transliterate_th_pron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_th_pron.py index 87b3d5287..b5351960a 100644 --- a/tests/core/test_transliterate_th_pron.py +++ b/tests/core/test_transliterate_th_pron.py @@ -157,4 +157,4 @@ def test_transliterate_paiboon_cases(self): self.assertEqual(transliterate(thai_text, mode="paiboon"), expected_translit) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 48d61a96133bbcf1df540050f0f4d3fb8b14fa7d Mon Sep 17 00:00:00 2001 From: wannaphong Date: Wed, 24 Jun 2026 09:30:39 +0000 Subject: [PATCH 30/32] Update transliterate_wiktionary --- docs/api/transliterate.rst | 6 +++++ pythainlp/transliterate/__init__.py | 6 ++--- .../{th_pron.py => wiktionary.py} | 9 ++++--- ...on.py => test_transliterate_wiktionary.py} | 24 ++++++++++++------- 4 files changed, 31 insertions(+), 14 deletions(-) rename pythainlp/transliterate/{th_pron.py => wiktionary.py} (98%) rename tests/core/{test_transliterate_th_pron.py => test_transliterate_wiktionary.py} (89%) diff --git a/docs/api/transliterate.rst b/docs/api/transliterate.rst index fc5c259c8..34b663e6b 100644 --- a/docs/api/transliterate.rst +++ b/docs/api/transliterate.rst @@ -17,6 +17,9 @@ Modules The `transliterate` function serves as a versatile transliteration tool, offering a range of transliteration engines to choose from. It provides flexibility and customization for your transliteration needs. +.. autofunction:: transliterate_wiktionary + :noindex: + .. autofunction:: pronunciate :noindex: @@ -27,6 +30,9 @@ Modules The `puan` function offers a unique transliteration feature known as "Puan." It provides a specialized transliteration method for Thai text and is an additional option for rendering Thai text into English characters. +.. autofunction:: get_word_dict + :noindex: + .. autoclass:: pythainlp.transliterate.wunsen.WunsenTransliterate :members: diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index 373848966..1840e9021 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -8,13 +8,13 @@ "pronunciate", "puan", "romanize", - "th_pron_transliterate", "transliterate", + "transliterate_wiktionary", ] from pythainlp.transliterate.core import pronunciate, romanize, transliterate from pythainlp.transliterate.spoonerism import puan -from pythainlp.transliterate.th_pron import ( +from pythainlp.transliterate.wiktionary import ( get_word_dict, - th_pron_transliterate, + transliterate_wiktionary, ) diff --git a/pythainlp/transliterate/th_pron.py b/pythainlp/transliterate/wiktionary.py similarity index 98% rename from pythainlp/transliterate/th_pron.py rename to pythainlp/transliterate/wiktionary.py index 566aa4a86..122e42321 100644 --- a/pythainlp/transliterate/th_pron.py +++ b/pythainlp/transliterate/wiktionary.py @@ -1,7 +1,10 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai pronunciation transliteration from Wiktionary th-pron module.""" +""" +Thai pronunciation transliteration from Wiktionary th-pron module. +Source code: https://en.wiktionary.org/wiki/Module:th-pron +""" from __future__ import annotations @@ -421,7 +424,7 @@ def syllable(match: re.Match[str]) -> str: def get_word_dict(word: str) -> dict[str, str]: - """Return th-pron transliteration outputs in all supported systems. + """Return Wiktionary transliteration outputs in all supported systems. :param str word: Thai input word. :return: ``dict[str, str]`` with ``word``, ``paiboon``, ``royin``, and ``ipa``. @@ -440,6 +443,6 @@ def get_word_dict(word: str) -> dict[str, str]: } -def transliterate(text: str, mode: str = "ipa") -> str: +def transliterate_wiktionary(text: str, mode: str = "ipa") -> str: """Backward-compatible alias for :func:`th_pron_transliterate`.""" return th_pron_transliterate(text, mode=mode) diff --git a/tests/core/test_transliterate_th_pron.py b/tests/core/test_transliterate_wiktionary.py similarity index 89% rename from tests/core/test_transliterate_th_pron.py rename to tests/core/test_transliterate_wiktionary.py index b5351960a..e025ae93d 100644 --- a/tests/core/test_transliterate_th_pron.py +++ b/tests/core/test_transliterate_wiktionary.py @@ -5,16 +5,21 @@ import unittest from pythainlp.transliterate import get_word_dict -from pythainlp.transliterate.th_pron import transliterate +from pythainlp.transliterate.wiktionary import transliterate_wiktionary class TransliterateThPronTestCase(unittest.TestCase): - def test_transliterate_supported_modes(self): - self.assertEqual(transliterate("แมว", mode="royin"), "maeo") - self.assertEqual(transliterate("คน", mode="paiboon"), "kon") - self.assertEqual(transliterate("คน", mode="ipa"), "kʰon˧") + def test_transliterate_wiktionary(self): + self.assertEqual( + transliterate_wiktionary("แมว", mode="royin"), "maeo" + ) + self.assertEqual( + transliterate_wiktionary("คน", mode="paiboon"), "kon" + ) + self.assertEqual( + transliterate_wiktionary("คน", mode="ipa"), "kʰon˧") - def test_transliterate_returns_input_for_unknown_mode(self): + def test_transliterate_wiktionary_returns_input_for_unknown_mode(self): self.assertEqual(transliterate("แมว", mode="unknown"), "แมว") def test_thai_digits_conversion(self): @@ -31,7 +36,7 @@ def test_get_word_dict(self): }, ) - def test_transliterate_paiboon_cases(self): + def test_transliterate_wiktionary_paiboon_cases(self): # test cases from https://w.wiki/RifX test_cases = [ ('เฮิ็้ย', 'hə́i'), @@ -154,7 +159,10 @@ def test_transliterate_paiboon_cases(self): for thai_text, expected_translit in test_cases: with self.subTest(thai_text=thai_text): - self.assertEqual(transliterate(thai_text, mode="paiboon"), expected_translit) + self.assertEqual( + transliterate_wiktionary(thai_text, mode="paiboon"), + expected_translit + ) if __name__ == '__main__': unittest.main() From 9422238949ae496d3e100b694d641dc6ed069343 Mon Sep 17 00:00:00 2001 From: wannaphong Date: Wed, 24 Jun 2026 09:37:37 +0000 Subject: [PATCH 31/32] Fixed test --- tests/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/__init__.py b/tests/core/__init__.py index 5ab824da0..66ca8a714 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -24,7 +24,7 @@ "tests.core.test_tokenize", "tests.core.test_tools", "tests.core.test_transliterate", - "tests.core.test_transliterate_th_pron", + "tests.core.test_transliterate_wiktionary", "tests.core.test_util", ] From 52c5f01f6642f78f1e1b8ca055132dc799ed6f7d Mon Sep 17 00:00:00 2001 From: wannaphong Date: Wed, 24 Jun 2026 09:40:17 +0000 Subject: [PATCH 32/32] Fixed test --- tests/core/test_transliterate_wiktionary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/test_transliterate_wiktionary.py b/tests/core/test_transliterate_wiktionary.py index e025ae93d..d7c19d2a9 100644 --- a/tests/core/test_transliterate_wiktionary.py +++ b/tests/core/test_transliterate_wiktionary.py @@ -20,10 +20,10 @@ def test_transliterate_wiktionary(self): transliterate_wiktionary("คน", mode="ipa"), "kʰon˧") def test_transliterate_wiktionary_returns_input_for_unknown_mode(self): - self.assertEqual(transliterate("แมว", mode="unknown"), "แมว") + self.assertEqual(transliterate_wiktionary("แมว", mode="unknown"), "แมว") def test_thai_digits_conversion(self): - self.assertEqual(transliterate("๑๒๓", mode="royin"), "123") + self.assertEqual(transliterate_wiktionary("๑๒๓", mode="royin"), "123") def test_get_word_dict(self): self.assertEqual(