From e1e746ae318246ccbfcbac1f2ee40d0a2d790cd5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 17:53:42 +0000 Subject: [PATCH 1/6] Initial plan From b5ec930f5bbc5708616dc9620fc87f93e178d700 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 17:59:14 +0000 Subject: [PATCH 2/6] Replace requests library with urllib.request from standard library Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 2 +- pythainlp/corpus/core.py | 63 +++++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d95aa7978..f488d1c01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ classifiers = [ ] # Core dependencies -dependencies = ["requests>=2.31", "tzdata; sys_platform == 'win32'"] +dependencies = ["tzdata; sys_platform == 'win32'"] [project.optional-dependencies] diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index e27af19f7..c57a40afe 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -22,15 +22,31 @@ def get_corpus_db(url: str): :param str url: URL corpus catalog """ - import requests + import json + from urllib.error import HTTPError, URLError + from urllib.request import Request, urlopen corpus_db = None try: - corpus_db = requests.get(url, timeout=10) - except requests.exceptions.HTTPError as http_err: + req = Request(url, headers={"User-Agent": "PyThaiNLP"}) + with urlopen(req, timeout=10) as response: + # Create a response object similar to requests.Response + class ResponseWrapper: + def __init__(self, response): + self.status_code = response.status + self.headers = response.headers + self._content = response.read() + + def json(self): + return json.loads(self._content.decode("utf-8")) + + corpus_db = ResponseWrapper(response) + except HTTPError as http_err: print(f"HTTP error occurred: {http_err}") - except requests.exceptions.RequestException as err: - print(f"Non-HTTP error occurred: {err}") + except URLError as err: + print(f"URL error occurred: {err}") + except Exception as err: + print(f"Error occurred: {err}") return corpus_db @@ -284,30 +300,31 @@ def _download(url: str, dst: str) -> int: """ CHUNK_SIZE = 64 * 1024 # 64 KiB - from urllib.request import urlopen + from urllib.request import Request, urlopen - import requests - - file_size = int(urlopen(url).info().get("Content-Length", -1)) - r = requests.get(url, stream=True, timeout=10) - with open(get_full_data_path(dst), "wb") as f: - pbar = None - try: - from tqdm.auto import tqdm - - pbar = tqdm(total=int(r.headers["Content-Length"])) - except ImportError: + req = Request(url, headers={"User-Agent": "PyThaiNLP"}) + with urlopen(req, timeout=10) as response: + file_size = int(response.info().get("Content-Length", -1)) + with open(get_full_data_path(dst), "wb") as f: pbar = None + try: + from tqdm.auto import tqdm + + pbar = tqdm(total=file_size) + except ImportError: + pbar = None - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): - if chunk: + while True: + chunk = response.read(CHUNK_SIZE) + if not chunk: + break f.write(chunk) if pbar: pbar.update(len(chunk)) - if pbar: - pbar.close() - else: - print("Done.") + if pbar: + pbar.close() + else: + print("Done.") return file_size From d4a4774ba3586e6a48f9562992675e5550c96685 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 18:01:38 +0000 Subject: [PATCH 3/6] Address code review feedback: improve code organization and error handling Co-authored-by: bact <128572+bact@users.noreply.github.com> --- CHANGELOG.md | 1 + pythainlp/corpus/core.py | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index feb4f3959..f568ac72b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ See PR for prompt and details. - Consolidate configuration into pyproject.toml #1188 - Update type hints; Use Python 3.9 features #1189 #1190 - Fix KeyError when transliterating text with Thai alphabet Kho Khon (U+0E05) #1187 +- Replace requests library with urllib.request from standard library to reduce core dependencies ## Version 5.1.2 -> 5.2.0 diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index c57a40afe..15c0ee800 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -17,12 +17,27 @@ _CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE") +class _ResponseWrapper: + """Wrapper to provide requests.Response-like interface for urllib response.""" + + def __init__(self, response): + self.status_code = response.status + self.headers = response.headers + self._content = response.read() + + def json(self): + """Parse JSON content from response.""" + try: + return json.loads(self._content.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as err: + raise ValueError(f"Failed to parse JSON response: {err}") + + def get_corpus_db(url: str): """Get corpus catalog from server. :param str url: URL corpus catalog """ - import json from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -30,17 +45,7 @@ def get_corpus_db(url: str): try: req = Request(url, headers={"User-Agent": "PyThaiNLP"}) with urlopen(req, timeout=10) as response: - # Create a response object similar to requests.Response - class ResponseWrapper: - def __init__(self, response): - self.status_code = response.status - self.headers = response.headers - self._content = response.read() - - def json(self): - return json.loads(self._content.decode("utf-8")) - - corpus_db = ResponseWrapper(response) + corpus_db = _ResponseWrapper(response) except HTTPError as http_err: print(f"HTTP error occurred: {http_err}") except URLError as err: @@ -314,10 +319,7 @@ def _download(url: str, dst: str) -> int: except ImportError: pbar = None - while True: - chunk = response.read(CHUNK_SIZE) - if not chunk: - break + while chunk := response.read(CHUNK_SIZE): f.write(chunk) if pbar: pbar.update(len(chunk)) From 7e64495123d8fca91786d2166b72d236423ae099 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 18:09:07 +0000 Subject: [PATCH 4/6] Use standard browser User-Agent string for HTTP requests Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 15c0ee800..56c6ea918 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -15,6 +15,10 @@ from pythainlp.tools import get_full_data_path _CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE") +_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" +) class _ResponseWrapper: @@ -43,7 +47,7 @@ def get_corpus_db(url: str): corpus_db = None try: - req = Request(url, headers={"User-Agent": "PyThaiNLP"}) + req = Request(url, headers={"User-Agent": _USER_AGENT}) with urlopen(req, timeout=10) as response: corpus_db = _ResponseWrapper(response) except HTTPError as http_err: @@ -307,7 +311,7 @@ def _download(url: str, dst: str) -> int: from urllib.request import Request, urlopen - req = Request(url, headers={"User-Agent": "PyThaiNLP"}) + req = Request(url, headers={"User-Agent": _USER_AGENT}) with urlopen(req, timeout=10) as response: file_size = int(response.info().get("Content-Length", -1)) with open(get_full_data_path(dst), "wb") as f: From 9507e0d8dbe2df3fbcb7d73c3c37f312aa5db853 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 18:12:39 +0000 Subject: [PATCH 5/6] Use library-appropriate User-Agent identifying PyThaiNLP client Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 56c6ea918..8baeb17e9 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -9,6 +9,7 @@ import json import os import re +import sys from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path @@ -16,8 +17,9 @@ _CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE") _USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + f"PyThaiNLP/{__version__} " + f"(Python/{sys.version_info.major}.{sys.version_info.minor}; " + f"{sys.platform})" ) From 16fbdc1419b9789b487d062bb744b9c4f2bcca2b Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 16 Jan 2026 18:15:51 +0000 Subject: [PATCH 6/6] Fix duplicate entry and update CHANGELOG Removed duplicate entry for KeyError fix and updated the requests library replacement note. --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f568ac72b..bca5fc44d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,10 +19,11 @@ Some features and fixes in this version are AI assisted. See PR for prompt and details. - Lazy load dictionaries to reduce memory usage #1186 +- Fix KeyError when transliterating text with Thai alphabet Kho Khon (U+0E05) #1187 - Consolidate configuration into pyproject.toml #1188 - Update type hints; Use Python 3.9 features #1189 #1190 -- Fix KeyError when transliterating text with Thai alphabet Kho Khon (U+0E05) #1187 -- Replace requests library with urllib.request from standard library to reduce core dependencies +- Replace requests library with urllib.request from standard library to reduce + core dependencies #1211 ## Version 5.1.2 -> 5.2.0