From 8999b8cb42ce6f7f666b4f07efbb8daa7fa0ca57 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:30:03 +0000 Subject: [PATCH 1/5] Initial plan From 61f0c8f5076fa6e89c12e29bb65f7078fb91b103 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:50:22 +0000 Subject: [PATCH 2/5] Add TypedDicts: RougeScore, TokenizationStats family, CorefResult; export from public APIs - metrics.py: add RougeScore TypedDict (precision/recall/fmeasure); rouge_score() now returns dict[str, RougeScore] instead of dict[str, tuple[float, float, float]] (breaking change) - word_tokenization.py: add CharLevelStats, WordLevelStats, GlobalStats, TokenizationStats TypedDicts; update compute_stats() return type; _flatten_result() uses @overload to accept both TokenizationStats and generic Mapping without a cast - benchmarks/__init__.py: export RougeScore, CharLevelStats, WordLevelStats, GlobalStats, TokenizationStats - _fastcoref.py: rename CorefResultDict -> CorefResult; alias the fastcoref.modeling.CorefResult import as FastCorefResult to avoid clash - coref/__init__.py: export CorefResult - coref/core.py: update return type and fallback return to use CorefResult - tag/named_entity.py: improve EntitySpan docstring - tests/extra/testx_benchmarks.py: update rouge tests for named-field access; add test_rouge_score_return_type and test_compute_stats_return_type - CHANGELOG.md: add migration notes for all TypedDicts Co-authored-by: bact <128572+bact@users.noreply.github.com> Agent-Logs-Url: https://github.com/PyThaiNLP/pythainlp/sessions/de0a9df1-25ba-4e89-a047-7ee0664753c1 --- CHANGELOG.md | 67 +++++++++++++++++ pythainlp/benchmarks/__init__.py | 14 +++- pythainlp/benchmarks/metrics.py | 40 +++++++--- pythainlp/benchmarks/word_tokenization.py | 90 ++++++++++++++++++----- pythainlp/coref/__init__.py | 3 +- pythainlp/coref/_fastcoref.py | 11 +-- pythainlp/coref/core.py | 11 ++- pythainlp/tag/named_entity.py | 3 +- tests/extra/testx_benchmarks.py | 86 +++++++++++++++++----- 9 files changed, 265 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d66e9a4..75597b998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,73 @@ and this project adheres to ## [Unreleased] +### Added + +- `RougeScore` TypedDict in `pythainlp.benchmarks`: precision, recall, and + fmeasure fields replace the previous `tuple[float, float, float]` value + in the `rouge_score()` return dict. **Breaking change** – migrate as follows: + + ```python + # Before (tuple indexing, error-prone) + precision, recall, fmeasure = scores["rouge1"] + + # After (named fields) + precision = scores["rouge1"]["precision"] + recall = scores["rouge1"]["recall"] + fmeasure = scores["rouge1"]["fmeasure"] + ``` + +- `CharLevelStats`, `WordLevelStats`, `GlobalStats`, and `TokenizationStats` + TypedDicts in `pythainlp.benchmarks`: give named, type-safe access to the + dict returned by `word_tokenization.compute_stats()`. + + ```python + # Before (opaque nested dict) + result = compute_stats(ref, hyp) + tp = result["char_level"]["tp"] + + # After (same access, now type-safe with TokenizationStats) + from pythainlp.benchmarks import TokenizationStats + result: TokenizationStats = compute_stats(ref, hyp) + tp = result["char_level"]["tp"] + ``` + +- `CorefResult` TypedDict is now exported from `pythainlp.coref`. + `coreference_resolution()` return type updated from `list[dict[str, Any]]` + to `list[CorefResult]`. + + ```python + # Before + from pythainlp.coref import coreference_resolution + + # After – import the TypedDict for type annotations + from pythainlp.coref import CorefResult, coreference_resolution + ``` + +- `EntitySpan` TypedDict (`pythainlp.tag.named_entity`, #1363) and + `BleuScore` TypedDict (`pythainlp.benchmarks`, #1365) were introduced in + previous releases. Migration notes for completeness: + + ```python + # EntitySpan – Before (plain dict) + entity = {"text": ["สมชาย"], "span": [0, 1], "entity_type": "PERSON"} + + # EntitySpan – After (TypedDict constructor) + from pythainlp.tag.named_entity import EntitySpan + entity = EntitySpan(text=["สมชาย"], span=[0, 1], entity_type="PERSON") + ``` + + ```python + # BleuScore – Before (plain dict, no type hint) + score = bleu_score(refs, hyps) + print(score["bleu"]) + + # BleuScore – After (TypedDict annotation) + from pythainlp.benchmarks import BleuScore, bleu_score + score: BleuScore = bleu_score(refs, hyps) + print(score["bleu"]) + ``` + ### Fixed - thai2rom_onnx: fix ONNX encoder model and fix inference bugs (#1349) diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index 716230cef..f610ae087 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -5,6 +5,11 @@ __all__: list[str] = [ "BleuScore", + "CharLevelStats", + "GlobalStats", + "RougeScore", + "TokenizationStats", + "WordLevelStats", "benchmark", "bleu_score", "character_error_rate", @@ -14,9 +19,16 @@ from pythainlp.benchmarks.metrics import ( BleuScore, + RougeScore, bleu_score, character_error_rate, rouge_score, word_error_rate, ) -from pythainlp.benchmarks.word_tokenization import benchmark +from pythainlp.benchmarks.word_tokenization import ( + CharLevelStats, + GlobalStats, + TokenizationStats, + WordLevelStats, + benchmark, +) diff --git a/pythainlp/benchmarks/metrics.py b/pythainlp/benchmarks/metrics.py index b2eb963ee..ae793c181 100644 --- a/pythainlp/benchmarks/metrics.py +++ b/pythainlp/benchmarks/metrics.py @@ -16,7 +16,7 @@ class BleuScore(TypedDict): - """BLEU score""" + """BLEU score components returned by :func:`bleu_score`.""" bleu: float # BLEU score as a percentage (0.0 to 100.0) precisions: list[float] @@ -26,6 +26,14 @@ class BleuScore(TypedDict): ref_length: int +class RougeScore(TypedDict): + """Precision, recall, and F-measure for a single ROUGE type.""" + + precision: float + recall: float + fmeasure: float + + def _get_ngrams(tokens: list[str], n: int) -> list[tuple[str, ...]]: """ Get n-grams from a list of tokens. @@ -249,7 +257,7 @@ def rouge_score( hypothesis: str, tokenize: str = "newmm", rouge_types: Optional[list[str]] = None, -) -> dict[str, tuple[float, float, float]]: +) -> dict[str, RougeScore]: """ Calculate ROUGE scores for Thai text with automatic tokenization. @@ -269,8 +277,9 @@ def rouge_score( :param Optional[list[str]] rouge_types: list of ROUGE types to calculate. Default is ["rouge1", "rouge2", "rougeL"] - :return: dictionary mapping ROUGE type to (precision, recall, fmeasure) - :rtype: dict[str, tuple[float, float, float]] + :return: dictionary mapping ROUGE type to a :class:`RougeScore` typed dict + with ``'precision'``, ``'recall'``, and ``'fmeasure'`` keys. + :rtype: dict[str, RougeScore] :Example: :: @@ -280,9 +289,9 @@ def rouge_score( reference = "สวัสดีครับ วันนี้อากาศดีมาก" hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" scores = rouge_score(reference, hypothesis) - print(f"ROUGE-1 F-measure: {scores['rouge1'][2]:.4f}") - print(f"ROUGE-2 F-measure: {scores['rouge2'][2]:.4f}") - print(f"ROUGE-L F-measure: {scores['rougeL'][2]:.4f}") + print(f"ROUGE-1 F-measure: {scores['rouge1']['fmeasure']:.4f}") + print(f"ROUGE-2 F-measure: {scores['rouge2']['fmeasure']:.4f}") + print(f"ROUGE-L F-measure: {scores['rougeL']['fmeasure']:.4f}") """ from pythainlp.tokenize import word_tokenize @@ -297,7 +306,7 @@ def rouge_score( hypothesis, engine=tokenize, keep_whitespace=False ) - result: dict[str, tuple[float, float, float]] = {} + result: dict[str, RougeScore] = {} for rouge_type in rouge_types: if rouge_type == "rouge1": @@ -309,9 +318,12 @@ def rouge_score( ref_count = len(ref_tokens) hyp_count = len(hyp_tokens) - result[rouge_type] = _calculate_precision_recall_fmeasure( + precision, recall, fmeasure = _calculate_precision_recall_fmeasure( overlap, hyp_count, ref_count ) + result[rouge_type] = RougeScore( + precision=precision, recall=recall, fmeasure=fmeasure + ) elif rouge_type == "rouge2": # Bigram-based @@ -325,9 +337,12 @@ def rouge_score( ref_count = len(ref_bigrams) hyp_count = len(hyp_bigrams) - result[rouge_type] = _calculate_precision_recall_fmeasure( + precision, recall, fmeasure = _calculate_precision_recall_fmeasure( overlap, hyp_count, ref_count ) + result[rouge_type] = RougeScore( + precision=precision, recall=recall, fmeasure=fmeasure + ) elif rouge_type == "rougeL": # Longest Common Subsequence-based @@ -335,9 +350,12 @@ def rouge_score( ref_count = len(ref_tokens) hyp_count = len(hyp_tokens) - result[rouge_type] = _calculate_precision_recall_fmeasure( + precision, recall, fmeasure = _calculate_precision_recall_fmeasure( lcs_len, hyp_count, ref_count ) + result[rouge_type] = RougeScore( + precision=precision, recall=recall, fmeasure=fmeasure + ) return result diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 01b67fee4..be02953e5 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -5,7 +5,8 @@ import re import sys -from typing import TYPE_CHECKING, Union +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypedDict, Union, overload if TYPE_CHECKING: import numpy as np @@ -29,6 +30,41 @@ TAILING_SEP_RX: re.Pattern[str] = re.compile(f"{re.escape(SEPARATOR)}$") +class CharLevelStats(TypedDict): + """Character-level confusion matrix statistics for tokenization.""" + + tp: int + fp: int + tn: int + fn: int + + +class WordLevelStats(TypedDict): + """Word-level tokenization statistics.""" + + correctly_tokenised_words: int + total_words_in_sample: int + total_words_in_ref_sample: int + + +class GlobalStats(TypedDict): + """Global tokenization indicators as a binary indicator string.""" + + tokenisation_indicators: str + + +# Functional form is required because 'global' is a Python reserved keyword. +TokenizationStats = TypedDict( + "TokenizationStats", + { + "char_level": CharLevelStats, + "word_level": WordLevelStats, + "global": GlobalStats, + }, +) +"""Tokenization quality statistics at character, word, and global level.""" + + def _f1(precision: float, recall: float) -> float: """Compute f1. @@ -43,8 +79,21 @@ def _f1(precision: float, recall: float) -> float: return 2 * precision * recall / (precision + recall) +@overload +def _flatten_result( + my_dict: TokenizationStats, sep: str = ... +) -> dict[str, Union[int, str]]: ... + + +@overload +def _flatten_result( + my_dict: Mapping[str, Mapping[str, Union[int, str]]], sep: str = ... +) -> dict[str, Union[int, str]]: ... + + def _flatten_result( - my_dict: dict[str, dict[str, Union[int, str]]], sep: str = ":" + my_dict: Any, + sep: str = ":", ) -> dict[str, Union[int, str]]: """Flatten two-dimension dictionary. @@ -55,8 +104,9 @@ def _flatten_result( { "a:b": 7 } - :param dict[str, dict[str, Union[int, str]]] my_dict: dictionary - containing stats + :param my_dict: dictionary containing stats + :type my_dict: TokenizationStats or + collections.abc.Mapping[str, collections.abc.Mapping[str, Union[int, str]]] :param str sep: separator between the two keys (default: ":") :return: a one-dimension dictionary with keys combined @@ -139,7 +189,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: def compute_stats( ref_sample: str, raw_sample: str -) -> dict[str, dict[str, Union[int, str]]]: +) -> TokenizationStats: """Compute statistics for tokenization quality These statistics include: @@ -156,7 +206,7 @@ def compute_stats( :param str samples: samples that we want to evaluate :return: metrics at character- and word-level and indicators of correctly tokenized words - :rtype: dict[str, dict[str, Union[int, str]]] + :rtype: TokenizationStats """ import numpy as np @@ -194,20 +244,20 @@ def compute_stats( tokenization_indicators_str = list(map(str, tokenization_indicators)) return { - "char_level": { - "tp": c_tp, - "fp": c_fp, - "tn": c_tn, - "fn": c_fn, - }, - "word_level": { - "correctly_tokenised_words": correctly_tokenised_words, - "total_words_in_sample": int(np.sum(sample_arr)), - "total_words_in_ref_sample": int(np.sum(ref_sample_arr)), - }, - "global": { - "tokenisation_indicators": "".join(tokenization_indicators_str) - }, + "char_level": CharLevelStats( + tp=c_tp, + fp=c_fp, + tn=c_tn, + fn=c_fn, + ), + "word_level": WordLevelStats( + correctly_tokenised_words=correctly_tokenised_words, + total_words_in_sample=int(np.sum(sample_arr)), + total_words_in_ref_sample=int(np.sum(ref_sample_arr)), + ), + "global": GlobalStats( + tokenisation_indicators="".join(tokenization_indicators_str), + ), } diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py index 883bc970c..2b85347ad 100644 --- a/pythainlp/coref/__init__.py +++ b/pythainlp/coref/__init__.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """PyThaiNLP Coreference Resolution""" -__all__: list[str] = ["coreference_resolution"] +__all__: list[str] = ["CorefResult", "coreference_resolution"] +from pythainlp.coref._fastcoref import CorefResult from pythainlp.coref.core import coreference_resolution diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index f1db6bc47..7c431201a 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -6,12 +6,13 @@ from typing import TYPE_CHECKING, Optional, TypedDict if TYPE_CHECKING: - from fastcoref.modeling import CorefModel, CorefResult + from fastcoref.modeling import CorefModel + from fastcoref.modeling import CorefResult as FastCorefResult from spacy.language import Language -class CorefResultDict(TypedDict): - """Dictionary representation of coreference resolution results.""" +class CorefResult(TypedDict): + """Coreference resolution result for a single text.""" text: str clusters_string: list[list[str]] @@ -42,14 +43,14 @@ def __init__( self.model_name, device=device, nlp=self.nlp ) - def _to_json(self, _predict: "CorefResult") -> CorefResultDict: + def _to_json(self, _predict: "FastCorefResult") -> CorefResult: return { "text": _predict.text, "clusters_string": _predict.get_clusters(as_strings=True), "clusters": _predict.get_clusters(as_strings=False), } - def predict(self, texts: list[str]) -> list[CorefResultDict]: + def predict(self, texts: list[str]) -> list[CorefResult]: return [ self._to_json(pred) for pred in self.model.predict(texts=texts) ] diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 94459e453..2776b5786 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -5,6 +5,8 @@ from typing import Any, Union, cast +from pythainlp.coref._fastcoref import CorefResult + _MODEL_CACHE: dict[tuple[str, str], Any] = {} @@ -12,7 +14,7 @@ def coreference_resolution( texts: Union[str, list[str]], model_name: str = "han-coref-v1.0", device: str = "cpu", -) -> list[dict[str, Any]]: +) -> list[CorefResult]: """Coreference Resolution :param Union[str, list[str]] texts: list of texts to apply coreference resolution to @@ -20,7 +22,7 @@ def coreference_resolution( :param str device: device for running coreference resolution model on\ ("cpu", "cuda", and others) :return: List of texts with coreference resolution - :rtype: list[dict[str, Any]] + :rtype: list[CorefResult] :Options for model_name: * *han-coref-v1.0* - (default) Han-Coref: Thai coreference resolution\ @@ -54,8 +56,9 @@ def coreference_resolution( model = _MODEL_CACHE.get(model_key) if model is not None: - return cast(list[dict[str, Any]], model.predict(texts)) + return cast(list[CorefResult], model.predict(texts)) return [ - {"text": text, "clusters_string": [], "clusters": []} for text in texts + CorefResult(text=text, clusters_string=[], clusters=[]) + for text in texts ] diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index e768b789c..c76ab525e 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -22,7 +22,8 @@ class EntitySpan(TypedDict): - """Entity span dictionary""" + """Named-entity span with its type, tokens, and position.""" + entity_type: str text: list[str] span: list[int] diff --git a/tests/extra/testx_benchmarks.py b/tests/extra/testx_benchmarks.py index d59611179..27c00ef41 100644 --- a/tests/extra/testx_benchmarks.py +++ b/tests/extra/testx_benchmarks.py @@ -9,6 +9,11 @@ from pythainlp.benchmarks import ( BleuScore, + CharLevelStats, + GlobalStats, + RougeScore, + TokenizationStats, + WordLevelStats, bleu_score, rouge_score, word_tokenization, @@ -53,6 +58,35 @@ def test_compute_stats(self): self.assertIsNotNone(result) + def test_compute_stats_return_type(self): + """Test that compute_stats returns a TokenizationStats typed dict.""" + ref = word_tokenization.preprocessing("อากาศ|ร้อน|มาก") + act = word_tokenization.preprocessing("อากาศ|ร้อนมาก") + + result: TokenizationStats = word_tokenization.compute_stats(ref, act) + + self.assertIsInstance(result, dict) + self.assertIn("char_level", result) + self.assertIn("word_level", result) + self.assertIn("global", result) + + char: CharLevelStats = result["char_level"] + self.assertIsInstance(char, dict) + self.assertIsInstance(char["tp"], int) + self.assertIsInstance(char["fp"], int) + self.assertIsInstance(char["tn"], int) + self.assertIsInstance(char["fn"], int) + + word: WordLevelStats = result["word_level"] + self.assertIsInstance(word, dict) + self.assertIsInstance(word["correctly_tokenised_words"], int) + self.assertIsInstance(word["total_words_in_sample"], int) + self.assertIsInstance(word["total_words_in_ref_sample"], int) + + glob: GlobalStats = result["global"] + self.assertIsInstance(glob, dict) + self.assertIsInstance(glob["tokenisation_indicators"], str) + def test_benchmark(self): expected = [] actual = [] @@ -195,16 +229,36 @@ def test_rouge_score_basic(self): self.assertIn("rouge2", scores) self.assertIn("rougeL", scores) - # Each score should be a tuple of (precision, recall, fmeasure) + # Each score is a RougeScore dict with precision, recall, fmeasure + for key in ["rouge1", "rouge2", "rougeL"]: + self.assertIsInstance(scores[key], dict) + self.assertIsInstance(scores[key]["precision"], float) + self.assertIsInstance(scores[key]["recall"], float) + self.assertIsInstance(scores[key]["fmeasure"], float) + self.assertGreaterEqual(scores[key]["precision"], 0.0) + self.assertLessEqual(scores[key]["precision"], 1.0) + self.assertGreaterEqual(scores[key]["recall"], 0.0) + self.assertLessEqual(scores[key]["recall"], 1.0) + self.assertGreaterEqual(scores[key]["fmeasure"], 0.0) + self.assertLessEqual(scores[key]["fmeasure"], 1.0) + + def test_rouge_score_return_type(self): + """Test that rouge_score returns dict[str, RougeScore] typed dicts.""" + reference = "สวัสดีครับ วันนี้อากาศดีมาก" + hypothesis = "สวัสดีค่ะ วันนี้อากาศดี" + + scores: dict[str, RougeScore] = rouge_score(reference, hypothesis) + + self.assertIsInstance(scores, dict) for key in ["rouge1", "rouge2", "rougeL"]: - self.assertEqual(len(scores[key]), 3) - precision, recall, fmeasure = scores[key] - self.assertGreaterEqual(precision, 0.0) - self.assertLessEqual(precision, 1.0) - self.assertGreaterEqual(recall, 0.0) - self.assertLessEqual(recall, 1.0) - self.assertGreaterEqual(fmeasure, 0.0) - self.assertLessEqual(fmeasure, 1.0) + score: RougeScore = scores[key] + self.assertIsInstance(score, dict) + self.assertIn("precision", score) + self.assertIn("recall", score) + self.assertIn("fmeasure", score) + self.assertIsInstance(score["precision"], float) + self.assertIsInstance(score["recall"], float) + self.assertIsInstance(score["fmeasure"], float) def test_rouge_score_identical_text(self): """Test ROUGE score when reference and hypothesis are identical.""" @@ -214,10 +268,9 @@ def test_rouge_score_identical_text(self): # All scores should be perfect (1.0) for key in ["rouge1", "rouge2", "rougeL"]: - precision, recall, fmeasure = scores[key] - self.assertEqual(precision, 1.0) - self.assertEqual(recall, 1.0) - self.assertEqual(fmeasure, 1.0) + self.assertEqual(scores[key]["precision"], 1.0) + self.assertEqual(scores[key]["recall"], 1.0) + self.assertEqual(scores[key]["fmeasure"], 1.0) def test_rouge_score_custom_types(self): """Test ROUGE score with custom rouge types.""" @@ -258,10 +311,9 @@ def test_rouge_score_no_overlap(self): # Scores should be 0 or very low since there's no overlap for key in ["rouge1", "rouge2", "rougeL"]: - precision, recall, fmeasure = scores[key] - self.assertGreaterEqual(precision, 0.0) - self.assertGreaterEqual(recall, 0.0) - self.assertGreaterEqual(fmeasure, 0.0) + self.assertGreaterEqual(scores[key]["precision"], 0.0) + self.assertGreaterEqual(scores[key]["recall"], 0.0) + self.assertGreaterEqual(scores[key]["fmeasure"], 0.0) def test_word_error_rate_basic(self): """Test WER with basic Thai text.""" From a1863d40dc3d9cbfdb483b6b257743fbf2ede979 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:57:48 +0000 Subject: [PATCH 3/5] =?UTF-8?q?Rename=20Stats=E2=86=92Stat=20(singular)=20?= =?UTF-8?q?and=20global=E2=86=92global=5F=20in=20TokenizationStat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CharLevelStats → CharLevelStat (singular, matches BleuScore/RougeScore) - WordLevelStats → WordLevelStat - GlobalStats → GlobalStat - TokenizationStats → TokenizationStat; converted from functional TypedDict form to class form now that global_ is a valid identifier - "global" key → "global_" in compute_stats() return value - Update __init__.py __all__ and imports - Update tests: import names, assertIn("global_"), typed annotations - Update CHANGELOG migration notes Co-authored-by: bact <128572+bact@users.noreply.github.com> Agent-Logs-Url: https://github.com/PyThaiNLP/pythainlp/sessions/d0fa4ca1-06a6-48cc-b353-4df24554e700 --- CHANGELOG.md | 12 ++++--- pythainlp/benchmarks/__init__.py | 16 +++++----- pythainlp/benchmarks/word_tokenization.py | 38 ++++++++++------------- tests/extra/testx_benchmarks.py | 20 ++++++------ 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75597b998..46adde6f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,19 +35,21 @@ and this project adheres to fmeasure = scores["rouge1"]["fmeasure"] ``` -- `CharLevelStats`, `WordLevelStats`, `GlobalStats`, and `TokenizationStats` +- `CharLevelStat`, `WordLevelStat`, `GlobalStat`, and `TokenizationStat` TypedDicts in `pythainlp.benchmarks`: give named, type-safe access to the - dict returned by `word_tokenization.compute_stats()`. + dict returned by `word_tokenization.compute_stats()`. The global-level key + is `"global_"` (trailing underscore avoids the Python reserved word). ```python # Before (opaque nested dict) result = compute_stats(ref, hyp) tp = result["char_level"]["tp"] - # After (same access, now type-safe with TokenizationStats) - from pythainlp.benchmarks import TokenizationStats - result: TokenizationStats = compute_stats(ref, hyp) + # After (same access, now type-safe with TokenizationStat) + from pythainlp.benchmarks import TokenizationStat + result: TokenizationStat = compute_stats(ref, hyp) tp = result["char_level"]["tp"] + indicators = result["global_"]["tokenisation_indicators"] ``` - `CorefResult` TypedDict is now exported from `pythainlp.coref`. diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index f610ae087..773ddd2bc 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -5,11 +5,11 @@ __all__: list[str] = [ "BleuScore", - "CharLevelStats", - "GlobalStats", + "CharLevelStat", + "GlobalStat", "RougeScore", - "TokenizationStats", - "WordLevelStats", + "TokenizationStat", + "WordLevelStat", "benchmark", "bleu_score", "character_error_rate", @@ -26,9 +26,9 @@ word_error_rate, ) from pythainlp.benchmarks.word_tokenization import ( - CharLevelStats, - GlobalStats, - TokenizationStats, - WordLevelStats, + CharLevelStat, + GlobalStat, + TokenizationStat, + WordLevelStat, benchmark, ) diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index be02953e5..c6d2df355 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -30,7 +30,7 @@ TAILING_SEP_RX: re.Pattern[str] = re.compile(f"{re.escape(SEPARATOR)}$") -class CharLevelStats(TypedDict): +class CharLevelStat(TypedDict): """Character-level confusion matrix statistics for tokenization.""" tp: int @@ -39,7 +39,7 @@ class CharLevelStats(TypedDict): fn: int -class WordLevelStats(TypedDict): +class WordLevelStat(TypedDict): """Word-level tokenization statistics.""" correctly_tokenised_words: int @@ -47,22 +47,18 @@ class WordLevelStats(TypedDict): total_words_in_ref_sample: int -class GlobalStats(TypedDict): - """Global tokenization indicators as a binary indicator string.""" +class GlobalStat(TypedDict): + """Global tokenization indicator as a binary indicator string.""" tokenisation_indicators: str -# Functional form is required because 'global' is a Python reserved keyword. -TokenizationStats = TypedDict( - "TokenizationStats", - { - "char_level": CharLevelStats, - "word_level": WordLevelStats, - "global": GlobalStats, - }, -) -"""Tokenization quality statistics at character, word, and global level.""" +class TokenizationStat(TypedDict): + """Tokenization quality statistics at character, word, and global level.""" + + char_level: CharLevelStat + word_level: WordLevelStat + global_: GlobalStat def _f1(precision: float, recall: float) -> float: @@ -81,7 +77,7 @@ def _f1(precision: float, recall: float) -> float: @overload def _flatten_result( - my_dict: TokenizationStats, sep: str = ... + my_dict: TokenizationStat, sep: str = ... ) -> dict[str, Union[int, str]]: ... @@ -105,7 +101,7 @@ def _flatten_result( :param my_dict: dictionary containing stats - :type my_dict: TokenizationStats or + :type my_dict: TokenizationStat or collections.abc.Mapping[str, collections.abc.Mapping[str, Union[int, str]]] :param str sep: separator between the two keys (default: ":") @@ -189,7 +185,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: def compute_stats( ref_sample: str, raw_sample: str -) -> TokenizationStats: +) -> TokenizationStat: """Compute statistics for tokenization quality These statistics include: @@ -206,7 +202,7 @@ def compute_stats( :param str samples: samples that we want to evaluate :return: metrics at character- and word-level and indicators of correctly tokenized words - :rtype: TokenizationStats + :rtype: TokenizationStat """ import numpy as np @@ -244,18 +240,18 @@ def compute_stats( tokenization_indicators_str = list(map(str, tokenization_indicators)) return { - "char_level": CharLevelStats( + "char_level": CharLevelStat( tp=c_tp, fp=c_fp, tn=c_tn, fn=c_fn, ), - "word_level": WordLevelStats( + "word_level": WordLevelStat( correctly_tokenised_words=correctly_tokenised_words, total_words_in_sample=int(np.sum(sample_arr)), total_words_in_ref_sample=int(np.sum(ref_sample_arr)), ), - "global": GlobalStats( + "global_": GlobalStat( tokenisation_indicators="".join(tokenization_indicators_str), ), } diff --git a/tests/extra/testx_benchmarks.py b/tests/extra/testx_benchmarks.py index 27c00ef41..aab53624d 100644 --- a/tests/extra/testx_benchmarks.py +++ b/tests/extra/testx_benchmarks.py @@ -9,11 +9,11 @@ from pythainlp.benchmarks import ( BleuScore, - CharLevelStats, - GlobalStats, + CharLevelStat, + GlobalStat, RougeScore, - TokenizationStats, - WordLevelStats, + TokenizationStat, + WordLevelStat, bleu_score, rouge_score, word_tokenization, @@ -59,31 +59,31 @@ def test_compute_stats(self): self.assertIsNotNone(result) def test_compute_stats_return_type(self): - """Test that compute_stats returns a TokenizationStats typed dict.""" + """Test that compute_stats returns a TokenizationStat typed dict.""" ref = word_tokenization.preprocessing("อากาศ|ร้อน|มาก") act = word_tokenization.preprocessing("อากาศ|ร้อนมาก") - result: TokenizationStats = word_tokenization.compute_stats(ref, act) + result: TokenizationStat = word_tokenization.compute_stats(ref, act) self.assertIsInstance(result, dict) self.assertIn("char_level", result) self.assertIn("word_level", result) - self.assertIn("global", result) + self.assertIn("global_", result) - char: CharLevelStats = result["char_level"] + char: CharLevelStat = result["char_level"] self.assertIsInstance(char, dict) self.assertIsInstance(char["tp"], int) self.assertIsInstance(char["fp"], int) self.assertIsInstance(char["tn"], int) self.assertIsInstance(char["fn"], int) - word: WordLevelStats = result["word_level"] + word: WordLevelStat = result["word_level"] self.assertIsInstance(word, dict) self.assertIsInstance(word["correctly_tokenised_words"], int) self.assertIsInstance(word["total_words_in_sample"], int) self.assertIsInstance(word["total_words_in_ref_sample"], int) - glob: GlobalStats = result["global"] + glob: GlobalStat = result["global_"] self.assertIsInstance(glob, dict) self.assertIsInstance(glob["tokenisation_indicators"], str) From 8b8e174e251bd8f2bbac75cad0f5a5f97c34592d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 13:10:58 +0000 Subject: [PATCH 4/5] =?UTF-8?q?Use=20US=20spelling:=20tokenised=E2=86=92to?= =?UTF-8?q?kenized,=20tokenisation=E2=86=92tokenization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - WordLevelStat field: correctly_tokenised_words → correctly_tokenized_words - GlobalStat field: tokenisation_indicators → tokenization_indicators - Private helper: _find_words_correctly_tokenised → _find_words_correctly_tokenized - All call sites in compute_stats(), cli/benchmark.py, and tests updated - CHANGELOG migration example updated Co-authored-by: bact <128572+bact@users.noreply.github.com> Agent-Logs-Url: https://github.com/PyThaiNLP/pythainlp/sessions/e8eff276-0f77-4d6f-9d4e-d9787e6f414b --- CHANGELOG.md | 2 +- pythainlp/benchmarks/word_tokenization.py | 14 +++++++------- pythainlp/cli/benchmark.py | 8 ++++---- tests/extra/testx_benchmarks.py | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46adde6f6..2081ee3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ and this project adheres to from pythainlp.benchmarks import TokenizationStat result: TokenizationStat = compute_stats(ref, hyp) tp = result["char_level"]["tp"] - indicators = result["global_"]["tokenisation_indicators"] + indicators = result["global_"]["tokenization_indicators"] ``` - `CorefResult` TypedDict is now exported from `pythainlp.coref`. diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index c6d2df355..e0475657c 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -42,7 +42,7 @@ class CharLevelStat(TypedDict): class WordLevelStat(TypedDict): """Word-level tokenization statistics.""" - correctly_tokenised_words: int + correctly_tokenized_words: int total_words_in_sample: int total_words_in_ref_sample: int @@ -50,7 +50,7 @@ class WordLevelStat(TypedDict): class GlobalStat(TypedDict): """Global tokenization indicator as a binary indicator string.""" - tokenisation_indicators: str + tokenization_indicators: str class TokenizationStat(TypedDict): @@ -231,11 +231,11 @@ def compute_stats( # Find correctly tokenized words in the sample ss_boundaries = _find_word_boundaries(sample_arr) - tokenization_indicators = _find_words_correctly_tokenised( + tokenization_indicators = _find_words_correctly_tokenized( word_boundaries, ss_boundaries ) - correctly_tokenised_words: int = int(np.sum(tokenization_indicators)) + correctly_tokenized_words: int = int(np.sum(tokenization_indicators)) tokenization_indicators_str = list(map(str, tokenization_indicators)) @@ -247,12 +247,12 @@ def compute_stats( fn=c_fn, ), "word_level": WordLevelStat( - correctly_tokenised_words=correctly_tokenised_words, + correctly_tokenized_words=correctly_tokenized_words, total_words_in_sample=int(np.sum(sample_arr)), total_words_in_ref_sample=int(np.sum(ref_sample_arr)), ), "global_": GlobalStat( - tokenisation_indicators="".join(tokenization_indicators_str), + tokenization_indicators="".join(tokenization_indicators_str), ), } @@ -317,7 +317,7 @@ def _find_word_boundaries( return list(zip(start_idx, end_idx)) -def _find_words_correctly_tokenised( +def _find_words_correctly_tokenized( ref_boundaries: list[tuple[int, int]], predicted_boundaries: list[tuple[int, int]], ) -> tuple[int, ...]: diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index 953bb8181..9eb5a57cd 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -108,7 +108,7 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: "char_level:fp", "char_level:tn", "char_level:fn", - "word_level:correctly_tokenised_words", + "word_level:correctly_tokenized_words", "word_level:total_words_in_sample", "word_level:total_words_in_ref_sample", ] @@ -127,12 +127,12 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: ) statistics["word_level:precision"] = ( - statistics["word_level:correctly_tokenised_words"] + statistics["word_level:correctly_tokenized_words"] / statistics["word_level:total_words_in_sample"] ) statistics["word_level:recall"] = ( - statistics["word_level:correctly_tokenised_words"] + statistics["word_level:correctly_tokenized_words"] / statistics["word_level:total_words_in_ref_sample"] ) @@ -146,7 +146,7 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: for c in [ "total_words_in_sample", "total_words_in_ref_sample", - "correctly_tokenised_words", + "correctly_tokenized_words", "precision", "recall", ]: diff --git a/tests/extra/testx_benchmarks.py b/tests/extra/testx_benchmarks.py index aab53624d..5a0857cbe 100644 --- a/tests/extra/testx_benchmarks.py +++ b/tests/extra/testx_benchmarks.py @@ -79,13 +79,13 @@ def test_compute_stats_return_type(self): word: WordLevelStat = result["word_level"] self.assertIsInstance(word, dict) - self.assertIsInstance(word["correctly_tokenised_words"], int) + self.assertIsInstance(word["correctly_tokenized_words"], int) self.assertIsInstance(word["total_words_in_sample"], int) self.assertIsInstance(word["total_words_in_ref_sample"], int) glob: GlobalStat = result["global_"] self.assertIsInstance(glob, dict) - self.assertIsInstance(glob["tokenisation_indicators"], str) + self.assertIsInstance(glob["tokenization_indicators"], str) def test_benchmark(self): expected = [] @@ -98,7 +98,7 @@ def test_benchmark(self): self.assertIsNotNone(df) - def test_count_correctly_tokenised_words(self): + def test_count_correctly_tokenized_words(self): for d in TEST_DATA["binary_sentences"]: sample = np.array(list(d["actual"])).astype(int) ref_sample = np.array(list(d["expected"])).astype(int) @@ -108,20 +108,20 @@ def test_count_correctly_tokenised_words(self): # in binary [{0, 1}, ...] correctly_tokenized_words = ( - word_tokenization._find_words_correctly_tokenised(rb, sb) + word_tokenization._find_words_correctly_tokenized(rb, sb) ) self.assertEqual( np.sum(correctly_tokenized_words), d["expected_count"] ) - def test_words_correctly_tokenised(self): + def test_words_correctly_tokenized(self): r = [(0, 2), (2, 10), (10, 12)] s = [(0, 10), (10, 12)] expected = "01" - labels = word_tokenization._find_words_correctly_tokenised(r, s) + labels = word_tokenization._find_words_correctly_tokenized(r, s) self.assertEqual(expected, "".join(np.array(labels).astype(str))) def test_flatten_result(self): From c80c2283f31247156e956a0c5b4b5d73c17391f7 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Wed, 25 Mar 2026 16:02:30 +0000 Subject: [PATCH 5/5] Revise CHANGELOG for new TypedDicts and breaking changes Updated CHANGELOG to reflect breaking changes and new TypedDicts in pythainlp.benchmarks and pythainlp.coref. Added migration notes for users to transition to the new TypedDict structures. --- CHANGELOG.md | 61 +++------------------------------------------------- 1 file changed, 3 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2081ee3ce..195e39db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,73 +21,18 @@ and this project adheres to ### Added -- `RougeScore` TypedDict in `pythainlp.benchmarks`: precision, recall, and - fmeasure fields replace the previous `tuple[float, float, float]` value - in the `rouge_score()` return dict. **Breaking change** – migrate as follows: - - ```python - # Before (tuple indexing, error-prone) - precision, recall, fmeasure = scores["rouge1"] - - # After (named fields) - precision = scores["rouge1"]["precision"] - recall = scores["rouge1"]["recall"] - fmeasure = scores["rouge1"]["fmeasure"] - ``` - -- `CharLevelStat`, `WordLevelStat`, `GlobalStat`, and `TokenizationStat` - TypedDicts in `pythainlp.benchmarks`: give named, type-safe access to the - dict returned by `word_tokenization.compute_stats()`. The global-level key - is `"global_"` (trailing underscore avoids the Python reserved word). - - ```python - # Before (opaque nested dict) - result = compute_stats(ref, hyp) - tp = result["char_level"]["tp"] - - # After (same access, now type-safe with TokenizationStat) - from pythainlp.benchmarks import TokenizationStat - result: TokenizationStat = compute_stats(ref, hyp) - tp = result["char_level"]["tp"] - indicators = result["global_"]["tokenization_indicators"] - ``` - -- `CorefResult` TypedDict is now exported from `pythainlp.coref`. - `coreference_resolution()` return type updated from `list[dict[str, Any]]` - to `list[CorefResult]`. - - ```python - # Before - from pythainlp.coref import coreference_resolution - - # After – import the TypedDict for type annotations - from pythainlp.coref import CorefResult, coreference_resolution - ``` - -- `EntitySpan` TypedDict (`pythainlp.tag.named_entity`, #1363) and - `BleuScore` TypedDict (`pythainlp.benchmarks`, #1365) were introduced in - previous releases. Migration notes for completeness: +- `EntitySpan` TypedDict (#1363). + Migration notes: ```python # EntitySpan – Before (plain dict) entity = {"text": ["สมชาย"], "span": [0, 1], "entity_type": "PERSON"} - # EntitySpan – After (TypedDict constructor) + # EntitySpan – After (TypedDict) from pythainlp.tag.named_entity import EntitySpan entity = EntitySpan(text=["สมชาย"], span=[0, 1], entity_type="PERSON") ``` - ```python - # BleuScore – Before (plain dict, no type hint) - score = bleu_score(refs, hyps) - print(score["bleu"]) - - # BleuScore – After (TypedDict annotation) - from pythainlp.benchmarks import BleuScore, bleu_score - score: BleuScore = bleu_score(refs, hyps) - print(score["bleu"]) - ``` - ### Fixed - thai2rom_onnx: fix ONNX encoder model and fix inference bugs (#1349)