From 2f4143f65fafbfa4a7901e24cd5e708b49dbdecc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:01:44 +0000 Subject: [PATCH 01/24] Initial plan From 712faf89f71a60661d6bb5b3dbdc107c6322c27c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:07:42 +0000 Subject: [PATCH 02/24] Mock large corpus file downloads in tests for speed and stability Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/core/test_corpus.py | 110 +++++++++++++++++++++++++++++++++++--- 1 file changed, 102 insertions(+), 8 deletions(-) diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index e1acc2913..5974f36ba 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -2,8 +2,10 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import io import os import unittest +from unittest.mock import mock_open, patch from pythainlp.corpus import ( countries, @@ -111,18 +113,110 @@ def test_corpus(self): self.assertFalse(download(name="test", version="0.0.1")) def test_oscar(self): - self.assertIsNotNone(oscar.word_freqs()) - self.assertIsNotNone(oscar.unigram_word_freqs()) + # Mock the oscar corpus file to avoid slow download and parsing + # Format: word,count + # Note: A line starting with space becomes empty string after strip() + mock_oscar_data = """word,count +คน,1000 +ไทย,500 +ภาษา,300 + ,100 +"test",50 +""" + mock_path = "/mock/path/oscar_icu" + + with patch('pythainlp.corpus.oscar.get_corpus_path', return_value=mock_path): + with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + result = oscar.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Verify parsing logic + self.assertEqual(result[0], ("คน", 1000)) + # Space line becomes empty string (not ) due to strip() + self.assertIn(("", 100), result) + # Verify quoted values are filtered out + for word, _ in result: + self.assertNotIn('"', word) + + # Reset mock for unigram test + with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + result_unigram = oscar.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) def test_tnc(self): - self.assertIsNotNone(tnc.word_freqs()) - self.assertIsNotNone(tnc.unigram_word_freqs()) - self.assertIsNotNone(tnc.bigram_word_freqs()) - self.assertIsNotNone(tnc.trigram_word_freqs()) + # Mock TNC unigram corpus + mock_unigram_data = """คน 1000 +ไทย 500 +ภาษา 300""" + + # Mock TNC bigram corpus + mock_bigram_data = """คน ไทย 100 +ไทย ภาษา 50 +ภาษา ไทย 30""" + + # Mock TNC trigram corpus + mock_trigram_data = """คน ไทย ภาษา 10 +ไทย ภาษา ไทย 5 +ภาษา ไทย คน 3""" + + # Test unigram functions + with patch('pythainlp.corpus.tnc.get_corpus', return_value=frozenset(mock_unigram_data.split('\n'))): + result = tnc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Check that at least one expected entry exists (order not guaranteed) + self.assertIn(("คน", 1000), result) + + result_unigram = tnc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) + + # Test bigram function + mock_bigram_path = "/mock/path/bigram" + with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_bigram_path): + with patch('builtins.open', mock_open(read_data=mock_bigram_data)): + result_bigram = tnc.bigram_word_freqs() + self.assertIsNotNone(result_bigram) + self.assertIsInstance(result_bigram, dict) + self.assertGreater(len(result_bigram), 0) + self.assertEqual(result_bigram[("คน", "ไทย")], 100) + + # Test trigram function + mock_trigram_path = "/mock/path/trigram" + with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_trigram_path): + with patch('builtins.open', mock_open(read_data=mock_trigram_data)): + result_trigram = tnc.trigram_word_freqs() + self.assertIsNotNone(result_trigram) + self.assertIsInstance(result_trigram, dict) + self.assertGreater(len(result_trigram), 0) + self.assertEqual(result_trigram[("คน", "ไทย", "ภาษา")], 10) def test_ttc(self): - self.assertIsNotNone(ttc.word_freqs()) - self.assertIsNotNone(ttc.unigram_word_freqs()) + # Mock TTC corpus + mock_ttc_data = """คน 1000 +ไทย 500 +ภาษา 300""" + + with patch('pythainlp.corpus.ttc.get_corpus', return_value=frozenset(mock_ttc_data.split('\n'))): + result = ttc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Check that at least one expected entry exists (order not guaranteed) + self.assertIn(("คน", 1000), result) + + result_unigram = ttc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) def test_revise_wordset(self): training_data = [ From 6f002435763c111f7a74443358982e100bcf096a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:10:13 +0000 Subject: [PATCH 03/24] Fix linting issues in test_corpus.py Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/core/test_corpus.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index 5974f36ba..6c3a82b10 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -2,7 +2,6 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -import io import os import unittest from unittest.mock import mock_open, patch @@ -124,7 +123,7 @@ def test_oscar(self): "test",50 """ mock_path = "/mock/path/oscar_icu" - + with patch('pythainlp.corpus.oscar.get_corpus_path', return_value=mock_path): with patch('builtins.open', mock_open(read_data=mock_oscar_data)): result = oscar.word_freqs() @@ -138,7 +137,7 @@ def test_oscar(self): # Verify quoted values are filtered out for word, _ in result: self.assertNotIn('"', word) - + # Reset mock for unigram test with patch('builtins.open', mock_open(read_data=mock_oscar_data)): result_unigram = oscar.unigram_word_freqs() @@ -152,17 +151,17 @@ def test_tnc(self): mock_unigram_data = """คน 1000 ไทย 500 ภาษา 300""" - + # Mock TNC bigram corpus mock_bigram_data = """คน ไทย 100 ไทย ภาษา 50 ภาษา ไทย 30""" - + # Mock TNC trigram corpus mock_trigram_data = """คน ไทย ภาษา 10 ไทย ภาษา ไทย 5 ภาษา ไทย คน 3""" - + # Test unigram functions with patch('pythainlp.corpus.tnc.get_corpus', return_value=frozenset(mock_unigram_data.split('\n'))): result = tnc.word_freqs() @@ -171,13 +170,13 @@ def test_tnc(self): self.assertGreater(len(result), 0) # Check that at least one expected entry exists (order not guaranteed) self.assertIn(("คน", 1000), result) - + result_unigram = tnc.unigram_word_freqs() self.assertIsNotNone(result_unigram) self.assertIsInstance(result_unigram, dict) self.assertGreater(len(result_unigram), 0) self.assertEqual(result_unigram["คน"], 1000) - + # Test bigram function mock_bigram_path = "/mock/path/bigram" with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_bigram_path): @@ -187,7 +186,7 @@ def test_tnc(self): self.assertIsInstance(result_bigram, dict) self.assertGreater(len(result_bigram), 0) self.assertEqual(result_bigram[("คน", "ไทย")], 100) - + # Test trigram function mock_trigram_path = "/mock/path/trigram" with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_trigram_path): @@ -203,7 +202,7 @@ def test_ttc(self): mock_ttc_data = """คน 1000 ไทย 500 ภาษา 300""" - + with patch('pythainlp.corpus.ttc.get_corpus', return_value=frozenset(mock_ttc_data.split('\n'))): result = ttc.word_freqs() self.assertIsNotNone(result) @@ -211,7 +210,7 @@ def test_ttc(self): self.assertGreater(len(result), 0) # Check that at least one expected entry exists (order not guaranteed) self.assertIn(("คน", 1000), result) - + result_unigram = ttc.unigram_word_freqs() self.assertIsNotNone(result_unigram) self.assertIsInstance(result_unigram, dict) From fa9f3a78226842ddfd2edd625d19ae8301757821 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:16:50 +0000 Subject: [PATCH 04/24] Add corpus integrity testing workflow and tests Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .github/workflows/corpus-integrity.yml | 57 ++++++ tests/corpus_integrity/__init__.py | 10 ++ tests/corpus_integrity/test_builtin_corpus.py | 164 ++++++++++++++++++ .../test_downloadable_corpus.py | 99 +++++++++++ 4 files changed, 330 insertions(+) create mode 100644 .github/workflows/corpus-integrity.yml create mode 100644 tests/corpus_integrity/__init__.py create mode 100644 tests/corpus_integrity/test_builtin_corpus.py create mode 100644 tests/corpus_integrity/test_downloadable_corpus.py diff --git a/.github/workflows/corpus-integrity.yml b/.github/workflows/corpus-integrity.yml new file mode 100644 index 000000000..e498f78a9 --- /dev/null +++ b/.github/workflows/corpus-integrity.yml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-License-Identifier: Apache-2.0 + +name: Corpus Integrity Test + +on: + push: + paths: + - ".github/workflows/corpus-integrity.yml" + - "pythainlp/corpus/**" + - "tests/corpus_integrity/**" + pull_request: + branches: + - dev + paths: + - ".github/workflows/corpus-integrity.yml" + - "pythainlp/corpus/**" + - "tests/corpus_integrity/**" + +# Avoid duplicate runs for the same source branch and repository +concurrency: + group: >- + ${{ github.workflow }}-${{ + github.event.pull_request.head.repo.full_name || github.repository + }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + corpus-integrity: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: "pip" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install . + + - name: Test built-in corpus files + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus_integrity -p "test_builtin_*.py" -v + + - name: Test downloadable corpus files + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus_integrity -p "test_downloadable_*.py" -v diff --git a/tests/corpus_integrity/__init__.py b/tests/corpus_integrity/__init__.py new file mode 100644 index 000000000..94bfd2ac3 --- /dev/null +++ b/tests/corpus_integrity/__init__.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Corpus integrity tests. + +These tests verify the integrity, format, and parseability of corpus files. +They are separate from regular unit tests to avoid slowing down development +test cycles with large file downloads and parsing. +""" diff --git a/tests/corpus_integrity/test_builtin_corpus.py b/tests/corpus_integrity/test_builtin_corpus.py new file mode 100644 index 000000000..1a11b9e39 --- /dev/null +++ b/tests/corpus_integrity/test_builtin_corpus.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test integrity and parseability of built-in corpus files. + +These tests verify that all corpus files included in the package +can be loaded and parsed correctly. +""" + +import unittest + +from pythainlp.corpus import ( + countries, + find_synonyms, + get_corpus, + provinces, + thai_family_names, + thai_female_names, + thai_icu_words, + thai_male_names, + thai_negations, + thai_orst_words, + thai_stopwords, + thai_syllables, + thai_synonyms, + thai_volubilis_words, + thai_wikipedia_titles, + thai_words, + ttc, +) + + +class BuiltinCorpusIntegrityTestCase(unittest.TestCase): + """Test integrity of built-in corpus files.""" + + def test_negations(self): + """Test thai_negations corpus can be loaded and is not empty.""" + result = thai_negations() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_stopwords(self): + """Test thai_stopwords corpus can be loaded and is not empty.""" + result = thai_stopwords() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_syllables(self): + """Test thai_syllables corpus can be loaded and is not empty.""" + result = thai_syllables() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_words(self): + """Test thai_words corpus can be loaded and is not empty.""" + result = thai_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_synonyms(self): + """Test thai_synonyms corpus can be loaded and parsed correctly.""" + result = thai_synonyms() + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + # Test that find_synonyms works with the loaded data + synonyms = find_synonyms("หมู") + self.assertIsInstance(synonyms, list) + self.assertGreater(len(synonyms), 0) + + def test_icu_words(self): + """Test thai_icu_words corpus can be loaded and is not empty.""" + result = thai_icu_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_orst_words(self): + """Test thai_orst_words corpus can be loaded and is not empty.""" + result = thai_orst_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_volubilis_words(self): + """Test thai_volubilis_words corpus can be loaded and is not empty.""" + result = thai_volubilis_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_wikipedia_titles(self): + """Test thai_wikipedia_titles corpus can be loaded and is not empty.""" + result = thai_wikipedia_titles() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_countries(self): + """Test countries corpus can be loaded and is not empty.""" + result = countries() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_provinces(self): + """Test provinces corpus can be loaded and parsed correctly.""" + result = provinces() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + # Test with details + result_details = provinces(details=True) + self.assertIsInstance(result_details, list) + self.assertEqual(len(result), len(result_details)) + + def test_family_names(self): + """Test thai_family_names corpus can be loaded and is not empty.""" + result = thai_family_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_female_names(self): + """Test thai_female_names corpus can be loaded and is not empty.""" + result = thai_female_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_male_names(self): + """Test thai_male_names corpus can be loaded and is not empty.""" + result = thai_male_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_ttc_freq(self): + """Test TTC frequency corpus can be loaded and parsed correctly.""" + result = ttc.word_freqs() + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Verify format: list of (word, frequency) tuples + for item in result[:10]: # Check first 10 items + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + + # Test unigram version + result_unigram = ttc.unigram_word_freqs() + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + def test_tnc_freq(self): + """Test TNC frequency corpus can be loaded and parsed correctly.""" + # Test unigram from built-in file + result = get_corpus("tnc_freq.txt") + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify format: tab-separated word and frequency + for line in list(result)[:10]: # Check first 10 lines + parts = line.split('\t') + self.assertGreaterEqual(len(parts), 2) diff --git a/tests/corpus_integrity/test_downloadable_corpus.py b/tests/corpus_integrity/test_downloadable_corpus.py new file mode 100644 index 000000000..62cb74d12 --- /dev/null +++ b/tests/corpus_integrity/test_downloadable_corpus.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test integrity and parseability of downloadable corpus files. + +These tests verify that corpus files that need to be downloaded +can be fetched and parsed correctly. These tests may take longer +to run due to network downloads. +""" + +import unittest + +from pythainlp.corpus import oscar, tnc + + +class DownloadableCorpusIntegrityTestCase(unittest.TestCase): + """Test integrity of downloadable corpus files.""" + + def test_oscar_corpus(self): + """Test OSCAR corpus can be downloaded and parsed correctly.""" + # Test word_freqs + result = oscar.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + # Verify format: list of (word, frequency) tuples + for item in result[:10]: # Check first 10 items + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + self.assertGreater(item[1], 0) + + # Test unigram_word_freqs + result_unigram = oscar.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + # Verify dict values are integers + for word, freq in list(result_unigram.items())[:10]: + self.assertIsInstance(word, str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) + + def test_tnc_unigram(self): + """Test TNC unigram corpus can be loaded and parsed correctly.""" + result = tnc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + # Verify format + for item in result[:10]: + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + + # Test unigram version + result_unigram = tnc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + def test_tnc_bigram(self): + """Test TNC bigram corpus can be downloaded and parsed correctly.""" + result = tnc.bigram_word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + + # Verify format: dict with tuple keys (word1, word2) -> frequency + for key, freq in list(result.items())[:10]: + self.assertIsInstance(key, tuple) + self.assertEqual(len(key), 2) + self.assertIsInstance(key[0], str) + self.assertIsInstance(key[1], str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) + + def test_tnc_trigram(self): + """Test TNC trigram corpus can be downloaded and parsed correctly.""" + result = tnc.trigram_word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + + # Verify format: dict with tuple keys (word1, word2, word3) -> frequency + for key, freq in list(result.items())[:10]: + self.assertIsInstance(key, tuple) + self.assertEqual(len(key), 3) + self.assertIsInstance(key[0], str) + self.assertIsInstance(key[1], str) + self.assertIsInstance(key[2], str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) From fc1aaec48569aac9fce9bc2e9ea719cbab4195df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:17:24 +0000 Subject: [PATCH 05/24] Add README for corpus integrity tests Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/corpus_integrity/README.md | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/corpus_integrity/README.md diff --git a/tests/corpus_integrity/README.md b/tests/corpus_integrity/README.md new file mode 100644 index 000000000..aaa0c500a --- /dev/null +++ b/tests/corpus_integrity/README.md @@ -0,0 +1,78 @@ +# Corpus Integrity Tests + +This directory contains tests that verify the integrity, format, and parseability of corpus files in PyThaiNLP. + +## Purpose + +These tests are separate from regular unit tests because: +1. They test actual file loading and parsing (not mocked) +2. Downloadable corpus tests require network access and can be slow +3. They verify corpus data format and structure +4. They should only run when corpus files or corpus code changes + +## Test Categories + +### Built-in Corpus Tests (`test_builtin_corpus.py`) + +Tests corpus files that are included in the package: +- Text word lists (negations, stopwords, syllables, words, etc.) +- CSV files (provinces) +- Frequency data (TNC, TTC) +- Name lists (family names, person names) + +**Run time:** < 1 second + +### Downloadable Corpus Tests (`test_downloadable_corpus.py`) + +Tests corpus files that need to be downloaded: +- OSCAR word frequencies (96MB) +- TNC bigram/trigram frequencies (41MB + 145MB) + +**Run time:** ~17 seconds (includes download time) + +## Running Tests + +### Run all corpus integrity tests: +```bash +python -m unittest discover -s tests/corpus_integrity -v +``` + +### Run only built-in corpus tests: +```bash +python -m unittest tests.corpus_integrity.test_builtin_corpus -v +``` + +### Run only downloadable corpus tests: +```bash +python -m unittest tests.corpus_integrity.test_downloadable_corpus -v +``` + +## CI Integration + +The corpus integrity tests run automatically via GitHub Actions workflow (`.github/workflows/corpus-integrity.yml`) when: +- Changes are made to `pythainlp/corpus/**` +- Changes are made to `tests/corpus_integrity/**` +- The workflow file itself is modified + +## What is Tested + +Each test verifies: +1. **Loadability**: File can be loaded without errors +2. **Type correctness**: Returns expected data type (frozenset, list, dict) +3. **Non-empty**: Contains actual data +4. **Format validity**: Data structure matches expected format +5. **Content validity**: Contains expected content (e.g., Thai characters) + +## Adding New Tests + +When adding a new corpus file or function to `pythainlp.corpus`: +1. Add a test to `test_builtin_corpus.py` if it's included in the package +2. Add a test to `test_downloadable_corpus.py` if it requires download +3. Verify the test catches format errors by temporarily breaking the corpus + +## Relationship to Unit Tests + +- **Unit tests** (`tests/core/test_corpus.py`): Use mocks for speed, test code logic +- **Corpus integrity tests** (this directory): Use real data, test file integrity + +Both test suites are important and complementary. From 0e28b159804ae3dd1a6cbe5d7882571d3339c0fd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:20:35 +0000 Subject: [PATCH 06/24] Add explicit permissions to corpus-integrity workflow for security Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .github/workflows/corpus-integrity.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/corpus-integrity.yml b/.github/workflows/corpus-integrity.yml index e498f78a9..b74577a1d 100644 --- a/.github/workflows/corpus-integrity.yml +++ b/.github/workflows/corpus-integrity.yml @@ -28,6 +28,8 @@ concurrency: jobs: corpus-integrity: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout From b6f2bc10cc5791c23edd0a5ea908e74e8448d912 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:27:16 +0000 Subject: [PATCH 07/24] Fix ruff issues and update tests/README.md with corpus_integrity reference Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/README.md | 30 +++++++++++++++++++ tests/corpus_integrity/test_builtin_corpus.py | 4 +-- .../test_downloadable_corpus.py | 14 ++++----- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/tests/README.md b/tests/README.md index af26a0fff..4541f3419 100644 --- a/tests/README.md +++ b/tests/README.md @@ -169,3 +169,33 @@ for real-world usage: - Thai-specific edge cases with combining characters and mixed scripts - Multi-engine robustness testing across all core tokenization engines - Very long strings that can cause performance issues (issue #893) + +## Corpus integrity tests (corpus_integrity/) + +A separate test suite that verifies the integrity, format, and parseability +of corpus files in PyThaiNLP. These tests are separate from regular unit tests +because they test actual file loading and parsing (not mocked) and downloadable +corpus tests require network access. + +For detailed information about corpus integrity tests, see: +[tests/corpus_integrity/README.md](corpus_integrity/README.md) + +The corpus integrity tests are triggered automatically via GitHub Actions +when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. + +**Key characteristics:** +- Tests actual file loading and parsing (not mocked) +- Verifies corpus data format and structure +- Only runs when corpus files or corpus code changes +- Includes tests for both built-in and downloadable corpus files + +**Run corpus integrity tests:** +```shell +unittest tests.corpus_integrity +``` + +Or run specific test files: +```shell +python -m unittest tests.corpus_integrity.test_builtin_corpus +python -m unittest tests.corpus_integrity.test_downloadable_corpus +``` diff --git a/tests/corpus_integrity/test_builtin_corpus.py b/tests/corpus_integrity/test_builtin_corpus.py index 1a11b9e39..d6a310d04 100644 --- a/tests/corpus_integrity/test_builtin_corpus.py +++ b/tests/corpus_integrity/test_builtin_corpus.py @@ -111,7 +111,7 @@ def test_provinces(self): result = provinces() self.assertIsInstance(result, frozenset) self.assertGreater(len(result), 0) - + # Test with details result_details = provinces(details=True) self.assertIsInstance(result_details, list) @@ -146,7 +146,7 @@ def test_ttc_freq(self): self.assertEqual(len(item), 2) self.assertIsInstance(item[0], str) self.assertIsInstance(item[1], int) - + # Test unigram version result_unigram = ttc.unigram_word_freqs() self.assertIsInstance(result_unigram, dict) diff --git a/tests/corpus_integrity/test_downloadable_corpus.py b/tests/corpus_integrity/test_downloadable_corpus.py index 62cb74d12..421656a76 100644 --- a/tests/corpus_integrity/test_downloadable_corpus.py +++ b/tests/corpus_integrity/test_downloadable_corpus.py @@ -24,7 +24,7 @@ def test_oscar_corpus(self): self.assertIsNotNone(result) self.assertIsInstance(result, list) self.assertGreater(len(result), 0) - + # Verify format: list of (word, frequency) tuples for item in result[:10]: # Check first 10 items self.assertIsInstance(item, tuple) @@ -32,13 +32,13 @@ def test_oscar_corpus(self): self.assertIsInstance(item[0], str) self.assertIsInstance(item[1], int) self.assertGreater(item[1], 0) - + # Test unigram_word_freqs result_unigram = oscar.unigram_word_freqs() self.assertIsNotNone(result_unigram) self.assertIsInstance(result_unigram, dict) self.assertGreater(len(result_unigram), 0) - + # Verify dict values are integers for word, freq in list(result_unigram.items())[:10]: self.assertIsInstance(word, str) @@ -51,14 +51,14 @@ def test_tnc_unigram(self): self.assertIsNotNone(result) self.assertIsInstance(result, list) self.assertGreater(len(result), 0) - + # Verify format for item in result[:10]: self.assertIsInstance(item, tuple) self.assertEqual(len(item), 2) self.assertIsInstance(item[0], str) self.assertIsInstance(item[1], int) - + # Test unigram version result_unigram = tnc.unigram_word_freqs() self.assertIsNotNone(result_unigram) @@ -71,7 +71,7 @@ def test_tnc_bigram(self): self.assertIsNotNone(result) self.assertIsInstance(result, dict) self.assertGreater(len(result), 0) - + # Verify format: dict with tuple keys (word1, word2) -> frequency for key, freq in list(result.items())[:10]: self.assertIsInstance(key, tuple) @@ -87,7 +87,7 @@ def test_tnc_trigram(self): self.assertIsNotNone(result) self.assertIsInstance(result, dict) self.assertGreater(len(result), 0) - + # Verify format: dict with tuple keys (word1, word2, word3) -> frequency for key, freq in list(result.items())[:10]: self.assertIsInstance(key, tuple) From 1addde9d58632ff4659d06854c22a157be68d892 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:27:57 +0000 Subject: [PATCH 08/24] Fix unittest command syntax in tests/README.md Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 4541f3419..e6308eff1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -191,7 +191,7 @@ when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. **Run corpus integrity tests:** ```shell -unittest tests.corpus_integrity +python -m unittest tests.corpus_integrity ``` Or run specific test files: From 8457587aac614318145a61b0b7a4941b0cb11adf Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 11:35:39 +0000 Subject: [PATCH 09/24] Update README.md --- tests/README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/README.md b/tests/README.md index e6308eff1..dba9c79cf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -155,7 +155,6 @@ By separating tests by dependency group, we can: - Requires: Internet connection, may involve large downloads - Test case class suffix: `TestCaseN` - ## Robustness tests (test_robustness.py) A comprehensive test suite within core tests that tests edge cases important @@ -183,19 +182,7 @@ For detailed information about corpus integrity tests, see: The corpus integrity tests are triggered automatically via GitHub Actions when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. -**Key characteristics:** -- Tests actual file loading and parsing (not mocked) -- Verifies corpus data format and structure -- Only runs when corpus files or corpus code changes -- Includes tests for both built-in and downloadable corpus files - **Run corpus integrity tests:** ```shell python -m unittest tests.corpus_integrity ``` - -Or run specific test files: -```shell -python -m unittest tests.corpus_integrity.test_builtin_corpus -python -m unittest tests.corpus_integrity.test_downloadable_corpus -``` From bb11971a84ffdb87dc9b561c53e5096926197183 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:43:43 +0000 Subject: [PATCH 10/24] Suppress CLI test output and mock network calls to avoid corpus downloads Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/core/test_cli.py | 262 ++++++++++++++++++++++----------------- tests/extra/testx_cli.py | 68 +++++----- 2 files changed, 186 insertions(+), 144 deletions(-) diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index 2c0d38854..56d3fe3bb 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -2,8 +2,11 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import io import unittest from argparse import ArgumentError +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import MagicMock, patch from pythainlp import __main__, cli from pythainlp.cli.data import App as DataApp @@ -19,147 +22,180 @@ def test_cli(self): cli.exit_if_empty("", None) def test_cli_main(self): - # call with no argument, should exit with 2 - with self.assertRaises(SystemExit) as ex: - __main__.main() - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + # call with no argument, should exit with 2 + with self.assertRaises(SystemExit) as ex: + __main__.main() + self.assertEqual(ex.exception.code, 2) - with self.assertRaises((ArgumentError, SystemExit)): - __main__.main(["thainlp"]) + with self.assertRaises((ArgumentError, SystemExit)): + __main__.main(["thainlp"]) - with self.assertRaises((ArgumentError, SystemExit)): - __main__.main(["thainlp", "NOT_EXIST", "command"]) + with self.assertRaises((ArgumentError, SystemExit)): + __main__.main(["thainlp", "NOT_EXIST", "command"]) - self.assertIsNone(__main__.main(["thainlp", "data", "path"])) + self.assertIsNone(__main__.main(["thainlp", "data", "path"])) def test_cli_data(self): self.assertTrue(hasattr(cli, "data")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "data"]) - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "data"]) + self.assertEqual(ex.exception.code, 2) + + # Mock network calls to avoid corpus downloads + mock_response = MagicMock() + mock_response.json.return_value = { + "test": {"latest_version": "0.1", "versions": {}}, + } + + with patch("pythainlp.corpus.get_corpus_db", return_value=mock_response): + self.assertIsNotNone(DataApp(["thainlp", "data", "catalog"])) + + self.assertIsNotNone(DataApp(["thainlp", "data", "path"])) + + # Mock download to avoid network + with patch("pythainlp.corpus.download", return_value=True): + self.assertIsNotNone(DataApp(["thainlp", "data", "get", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "catalog"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "path"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "get", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "info", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "get", "NOT_EXIST"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "info", "NOT_EXIST"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "NOT_EXIST"])) + self.assertIsNotNone(DataApp(["thainlp", "data", "info", "test"])) + + # Mock remove to avoid side effects + with patch("pythainlp.corpus.remove", return_value=True): + self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "test"])) + + # Test with non-existing corpus + with patch("pythainlp.corpus.download", return_value=False): + self.assertIsNotNone(DataApp(["thainlp", "data", "get", "NOT_EXIST"])) + + self.assertIsNotNone(DataApp(["thainlp", "data", "info", "NOT_EXIST"])) + + with patch("pythainlp.corpus.remove", return_value=False): + self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "NOT_EXIST"])) def test_cli_misspell(self): self.assertTrue(hasattr(cli, "misspell")) - with self.assertRaises(SystemExit) as ex: - MisspellApp(["thainlp", "misspell"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - MisspellApp( - [ - "thainlp", - "misspell", - "--file", - "./tests/data/text.txt", - "--seed", - "1", - "--misspell-ratio", - "0.05", - "--output", - "./tests/data/misspell_output.tmp", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + MisspellApp(["thainlp", "misspell"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + MisspellApp( + [ + "thainlp", + "misspell", + "--file", + "./tests/data/text.txt", + "--seed", + "1", + "--misspell-ratio", + "0.05", + "--output", + "./tests/data/misspell_output.tmp", + ] + ) ) - ) def test_cli_soundex(self): self.assertTrue(hasattr(cli, "soundex")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "soundex"]) - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "soundex"]) + self.assertEqual(ex.exception.code, 2) - self.assertIsNotNone(SoundexApp(["thainlp", "soundex", "ทดสอบ"])) + self.assertIsNotNone(SoundexApp(["thainlp", "soundex", "ทดสอบ"])) def test_cli_tag(self): self.assertTrue(hasattr(cli, "tag")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "tag"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - TagApp( - [ - "thainlp", - "tag", - "pos", - "-s", - " ", - "มอเตอร์ไซค์ ความว่างเปล่า", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "tag"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + TagApp( + [ + "thainlp", + "tag", + "pos", + "-s", + " ", + "มอเตอร์ไซค์ ความว่างเปล่า", + ] + ) ) - ) - self.assertIsNotNone( - TagApp( - [ - "thainlp", - "tag", - "role", - "-s", - " ", - "มอเตอร์ไซค์ ความว่างเปล่า", - ] + self.assertIsNotNone( + TagApp( + [ + "thainlp", + "tag", + "role", + "-s", + " ", + "มอเตอร์ไซค์ ความว่างเปล่า", + ] + ) ) - ) def test_cli_tokenize(self): self.assertTrue(hasattr(cli, "tokenize")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "tokenize"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - TokenizeApp(["thainlp", "tokenize", "NOT_EXIST", "ไม่มีอยู่ จริง"]) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "subword", - "-s", - "|", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "tokenize"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + TokenizeApp(["thainlp", "tokenize", "NOT_EXIST", "ไม่มีอยู่ จริง"]) + ) + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "subword", + "-s", + "|", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "syllable", - "-s", - "|", - "-w", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "syllable", + "-s", + "|", + "-w", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "word", - "-nw", - "-a", - "newmm", - "-s", - "|", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "word", + "-nw", + "-a", + "newmm", + "-s", + "|", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) diff --git a/tests/extra/testx_cli.py b/tests/extra/testx_cli.py index 7ec46d6bc..e53c55762 100644 --- a/tests/extra/testx_cli.py +++ b/tests/extra/testx_cli.py @@ -2,7 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import io import unittest +from contextlib import redirect_stderr, redirect_stdout from pythainlp import cli from pythainlp.cli.benchmark import App as BenchmarkApp @@ -14,39 +16,43 @@ class CliTestCaseX(unittest.TestCase): def test_cli_benchmark(self): self.assertTrue(hasattr(cli, "benchmark")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "benchmark"]) - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "benchmark"]) + self.assertEqual(ex.exception.code, 2) - self.assertIsNotNone( - BenchmarkApp( - [ - "thainlp", - "benchmark", - "word-tokenization", - "--input-file", - "./tests/data/input.txt", - "--test-file", - "./tests/data/test.txt", - "--save-details", - ] + self.assertIsNotNone( + BenchmarkApp( + [ + "thainlp", + "benchmark", + "word-tokenization", + "--input-file", + "./tests/data/input.txt", + "--test-file", + "./tests/data/test.txt", + "--save-details", + ] + ) ) - ) def test_cli_tokenize(self): - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "sent", - "-s", - "|", - ( - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้" - "กระสุนสำหรับสมองของคุณวันนี้" - "แต่คุณก็จะลืมมันไปทั้งหมดอีกครั้ง" - ), - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "sent", + "-s", + "|", + ( + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้" + "กระสุนสำหรับสมองของคุณวันนี้" + "แต่คุณก็จะลืมมันไปทั้งหมดอีกครั้ง" + ), + ] + ) ) - ) From 01879e34ba8e38ab5df2bd5cc3415ff4fa2fb947 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:55:45 +0000 Subject: [PATCH 11/24] Add corpus catalog tests and rename workflow to "Corpus Data Test" Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .../{corpus-integrity.yml => corpus-data.yml} | 14 +- tests/README.md | 30 ++-- tests/corpus_integrity/README.md | 33 ++++- tests/corpus_integrity/test_catalog.py | 128 ++++++++++++++++++ 4 files changed, 186 insertions(+), 19 deletions(-) rename .github/workflows/{corpus-integrity.yml => corpus-data.yml} (81%) create mode 100644 tests/corpus_integrity/test_catalog.py diff --git a/.github/workflows/corpus-integrity.yml b/.github/workflows/corpus-data.yml similarity index 81% rename from .github/workflows/corpus-integrity.yml rename to .github/workflows/corpus-data.yml index b74577a1d..cbfffa579 100644 --- a/.github/workflows/corpus-integrity.yml +++ b/.github/workflows/corpus-data.yml @@ -1,19 +1,19 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-License-Identifier: Apache-2.0 -name: Corpus Integrity Test +name: Corpus Data Test on: push: paths: - - ".github/workflows/corpus-integrity.yml" + - ".github/workflows/corpus-data.yml" - "pythainlp/corpus/**" - "tests/corpus_integrity/**" pull_request: branches: - dev paths: - - ".github/workflows/corpus-integrity.yml" + - ".github/workflows/corpus-data.yml" - "pythainlp/corpus/**" - "tests/corpus_integrity/**" @@ -26,7 +26,7 @@ concurrency: cancel-in-progress: true jobs: - corpus-integrity: + corpus-data: runs-on: ubuntu-latest permissions: contents: read @@ -46,6 +46,12 @@ jobs: pip install --upgrade pip pip install . + - name: Test corpus catalog + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus_integrity -p "test_catalog*.py" -v + - name: Test built-in corpus files env: PYTHONIOENCODING: utf-8 diff --git a/tests/README.md b/tests/README.md index dba9c79cf..6560c376d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -169,20 +169,34 @@ for real-world usage: - Multi-engine robustness testing across all core tokenization engines - Very long strings that can cause performance issues (issue #893) -## Corpus integrity tests (corpus_integrity/) +## Corpus data tests (corpus_integrity/) -A separate test suite that verifies the integrity, format, and parseability -of corpus files in PyThaiNLP. These tests are separate from regular unit tests -because they test actual file loading and parsing (not mocked) and downloadable -corpus tests require network access. +A separate test suite that verifies the integrity, format, parseability, and catalog +functionality of corpus data in PyThaiNLP. These tests are separate from regular unit tests +because they test actual file loading and parsing (not mocked), downloadable corpus tests +require network access, and they verify corpus catalog operations. -For detailed information about corpus integrity tests, see: +For detailed information about corpus data tests, see: [tests/corpus_integrity/README.md](corpus_integrity/README.md) -The corpus integrity tests are triggered automatically via GitHub Actions +The corpus data tests are triggered automatically via GitHub Actions when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. -**Run corpus integrity tests:** +**Key characteristics:** +- Tests actual file loading and parsing (not mocked) +- Verifies corpus data format and structure +- Tests corpus catalog download and query +- Only runs when corpus files or corpus code changes +- Includes tests for both built-in and downloadable corpus files + +**Run corpus data tests:** ```shell python -m unittest tests.corpus_integrity ``` + +Or run specific test files: +```shell +python -m unittest tests.corpus_integrity.test_catalog +python -m unittest tests.corpus_integrity.test_builtin_corpus +python -m unittest tests.corpus_integrity.test_downloadable_corpus +``` diff --git a/tests/corpus_integrity/README.md b/tests/corpus_integrity/README.md index aaa0c500a..d00da903f 100644 --- a/tests/corpus_integrity/README.md +++ b/tests/corpus_integrity/README.md @@ -1,6 +1,6 @@ -# Corpus Integrity Tests +# Corpus Data Tests -This directory contains tests that verify the integrity, format, and parseability of corpus files in PyThaiNLP. +This directory contains tests that verify the integrity, format, parseability, and catalog functionality of corpus data in PyThaiNLP. ## Purpose @@ -8,10 +8,22 @@ These tests are separate from regular unit tests because: 1. They test actual file loading and parsing (not mocked) 2. Downloadable corpus tests require network access and can be slow 3. They verify corpus data format and structure -4. They should only run when corpus files or corpus code changes +4. They test corpus catalog download and query functionality +5. They should only run when corpus files or corpus code changes ## Test Categories +### Corpus Catalog Tests (`test_catalog.py`) + +Tests corpus catalog functionality: +- Catalog download from remote server +- Catalog URL and path validation +- Catalog JSON structure verification +- Querying specific corpus details +- Version information validation + +**Run time:** < 1 second + ### Built-in Corpus Tests (`test_builtin_corpus.py`) Tests corpus files that are included in the package: @@ -32,11 +44,16 @@ Tests corpus files that need to be downloaded: ## Running Tests -### Run all corpus integrity tests: +### Run all corpus data tests: ```bash python -m unittest discover -s tests/corpus_integrity -v ``` +### Run only catalog tests: +```bash +python -m unittest tests.corpus_integrity.test_catalog -v +``` + ### Run only built-in corpus tests: ```bash python -m unittest tests.corpus_integrity.test_builtin_corpus -v @@ -49,7 +66,7 @@ python -m unittest tests.corpus_integrity.test_downloadable_corpus -v ## CI Integration -The corpus integrity tests run automatically via GitHub Actions workflow (`.github/workflows/corpus-integrity.yml`) when: +The corpus data tests run automatically via GitHub Actions workflow (`.github/workflows/corpus-data.yml`) when: - Changes are made to `pythainlp/corpus/**` - Changes are made to `tests/corpus_integrity/**` - The workflow file itself is modified @@ -62,17 +79,19 @@ Each test verifies: 3. **Non-empty**: Contains actual data 4. **Format validity**: Data structure matches expected format 5. **Content validity**: Contains expected content (e.g., Thai characters) +6. **Catalog functionality**: Catalog can be downloaded and queried correctly ## Adding New Tests When adding a new corpus file or function to `pythainlp.corpus`: 1. Add a test to `test_builtin_corpus.py` if it's included in the package 2. Add a test to `test_downloadable_corpus.py` if it requires download -3. Verify the test catches format errors by temporarily breaking the corpus +3. Add a test to `test_catalog.py` if it involves catalog operations +4. Verify the test catches format errors by temporarily breaking the corpus ## Relationship to Unit Tests - **Unit tests** (`tests/core/test_corpus.py`): Use mocks for speed, test code logic -- **Corpus integrity tests** (this directory): Use real data, test file integrity +- **Corpus data tests** (this directory): Use real data, test file integrity and catalog Both test suites are important and complementary. diff --git a/tests/corpus_integrity/test_catalog.py b/tests/corpus_integrity/test_catalog.py new file mode 100644 index 000000000..0ac49ecd4 --- /dev/null +++ b/tests/corpus_integrity/test_catalog.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test corpus catalog download and query functionality. + +These tests verify that the corpus catalog can be downloaded from +the remote server and queried correctly. +""" + +import unittest + +from pythainlp.corpus import ( + corpus_db_path, + corpus_db_url, + get_corpus_db, + get_corpus_db_detail, +) + + +class CorpusCatalogTestCase(unittest.TestCase): + """Test corpus catalog functionality.""" + + def test_catalog_url(self): + """Test that corpus catalog URL is valid and accessible.""" + url = corpus_db_url() + self.assertIsNotNone(url) + self.assertIsInstance(url, str) + self.assertTrue(url.startswith("https://")) + self.assertIn("pythainlp", url.lower()) + + def test_catalog_path(self): + """Test that corpus catalog path is valid.""" + path = corpus_db_path() + self.assertIsNotNone(path) + self.assertIsInstance(path, str) + self.assertTrue(path.endswith("db.json")) + + def test_catalog_download(self): + """Test that corpus catalog can be downloaded from remote server.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog, "Catalog download should succeed") + self.assertTrue(hasattr(catalog, "json"), "Catalog should have json method") + self.assertTrue( + hasattr(catalog, "status_code"), "Catalog should have status_code" + ) + + def test_catalog_structure(self): + """Test that downloaded catalog has valid JSON structure.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + self.assertIsInstance(catalog_data, dict, "Catalog should be a dictionary") + self.assertGreater( + len(catalog_data), 0, "Catalog should contain at least one corpus" + ) + + # Check that catalog entries have expected structure + for corpus_name, corpus_info in list(catalog_data.items())[:5]: + self.assertIsInstance(corpus_name, str) + self.assertIsInstance(corpus_info, dict) + self.assertIn("latest_version", corpus_info) + self.assertIn("versions", corpus_info) + self.assertIsInstance(corpus_info["versions"], dict) + + def test_catalog_known_entries(self): + """Test that catalog contains known corpus entries.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + # Check for some known corpus entries + # "test" is a standard test corpus that should always exist + self.assertIn("test", catalog_data, "Catalog should contain 'test' corpus") + + # Verify the test corpus has required fields + test_corpus = catalog_data["test"] + self.assertIn("latest_version", test_corpus) + self.assertIn("versions", test_corpus) + + def test_corpus_detail_query(self): + """Test querying details for specific corpus from local database.""" + # This tests the local query function + # First, ensure we have a local database by attempting a download/query + url = corpus_db_url() + catalog = get_corpus_db(url) + self.assertIsNotNone(catalog) + + # Test querying a corpus that may exist locally + # get_corpus_db_detail reads from local db.json file + detail = get_corpus_db_detail("test") + # Detail could be empty if not downloaded, but should return a dict + self.assertIsInstance(detail, dict) + + # Test querying non-existent corpus + detail_nonexist = get_corpus_db_detail("NONEXISTENT_CORPUS_12345") + self.assertIsInstance(detail_nonexist, dict) + self.assertEqual(len(detail_nonexist), 0, "Non-existent corpus should return empty dict") + + def test_catalog_version_info(self): + """Test that catalog entries contain valid version information.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + # Check version information for test corpus + if "test" in catalog_data: + test_corpus = catalog_data["test"] + versions = test_corpus.get("versions", {}) + + self.assertIsInstance(versions, dict) + if versions: + # Check structure of version entries + for version_key, version_info in list(versions.items())[:1]: + self.assertIsInstance(version_key, str) + self.assertIsInstance(version_info, dict) + # Version info should have these fields + self.assertIn("filename", version_info) + self.assertIn("download_url", version_info) From 1b03213d53e0c42eb6137b2408f58bfa4b5f9340 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 13:04:03 +0000 Subject: [PATCH 12/24] Address PR review feedback: remove trailing whitespace and unnecessary assertion Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .github/workflows/corpus-data.yml | 2 +- tests/core/test_cli.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-data.yml b/.github/workflows/corpus-data.yml index cbfffa579..e27fba0f8 100644 --- a/.github/workflows/corpus-data.yml +++ b/.github/workflows/corpus-data.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - + steps: - name: Checkout uses: actions/checkout@v6 diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index 56d3fe3bb..b9e8d50e7 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -35,7 +35,8 @@ def test_cli_main(self): with self.assertRaises((ArgumentError, SystemExit)): __main__.main(["thainlp", "NOT_EXIST", "command"]) - self.assertIsNone(__main__.main(["thainlp", "data", "path"])) + # Test a valid command that should succeed + __main__.main(["thainlp", "data", "path"]) def test_cli_data(self): self.assertTrue(hasattr(cli, "data")) From 8ffba33f9c937eb1e5489f4a93b001f1f81b3e53 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 13:25:04 +0000 Subject: [PATCH 13/24] Update README.md --- tests/corpus_integrity/README.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/corpus_integrity/README.md b/tests/corpus_integrity/README.md index d00da903f..4ae0217a0 100644 --- a/tests/corpus_integrity/README.md +++ b/tests/corpus_integrity/README.md @@ -1,10 +1,12 @@ -# Corpus Data Tests +# Corpus data tests -This directory contains tests that verify the integrity, format, parseability, and catalog functionality of corpus data in PyThaiNLP. +This directory contains tests that verify the integrity, format, +parseability, and catalog functionality of corpus data in PyThaiNLP. ## Purpose These tests are separate from regular unit tests because: + 1. They test actual file loading and parsing (not mocked) 2. Downloadable corpus tests require network access and can be slow 3. They verify corpus data format and structure @@ -16,50 +18,54 @@ These tests are separate from regular unit tests because: ### Corpus Catalog Tests (`test_catalog.py`) Tests corpus catalog functionality: + - Catalog download from remote server - Catalog URL and path validation - Catalog JSON structure verification - Querying specific corpus details - Version information validation -**Run time:** < 1 second - ### Built-in Corpus Tests (`test_builtin_corpus.py`) Tests corpus files that are included in the package: + - Text word lists (negations, stopwords, syllables, words, etc.) - CSV files (provinces) - Frequency data (TNC, TTC) - Name lists (family names, person names) -**Run time:** < 1 second - ### Downloadable Corpus Tests (`test_downloadable_corpus.py`) Tests corpus files that need to be downloaded: + - OSCAR word frequencies (96MB) - TNC bigram/trigram frequencies (41MB + 145MB) -**Run time:** ~17 seconds (includes download time) +This test will take longer time than others due to +size of the downloads. ## Running Tests -### Run all corpus data tests: +Run all corpus data tests: + ```bash python -m unittest discover -s tests/corpus_integrity -v ``` -### Run only catalog tests: +Run only catalog tests: + ```bash python -m unittest tests.corpus_integrity.test_catalog -v ``` -### Run only built-in corpus tests: +Run only built-in corpus tests: + ```bash python -m unittest tests.corpus_integrity.test_builtin_corpus -v ``` -### Run only downloadable corpus tests: +Run only downloadable corpus tests: + ```bash python -m unittest tests.corpus_integrity.test_downloadable_corpus -v ``` @@ -74,6 +80,7 @@ The corpus data tests run automatically via GitHub Actions workflow (`.github/wo ## What is Tested Each test verifies: + 1. **Loadability**: File can be loaded without errors 2. **Type correctness**: Returns expected data type (frozenset, list, dict) 3. **Non-empty**: Contains actual data @@ -84,6 +91,7 @@ Each test verifies: ## Adding New Tests When adding a new corpus file or function to `pythainlp.corpus`: + 1. Add a test to `test_builtin_corpus.py` if it's included in the package 2. Add a test to `test_downloadable_corpus.py` if it requires download 3. Add a test to `test_catalog.py` if it involves catalog operations From aa1657eebeb3fa1a51ac401cf3f62773253dcb9d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 13:27:37 +0000 Subject: [PATCH 14/24] Restore assertion with clarifying comment to verify command success Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/core/test_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index b9e8d50e7..26232ca3d 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -35,8 +35,9 @@ def test_cli_main(self): with self.assertRaises((ArgumentError, SystemExit)): __main__.main(["thainlp", "NOT_EXIST", "command"]) - # Test a valid command that should succeed - __main__.main(["thainlp", "data", "path"]) + # Test a valid command that should succeed (returns None) + # The assertion verifies the command completes without raising an exception + self.assertIsNone(__main__.main(["thainlp", "data", "path"])) def test_cli_data(self): self.assertTrue(hasattr(cli, "data")) From 6a4d46a9e7546c5c7911d2eb4219ebb19328e2be Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 13:32:28 +0000 Subject: [PATCH 15/24] Simplify test assertion and make README more concise Co-authored-by: bact <128572+bact@users.noreply.github.com> --- tests/README.md | 14 -------------- tests/core/test_cli.py | 9 ++++++--- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/README.md b/tests/README.md index 6560c376d..b762d1c1a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -182,21 +182,7 @@ For detailed information about corpus data tests, see: The corpus data tests are triggered automatically via GitHub Actions when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. -**Key characteristics:** -- Tests actual file loading and parsing (not mocked) -- Verifies corpus data format and structure -- Tests corpus catalog download and query -- Only runs when corpus files or corpus code changes -- Includes tests for both built-in and downloadable corpus files - **Run corpus data tests:** ```shell python -m unittest tests.corpus_integrity ``` - -Or run specific test files: -```shell -python -m unittest tests.corpus_integrity.test_catalog -python -m unittest tests.corpus_integrity.test_builtin_corpus -python -m unittest tests.corpus_integrity.test_downloadable_corpus -``` diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index 26232ca3d..14f4ea7e7 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -35,9 +35,12 @@ def test_cli_main(self): with self.assertRaises((ArgumentError, SystemExit)): __main__.main(["thainlp", "NOT_EXIST", "command"]) - # Test a valid command that should succeed (returns None) - # The assertion verifies the command completes without raising an exception - self.assertIsNone(__main__.main(["thainlp", "data", "path"])) + # Test a valid command - should not raise an exception + # (main() returns None on success, raises SystemExit on failure) + try: + __main__.main(["thainlp", "data", "path"]) + except SystemExit: + self.fail("Valid command should not raise SystemExit") def test_cli_data(self): self.assertTrue(hasattr(cli, "data")) From 464ca6b1e69268d25cb8d8add0b6571a597daa64 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 15:41:32 +0000 Subject: [PATCH 16/24] Rename corpus_integrity to corpus_data to reflect expanded test scope Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .github/workflows/corpus-data.yml | 10 +++++----- tests/README.md | 8 ++++---- tests/{corpus_integrity => corpus_data}/README.md | 10 +++++----- tests/corpus_data/__init__.py | 11 +++++++++++ .../test_builtin_corpus.py | 0 .../{corpus_integrity => corpus_data}/test_catalog.py | 0 .../test_downloadable_corpus.py | 0 tests/corpus_integrity/__init__.py | 10 ---------- 8 files changed, 25 insertions(+), 24 deletions(-) rename tests/{corpus_integrity => corpus_data}/README.md (90%) create mode 100644 tests/corpus_data/__init__.py rename tests/{corpus_integrity => corpus_data}/test_builtin_corpus.py (100%) rename tests/{corpus_integrity => corpus_data}/test_catalog.py (100%) rename tests/{corpus_integrity => corpus_data}/test_downloadable_corpus.py (100%) delete mode 100644 tests/corpus_integrity/__init__.py diff --git a/.github/workflows/corpus-data.yml b/.github/workflows/corpus-data.yml index e27fba0f8..11262182c 100644 --- a/.github/workflows/corpus-data.yml +++ b/.github/workflows/corpus-data.yml @@ -8,14 +8,14 @@ on: paths: - ".github/workflows/corpus-data.yml" - "pythainlp/corpus/**" - - "tests/corpus_integrity/**" + - "tests/corpus_data/**" pull_request: branches: - dev paths: - ".github/workflows/corpus-data.yml" - "pythainlp/corpus/**" - - "tests/corpus_integrity/**" + - "tests/corpus_data/**" # Avoid duplicate runs for the same source branch and repository concurrency: @@ -50,16 +50,16 @@ jobs: env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_integrity -p "test_catalog*.py" -v + python -m unittest discover -s tests/corpus_data -p "test_catalog*.py" -v - name: Test built-in corpus files env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_integrity -p "test_builtin_*.py" -v + python -m unittest discover -s tests/corpus_data -p "test_builtin_*.py" -v - name: Test downloadable corpus files env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_integrity -p "test_downloadable_*.py" -v + python -m unittest discover -s tests/corpus_data -p "test_downloadable_*.py" -v diff --git a/tests/README.md b/tests/README.md index b762d1c1a..c0ec9b3e4 100644 --- a/tests/README.md +++ b/tests/README.md @@ -169,7 +169,7 @@ for real-world usage: - Multi-engine robustness testing across all core tokenization engines - Very long strings that can cause performance issues (issue #893) -## Corpus data tests (corpus_integrity/) +## Corpus data tests (corpus_data/) A separate test suite that verifies the integrity, format, parseability, and catalog functionality of corpus data in PyThaiNLP. These tests are separate from regular unit tests @@ -177,12 +177,12 @@ because they test actual file loading and parsing (not mocked), downloadable cor require network access, and they verify corpus catalog operations. For detailed information about corpus data tests, see: -[tests/corpus_integrity/README.md](corpus_integrity/README.md) +[tests/corpus_data/README.md](corpus_data/README.md) The corpus data tests are triggered automatically via GitHub Actions -when changes are made to `pythainlp/corpus/**` or `tests/corpus_integrity/**`. +when changes are made to `pythainlp/corpus/**` or `tests/corpus_data/**`. **Run corpus data tests:** ```shell -python -m unittest tests.corpus_integrity +python -m unittest tests.corpus_data ``` diff --git a/tests/corpus_integrity/README.md b/tests/corpus_data/README.md similarity index 90% rename from tests/corpus_integrity/README.md rename to tests/corpus_data/README.md index 4ae0217a0..e4e737f61 100644 --- a/tests/corpus_integrity/README.md +++ b/tests/corpus_data/README.md @@ -49,32 +49,32 @@ size of the downloads. Run all corpus data tests: ```bash -python -m unittest discover -s tests/corpus_integrity -v +python -m unittest discover -s tests/corpus_data -v ``` Run only catalog tests: ```bash -python -m unittest tests.corpus_integrity.test_catalog -v +python -m unittest tests.corpus_data.test_catalog -v ``` Run only built-in corpus tests: ```bash -python -m unittest tests.corpus_integrity.test_builtin_corpus -v +python -m unittest tests.corpus_data.test_builtin_corpus -v ``` Run only downloadable corpus tests: ```bash -python -m unittest tests.corpus_integrity.test_downloadable_corpus -v +python -m unittest tests.corpus_data.test_downloadable_corpus -v ``` ## CI Integration The corpus data tests run automatically via GitHub Actions workflow (`.github/workflows/corpus-data.yml`) when: - Changes are made to `pythainlp/corpus/**` -- Changes are made to `tests/corpus_integrity/**` +- Changes are made to `tests/corpus_data/**` - The workflow file itself is modified ## What is Tested diff --git a/tests/corpus_data/__init__.py b/tests/corpus_data/__init__.py new file mode 100644 index 000000000..68752982a --- /dev/null +++ b/tests/corpus_data/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Corpus data tests. + +These tests verify the integrity, format, parseability, and catalog +functionality of corpus data in PyThaiNLP. They are separate from +regular unit tests to avoid slowing down development test cycles +with large file downloads and network access. +""" diff --git a/tests/corpus_integrity/test_builtin_corpus.py b/tests/corpus_data/test_builtin_corpus.py similarity index 100% rename from tests/corpus_integrity/test_builtin_corpus.py rename to tests/corpus_data/test_builtin_corpus.py diff --git a/tests/corpus_integrity/test_catalog.py b/tests/corpus_data/test_catalog.py similarity index 100% rename from tests/corpus_integrity/test_catalog.py rename to tests/corpus_data/test_catalog.py diff --git a/tests/corpus_integrity/test_downloadable_corpus.py b/tests/corpus_data/test_downloadable_corpus.py similarity index 100% rename from tests/corpus_integrity/test_downloadable_corpus.py rename to tests/corpus_data/test_downloadable_corpus.py diff --git a/tests/corpus_integrity/__init__.py b/tests/corpus_integrity/__init__.py deleted file mode 100644 index 94bfd2ac3..000000000 --- a/tests/corpus_integrity/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -""" -Corpus integrity tests. - -These tests verify the integrity, format, and parseability of corpus files. -They are separate from regular unit tests to avoid slowing down development -test cycles with large file downloads and parsing. -""" From 1bcfbe300ddfb416421acfd834cc41a0f01b6f87 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 6 Feb 2026 19:18:06 +0000 Subject: [PATCH 17/24] Simplify to "Corpus test" with tests.corpus package name Co-authored-by: bact <128572+bact@users.noreply.github.com> --- .../workflows/{corpus-data.yml => corpus.yml} | 18 +++++++-------- tests/README.md | 16 +++++++------- tests/{corpus_data => corpus}/README.md | 22 +++++++++---------- tests/{corpus_data => corpus}/__init__.py | 4 ++-- .../test_builtin_corpus.py | 0 tests/{corpus_data => corpus}/test_catalog.py | 0 .../test_downloadable_corpus.py | 0 7 files changed, 30 insertions(+), 30 deletions(-) rename .github/workflows/{corpus-data.yml => corpus.yml} (72%) rename tests/{corpus_data => corpus}/README.md (79%) rename tests/{corpus_data => corpus}/__init__.py (78%) rename tests/{corpus_data => corpus}/test_builtin_corpus.py (100%) rename tests/{corpus_data => corpus}/test_catalog.py (100%) rename tests/{corpus_data => corpus}/test_downloadable_corpus.py (100%) diff --git a/.github/workflows/corpus-data.yml b/.github/workflows/corpus.yml similarity index 72% rename from .github/workflows/corpus-data.yml rename to .github/workflows/corpus.yml index 11262182c..8091bfac0 100644 --- a/.github/workflows/corpus-data.yml +++ b/.github/workflows/corpus.yml @@ -1,21 +1,21 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-License-Identifier: Apache-2.0 -name: Corpus Data Test +name: Corpus Test on: push: paths: - - ".github/workflows/corpus-data.yml" + - ".github/workflows/corpus.yml" - "pythainlp/corpus/**" - - "tests/corpus_data/**" + - "tests/corpus/**" pull_request: branches: - dev paths: - - ".github/workflows/corpus-data.yml" + - ".github/workflows/corpus.yml" - "pythainlp/corpus/**" - - "tests/corpus_data/**" + - "tests/corpus/**" # Avoid duplicate runs for the same source branch and repository concurrency: @@ -26,7 +26,7 @@ concurrency: cancel-in-progress: true jobs: - corpus-data: + corpus: runs-on: ubuntu-latest permissions: contents: read @@ -50,16 +50,16 @@ jobs: env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_data -p "test_catalog*.py" -v + python -m unittest discover -s tests/corpus -p "test_catalog*.py" -v - name: Test built-in corpus files env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_data -p "test_builtin_*.py" -v + python -m unittest discover -s tests/corpus -p "test_builtin_*.py" -v - name: Test downloadable corpus files env: PYTHONIOENCODING: utf-8 run: | - python -m unittest discover -s tests/corpus_data -p "test_downloadable_*.py" -v + python -m unittest discover -s tests/corpus -p "test_downloadable_*.py" -v diff --git a/tests/README.md b/tests/README.md index c0ec9b3e4..f33ec5b3e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -169,20 +169,20 @@ for real-world usage: - Multi-engine robustness testing across all core tokenization engines - Very long strings that can cause performance issues (issue #893) -## Corpus data tests (corpus_data/) +## Corpus test (corpus/) A separate test suite that verifies the integrity, format, parseability, and catalog -functionality of corpus data in PyThaiNLP. These tests are separate from regular unit tests +functionality of corpus in PyThaiNLP. These tests are separate from regular unit tests because they test actual file loading and parsing (not mocked), downloadable corpus tests require network access, and they verify corpus catalog operations. -For detailed information about corpus data tests, see: -[tests/corpus_data/README.md](corpus_data/README.md) +For detailed information about corpus test, see: +[tests/corpus/README.md](corpus/README.md) -The corpus data tests are triggered automatically via GitHub Actions -when changes are made to `pythainlp/corpus/**` or `tests/corpus_data/**`. +The corpus test is triggered automatically via GitHub Actions +when changes are made to `pythainlp/corpus/**` or `tests/corpus/**`. -**Run corpus data tests:** +**Run corpus test:** ```shell -python -m unittest tests.corpus_data +python -m unittest tests.corpus ``` diff --git a/tests/corpus_data/README.md b/tests/corpus/README.md similarity index 79% rename from tests/corpus_data/README.md rename to tests/corpus/README.md index e4e737f61..023986b5c 100644 --- a/tests/corpus_data/README.md +++ b/tests/corpus/README.md @@ -1,7 +1,7 @@ -# Corpus data tests +# Corpus test This directory contains tests that verify the integrity, format, -parseability, and catalog functionality of corpus data in PyThaiNLP. +parseability, and catalog functionality of corpus in PyThaiNLP. ## Purpose @@ -9,7 +9,7 @@ These tests are separate from regular unit tests because: 1. They test actual file loading and parsing (not mocked) 2. Downloadable corpus tests require network access and can be slow -3. They verify corpus data format and structure +3. They verify corpus format and structure 4. They test corpus catalog download and query functionality 5. They should only run when corpus files or corpus code changes @@ -46,35 +46,35 @@ size of the downloads. ## Running Tests -Run all corpus data tests: +Run all corpus tests: ```bash -python -m unittest discover -s tests/corpus_data -v +python -m unittest discover -s tests/corpus -v ``` Run only catalog tests: ```bash -python -m unittest tests.corpus_data.test_catalog -v +python -m unittest tests.corpus.test_catalog -v ``` Run only built-in corpus tests: ```bash -python -m unittest tests.corpus_data.test_builtin_corpus -v +python -m unittest tests.corpus.test_builtin_corpus -v ``` Run only downloadable corpus tests: ```bash -python -m unittest tests.corpus_data.test_downloadable_corpus -v +python -m unittest tests.corpus.test_downloadable_corpus -v ``` ## CI Integration -The corpus data tests run automatically via GitHub Actions workflow (`.github/workflows/corpus-data.yml`) when: +The corpus test runs automatically via GitHub Actions workflow (`.github/workflows/corpus.yml`) when: - Changes are made to `pythainlp/corpus/**` -- Changes are made to `tests/corpus_data/**` +- Changes are made to `tests/corpus/**` - The workflow file itself is modified ## What is Tested @@ -100,6 +100,6 @@ When adding a new corpus file or function to `pythainlp.corpus`: ## Relationship to Unit Tests - **Unit tests** (`tests/core/test_corpus.py`): Use mocks for speed, test code logic -- **Corpus data tests** (this directory): Use real data, test file integrity and catalog +- **Corpus test** (this directory): Use real data, test file integrity and catalog Both test suites are important and complementary. diff --git a/tests/corpus_data/__init__.py b/tests/corpus/__init__.py similarity index 78% rename from tests/corpus_data/__init__.py rename to tests/corpus/__init__.py index 68752982a..9d202d4e6 100644 --- a/tests/corpus_data/__init__.py +++ b/tests/corpus/__init__.py @@ -2,10 +2,10 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 """ -Corpus data tests. +Corpus test. These tests verify the integrity, format, parseability, and catalog -functionality of corpus data in PyThaiNLP. They are separate from +functionality of corpus in PyThaiNLP. They are separate from regular unit tests to avoid slowing down development test cycles with large file downloads and network access. """ diff --git a/tests/corpus_data/test_builtin_corpus.py b/tests/corpus/test_builtin_corpus.py similarity index 100% rename from tests/corpus_data/test_builtin_corpus.py rename to tests/corpus/test_builtin_corpus.py diff --git a/tests/corpus_data/test_catalog.py b/tests/corpus/test_catalog.py similarity index 100% rename from tests/corpus_data/test_catalog.py rename to tests/corpus/test_catalog.py diff --git a/tests/corpus_data/test_downloadable_corpus.py b/tests/corpus/test_downloadable_corpus.py similarity index 100% rename from tests/corpus_data/test_downloadable_corpus.py rename to tests/corpus/test_downloadable_corpus.py From dde237b68d714ee988c28a6f6f400bc0107e1eaa Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:52:24 +0000 Subject: [PATCH 18/24] Improve clarity of corpus test suite description Reworded the description of the corpus test suite for clarity and readability. --- tests/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index f33ec5b3e..c2cac364b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -171,10 +171,12 @@ for real-world usage: ## Corpus test (corpus/) -A separate test suite that verifies the integrity, format, parseability, and catalog -functionality of corpus in PyThaiNLP. These tests are separate from regular unit tests -because they test actual file loading and parsing (not mocked), downloadable corpus tests -require network access, and they verify corpus catalog operations. +A separate test suite that verifies the integrity, format, parseability, +and catalog functionality of corpus in PyThaiNLP. + +These tests are separate from regular unit tests because they test actual +file loading and parsing (not mocked), require network access, and +can be resource intensive. For detailed information about corpus test, see: [tests/corpus/README.md](corpus/README.md) From 88a9b3c8659cab6924cdc270cb205fe504847ba3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:53:59 +0000 Subject: [PATCH 19/24] Fix formatting and capitalization in README.md --- tests/corpus/README.md | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tests/corpus/README.md b/tests/corpus/README.md index 023986b5c..a99bc1889 100644 --- a/tests/corpus/README.md +++ b/tests/corpus/README.md @@ -13,9 +13,9 @@ These tests are separate from regular unit tests because: 4. They test corpus catalog download and query functionality 5. They should only run when corpus files or corpus code changes -## Test Categories +## Test categories -### Corpus Catalog Tests (`test_catalog.py`) +### Corpus catalog tests (`test_catalog.py`) Tests corpus catalog functionality: @@ -25,7 +25,7 @@ Tests corpus catalog functionality: - Querying specific corpus details - Version information validation -### Built-in Corpus Tests (`test_builtin_corpus.py`) +### Built-in corpus tests (`test_builtin_corpus.py`) Tests corpus files that are included in the package: @@ -34,7 +34,7 @@ Tests corpus files that are included in the package: - Frequency data (TNC, TTC) - Name lists (family names, person names) -### Downloadable Corpus Tests (`test_downloadable_corpus.py`) +### Downloadable corpus tests (`test_downloadable_corpus.py`) Tests corpus files that need to be downloaded: @@ -44,7 +44,7 @@ Tests corpus files that need to be downloaded: This test will take longer time than others due to size of the downloads. -## Running Tests +## Running tests Run all corpus tests: @@ -70,25 +70,30 @@ Run only downloadable corpus tests: python -m unittest tests.corpus.test_downloadable_corpus -v ``` -## CI Integration +## CI integration + +The corpus test runs automatically via +GitHub Actions workflow (`.github/workflows/corpus.yml`) when: -The corpus test runs automatically via GitHub Actions workflow (`.github/workflows/corpus.yml`) when: - Changes are made to `pythainlp/corpus/**` - Changes are made to `tests/corpus/**` - The workflow file itself is modified -## What is Tested +## What is tested Each test verifies: 1. **Loadability**: File can be loaded without errors -2. **Type correctness**: Returns expected data type (frozenset, list, dict) -3. **Non-empty**: Contains actual data -4. **Format validity**: Data structure matches expected format -5. **Content validity**: Contains expected content (e.g., Thai characters) -6. **Catalog functionality**: Catalog can be downloaded and queried correctly +2. **Type correctness**: Returns expected data type + (frozenset, list, dict) +4. **Non-empty**: Contains actual data +5. **Format validity**: Data structure matches expected format +6. **Content validity**: Contains expected content + (e.g., Thai characters) +8. **Catalog functionality**: Catalog can be downloaded + and queried correctly -## Adding New Tests +## Adding new tests When adding a new corpus file or function to `pythainlp.corpus`: @@ -97,9 +102,11 @@ When adding a new corpus file or function to `pythainlp.corpus`: 3. Add a test to `test_catalog.py` if it involves catalog operations 4. Verify the test catches format errors by temporarily breaking the corpus -## Relationship to Unit Tests +## Relationship to unit tests -- **Unit tests** (`tests/core/test_corpus.py`): Use mocks for speed, test code logic -- **Corpus test** (this directory): Use real data, test file integrity and catalog +- **Unit tests** (`tests/core/test_corpus.py`): + Use mocks for speed, test code logic +- **Corpus test** (this directory): + Use real data, test file integrity and catalog Both test suites are important and complementary. From 70534888f804ad3095c43378efcc495acb63466a Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:55:58 +0000 Subject: [PATCH 20/24] Update README with SPDX metadata Add SPDX license and copyright information to README --- tests/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/README.md b/tests/README.md index c2cac364b..8563a8a22 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: DOCUMENTATION +# SPDX-License-Identifier: Apache-2.0 + # Test suites and execution To run a test suite, run: From 03e53bcd93917eb8ad88c55892daafcace3fffa3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:56:32 +0000 Subject: [PATCH 21/24] Update README with SPDX metadata Add SPDX license and copyright information to README --- tests/corpus/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/corpus/README.md b/tests/corpus/README.md index a99bc1889..a1a5f434c 100644 --- a/tests/corpus/README.md +++ b/tests/corpus/README.md @@ -1,3 +1,8 @@ +SPDX-FileCopyrightText: 2026 PyThaiNLP Project +SPDX-FileType: DOCUMENTATION +SPDX-License-Identifier: Apache-2.0 +--- + # Corpus test This directory contains tests that verify the integrity, format, From 99cfe6af35d2a0dae6a163a54287dcb399e7b053 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:56:46 +0000 Subject: [PATCH 22/24] Fix SPDX headers formatting in README.md --- tests/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 8563a8a22..458097b08 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,7 @@ -# SPDX-FileCopyrightText: 2026 PyThaiNLP Project -# SPDX-FileType: DOCUMENTATION -# SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 PyThaiNLP Project +SPDX-FileType: DOCUMENTATION +SPDX-License-Identifier: Apache-2.0 +--- # Test suites and execution From d2f6d6bc068d4e88c37369707aae1e6c3df182dc Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:58:01 +0000 Subject: [PATCH 23/24] Update corpus.yml --- .github/workflows/corpus.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml index 8091bfac0..1609af5cf 100644 --- a/.github/workflows/corpus.yml +++ b/.github/workflows/corpus.yml @@ -1,10 +1,13 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -name: Corpus Test +name: Corpus test on: push: + branches: + - dev paths: - ".github/workflows/corpus.yml" - "pythainlp/corpus/**" From 2a817fd389ca61faf39f3a36c8756d368c69f9be Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Fri, 6 Feb 2026 19:59:04 +0000 Subject: [PATCH 24/24] Correct README formatting for corpus test instructions Fix formatting of the 'Run corpus test' section in README. --- tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 458097b08..f17bc06eb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -189,7 +189,8 @@ For detailed information about corpus test, see: The corpus test is triggered automatically via GitHub Actions when changes are made to `pythainlp/corpus/**` or `tests/corpus/**`. -**Run corpus test:** +Run corpus test: + ```shell python -m unittest tests.corpus ```