diff --git a/.github/workflows/corpus.yml b/.github/workflows/corpus.yml new file mode 100644 index 000000000..1609af5cf --- /dev/null +++ b/.github/workflows/corpus.yml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +name: Corpus test + +on: + push: + branches: + - dev + paths: + - ".github/workflows/corpus.yml" + - "pythainlp/corpus/**" + - "tests/corpus/**" + pull_request: + branches: + - dev + paths: + - ".github/workflows/corpus.yml" + - "pythainlp/corpus/**" + - "tests/corpus/**" + +# Avoid duplicate runs for the same source branch and repository +concurrency: + group: >- + ${{ github.workflow }}-${{ + github.event.pull_request.head.repo.full_name || github.repository + }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + corpus: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: "pip" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install . + + - name: Test corpus catalog + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus -p "test_catalog*.py" -v + + - name: Test built-in corpus files + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus -p "test_builtin_*.py" -v + + - name: Test downloadable corpus files + env: + PYTHONIOENCODING: utf-8 + run: | + python -m unittest discover -s tests/corpus -p "test_downloadable_*.py" -v diff --git a/tests/README.md b/tests/README.md index af26a0fff..f17bc06eb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,3 +1,8 @@ +SPDX-FileCopyrightText: 2026 PyThaiNLP Project +SPDX-FileType: DOCUMENTATION +SPDX-License-Identifier: Apache-2.0 +--- + # Test suites and execution To run a test suite, run: @@ -155,7 +160,6 @@ By separating tests by dependency group, we can: - Requires: Internet connection, may involve large downloads - Test case class suffix: `TestCaseN` - ## Robustness tests (test_robustness.py) A comprehensive test suite within core tests that tests edge cases important @@ -169,3 +173,24 @@ for real-world usage: - Thai-specific edge cases with combining characters and mixed scripts - Multi-engine robustness testing across all core tokenization engines - Very long strings that can cause performance issues (issue #893) + +## Corpus test (corpus/) + +A separate test suite that verifies the integrity, format, parseability, +and catalog functionality of corpus in PyThaiNLP. + +These tests are separate from regular unit tests because they test actual +file loading and parsing (not mocked), require network access, and +can be resource intensive. + +For detailed information about corpus test, see: +[tests/corpus/README.md](corpus/README.md) + +The corpus test is triggered automatically via GitHub Actions +when changes are made to `pythainlp/corpus/**` or `tests/corpus/**`. + +Run corpus test: + +```shell +python -m unittest tests.corpus +``` diff --git a/tests/core/test_cli.py b/tests/core/test_cli.py index 2c0d38854..14f4ea7e7 100644 --- a/tests/core/test_cli.py +++ b/tests/core/test_cli.py @@ -2,8 +2,11 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import io import unittest from argparse import ArgumentError +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import MagicMock, patch from pythainlp import __main__, cli from pythainlp.cli.data import App as DataApp @@ -19,147 +22,185 @@ def test_cli(self): cli.exit_if_empty("", None) def test_cli_main(self): - # call with no argument, should exit with 2 - with self.assertRaises(SystemExit) as ex: - __main__.main() - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + # call with no argument, should exit with 2 + with self.assertRaises(SystemExit) as ex: + __main__.main() + self.assertEqual(ex.exception.code, 2) + + with self.assertRaises((ArgumentError, SystemExit)): + __main__.main(["thainlp"]) + + with self.assertRaises((ArgumentError, SystemExit)): + __main__.main(["thainlp", "NOT_EXIST", "command"]) + + # Test a valid command - should not raise an exception + # (main() returns None on success, raises SystemExit on failure) + try: + __main__.main(["thainlp", "data", "path"]) + except SystemExit: + self.fail("Valid command should not raise SystemExit") - with self.assertRaises((ArgumentError, SystemExit)): - __main__.main(["thainlp"]) + def test_cli_data(self): + self.assertTrue(hasattr(cli, "data")) - with self.assertRaises((ArgumentError, SystemExit)): - __main__.main(["thainlp", "NOT_EXIST", "command"]) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "data"]) + self.assertEqual(ex.exception.code, 2) - self.assertIsNone(__main__.main(["thainlp", "data", "path"])) + # Mock network calls to avoid corpus downloads + mock_response = MagicMock() + mock_response.json.return_value = { + "test": {"latest_version": "0.1", "versions": {}}, + } - def test_cli_data(self): - self.assertTrue(hasattr(cli, "data")) + with patch("pythainlp.corpus.get_corpus_db", return_value=mock_response): + self.assertIsNotNone(DataApp(["thainlp", "data", "catalog"])) + + self.assertIsNotNone(DataApp(["thainlp", "data", "path"])) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "data"]) - self.assertEqual(ex.exception.code, 2) + # Mock download to avoid network + with patch("pythainlp.corpus.download", return_value=True): + self.assertIsNotNone(DataApp(["thainlp", "data", "get", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "catalog"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "path"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "get", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "info", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "test"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "get", "NOT_EXIST"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "info", "NOT_EXIST"])) - self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "NOT_EXIST"])) + self.assertIsNotNone(DataApp(["thainlp", "data", "info", "test"])) + + # Mock remove to avoid side effects + with patch("pythainlp.corpus.remove", return_value=True): + self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "test"])) + + # Test with non-existing corpus + with patch("pythainlp.corpus.download", return_value=False): + self.assertIsNotNone(DataApp(["thainlp", "data", "get", "NOT_EXIST"])) + + self.assertIsNotNone(DataApp(["thainlp", "data", "info", "NOT_EXIST"])) + + with patch("pythainlp.corpus.remove", return_value=False): + self.assertIsNotNone(DataApp(["thainlp", "data", "rm", "NOT_EXIST"])) def test_cli_misspell(self): self.assertTrue(hasattr(cli, "misspell")) - with self.assertRaises(SystemExit) as ex: - MisspellApp(["thainlp", "misspell"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - MisspellApp( - [ - "thainlp", - "misspell", - "--file", - "./tests/data/text.txt", - "--seed", - "1", - "--misspell-ratio", - "0.05", - "--output", - "./tests/data/misspell_output.tmp", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + MisspellApp(["thainlp", "misspell"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + MisspellApp( + [ + "thainlp", + "misspell", + "--file", + "./tests/data/text.txt", + "--seed", + "1", + "--misspell-ratio", + "0.05", + "--output", + "./tests/data/misspell_output.tmp", + ] + ) ) - ) def test_cli_soundex(self): self.assertTrue(hasattr(cli, "soundex")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "soundex"]) - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "soundex"]) + self.assertEqual(ex.exception.code, 2) - self.assertIsNotNone(SoundexApp(["thainlp", "soundex", "ทดสอบ"])) + self.assertIsNotNone(SoundexApp(["thainlp", "soundex", "ทดสอบ"])) def test_cli_tag(self): self.assertTrue(hasattr(cli, "tag")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "tag"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - TagApp( - [ - "thainlp", - "tag", - "pos", - "-s", - " ", - "มอเตอร์ไซค์ ความว่างเปล่า", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "tag"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + TagApp( + [ + "thainlp", + "tag", + "pos", + "-s", + " ", + "มอเตอร์ไซค์ ความว่างเปล่า", + ] + ) ) - ) - self.assertIsNotNone( - TagApp( - [ - "thainlp", - "tag", - "role", - "-s", - " ", - "มอเตอร์ไซค์ ความว่างเปล่า", - ] + self.assertIsNotNone( + TagApp( + [ + "thainlp", + "tag", + "role", + "-s", + " ", + "มอเตอร์ไซค์ ความว่างเปล่า", + ] + ) ) - ) def test_cli_tokenize(self): self.assertTrue(hasattr(cli, "tokenize")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "tokenize"]) - self.assertEqual(ex.exception.code, 2) - - self.assertIsNotNone( - TokenizeApp(["thainlp", "tokenize", "NOT_EXIST", "ไม่มีอยู่ จริง"]) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "subword", - "-s", - "|", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "tokenize"]) + self.assertEqual(ex.exception.code, 2) + + self.assertIsNotNone( + TokenizeApp(["thainlp", "tokenize", "NOT_EXIST", "ไม่มีอยู่ จริง"]) + ) + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "subword", + "-s", + "|", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "syllable", - "-s", - "|", - "-w", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "syllable", + "-s", + "|", + "-w", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "word", - "-nw", - "-a", - "newmm", - "-s", - "|", - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", - ] + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "word", + "-nw", + "-a", + "newmm", + "-s", + "|", + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้", + ] + ) ) - ) diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index e1acc2913..6c3a82b10 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -4,6 +4,7 @@ import os import unittest +from unittest.mock import mock_open, patch from pythainlp.corpus import ( countries, @@ -111,18 +112,110 @@ def test_corpus(self): self.assertFalse(download(name="test", version="0.0.1")) def test_oscar(self): - self.assertIsNotNone(oscar.word_freqs()) - self.assertIsNotNone(oscar.unigram_word_freqs()) + # Mock the oscar corpus file to avoid slow download and parsing + # Format: word,count + # Note: A line starting with space becomes empty string after strip() + mock_oscar_data = """word,count +คน,1000 +ไทย,500 +ภาษา,300 + ,100 +"test",50 +""" + mock_path = "/mock/path/oscar_icu" + + with patch('pythainlp.corpus.oscar.get_corpus_path', return_value=mock_path): + with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + result = oscar.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Verify parsing logic + self.assertEqual(result[0], ("คน", 1000)) + # Space line becomes empty string (not ) due to strip() + self.assertIn(("", 100), result) + # Verify quoted values are filtered out + for word, _ in result: + self.assertNotIn('"', word) + + # Reset mock for unigram test + with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + result_unigram = oscar.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) def test_tnc(self): - self.assertIsNotNone(tnc.word_freqs()) - self.assertIsNotNone(tnc.unigram_word_freqs()) - self.assertIsNotNone(tnc.bigram_word_freqs()) - self.assertIsNotNone(tnc.trigram_word_freqs()) + # Mock TNC unigram corpus + mock_unigram_data = """คน 1000 +ไทย 500 +ภาษา 300""" + + # Mock TNC bigram corpus + mock_bigram_data = """คน ไทย 100 +ไทย ภาษา 50 +ภาษา ไทย 30""" + + # Mock TNC trigram corpus + mock_trigram_data = """คน ไทย ภาษา 10 +ไทย ภาษา ไทย 5 +ภาษา ไทย คน 3""" + + # Test unigram functions + with patch('pythainlp.corpus.tnc.get_corpus', return_value=frozenset(mock_unigram_data.split('\n'))): + result = tnc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Check that at least one expected entry exists (order not guaranteed) + self.assertIn(("คน", 1000), result) + + result_unigram = tnc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) + + # Test bigram function + mock_bigram_path = "/mock/path/bigram" + with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_bigram_path): + with patch('builtins.open', mock_open(read_data=mock_bigram_data)): + result_bigram = tnc.bigram_word_freqs() + self.assertIsNotNone(result_bigram) + self.assertIsInstance(result_bigram, dict) + self.assertGreater(len(result_bigram), 0) + self.assertEqual(result_bigram[("คน", "ไทย")], 100) + + # Test trigram function + mock_trigram_path = "/mock/path/trigram" + with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_trigram_path): + with patch('builtins.open', mock_open(read_data=mock_trigram_data)): + result_trigram = tnc.trigram_word_freqs() + self.assertIsNotNone(result_trigram) + self.assertIsInstance(result_trigram, dict) + self.assertGreater(len(result_trigram), 0) + self.assertEqual(result_trigram[("คน", "ไทย", "ภาษา")], 10) def test_ttc(self): - self.assertIsNotNone(ttc.word_freqs()) - self.assertIsNotNone(ttc.unigram_word_freqs()) + # Mock TTC corpus + mock_ttc_data = """คน 1000 +ไทย 500 +ภาษา 300""" + + with patch('pythainlp.corpus.ttc.get_corpus', return_value=frozenset(mock_ttc_data.split('\n'))): + result = ttc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Check that at least one expected entry exists (order not guaranteed) + self.assertIn(("คน", 1000), result) + + result_unigram = ttc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + self.assertEqual(result_unigram["คน"], 1000) def test_revise_wordset(self): training_data = [ diff --git a/tests/corpus/README.md b/tests/corpus/README.md new file mode 100644 index 000000000..a1a5f434c --- /dev/null +++ b/tests/corpus/README.md @@ -0,0 +1,117 @@ +SPDX-FileCopyrightText: 2026 PyThaiNLP Project +SPDX-FileType: DOCUMENTATION +SPDX-License-Identifier: Apache-2.0 +--- + +# Corpus test + +This directory contains tests that verify the integrity, format, +parseability, and catalog functionality of corpus in PyThaiNLP. + +## Purpose + +These tests are separate from regular unit tests because: + +1. They test actual file loading and parsing (not mocked) +2. Downloadable corpus tests require network access and can be slow +3. They verify corpus format and structure +4. They test corpus catalog download and query functionality +5. They should only run when corpus files or corpus code changes + +## Test categories + +### Corpus catalog tests (`test_catalog.py`) + +Tests corpus catalog functionality: + +- Catalog download from remote server +- Catalog URL and path validation +- Catalog JSON structure verification +- Querying specific corpus details +- Version information validation + +### Built-in corpus tests (`test_builtin_corpus.py`) + +Tests corpus files that are included in the package: + +- Text word lists (negations, stopwords, syllables, words, etc.) +- CSV files (provinces) +- Frequency data (TNC, TTC) +- Name lists (family names, person names) + +### Downloadable corpus tests (`test_downloadable_corpus.py`) + +Tests corpus files that need to be downloaded: + +- OSCAR word frequencies (96MB) +- TNC bigram/trigram frequencies (41MB + 145MB) + +This test will take longer time than others due to +size of the downloads. + +## Running tests + +Run all corpus tests: + +```bash +python -m unittest discover -s tests/corpus -v +``` + +Run only catalog tests: + +```bash +python -m unittest tests.corpus.test_catalog -v +``` + +Run only built-in corpus tests: + +```bash +python -m unittest tests.corpus.test_builtin_corpus -v +``` + +Run only downloadable corpus tests: + +```bash +python -m unittest tests.corpus.test_downloadable_corpus -v +``` + +## CI integration + +The corpus test runs automatically via +GitHub Actions workflow (`.github/workflows/corpus.yml`) when: + +- Changes are made to `pythainlp/corpus/**` +- Changes are made to `tests/corpus/**` +- The workflow file itself is modified + +## What is tested + +Each test verifies: + +1. **Loadability**: File can be loaded without errors +2. **Type correctness**: Returns expected data type + (frozenset, list, dict) +4. **Non-empty**: Contains actual data +5. **Format validity**: Data structure matches expected format +6. **Content validity**: Contains expected content + (e.g., Thai characters) +8. **Catalog functionality**: Catalog can be downloaded + and queried correctly + +## Adding new tests + +When adding a new corpus file or function to `pythainlp.corpus`: + +1. Add a test to `test_builtin_corpus.py` if it's included in the package +2. Add a test to `test_downloadable_corpus.py` if it requires download +3. Add a test to `test_catalog.py` if it involves catalog operations +4. Verify the test catches format errors by temporarily breaking the corpus + +## Relationship to unit tests + +- **Unit tests** (`tests/core/test_corpus.py`): + Use mocks for speed, test code logic +- **Corpus test** (this directory): + Use real data, test file integrity and catalog + +Both test suites are important and complementary. diff --git a/tests/corpus/__init__.py b/tests/corpus/__init__.py new file mode 100644 index 000000000..9d202d4e6 --- /dev/null +++ b/tests/corpus/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Corpus test. + +These tests verify the integrity, format, parseability, and catalog +functionality of corpus in PyThaiNLP. They are separate from +regular unit tests to avoid slowing down development test cycles +with large file downloads and network access. +""" diff --git a/tests/corpus/test_builtin_corpus.py b/tests/corpus/test_builtin_corpus.py new file mode 100644 index 000000000..d6a310d04 --- /dev/null +++ b/tests/corpus/test_builtin_corpus.py @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test integrity and parseability of built-in corpus files. + +These tests verify that all corpus files included in the package +can be loaded and parsed correctly. +""" + +import unittest + +from pythainlp.corpus import ( + countries, + find_synonyms, + get_corpus, + provinces, + thai_family_names, + thai_female_names, + thai_icu_words, + thai_male_names, + thai_negations, + thai_orst_words, + thai_stopwords, + thai_syllables, + thai_synonyms, + thai_volubilis_words, + thai_wikipedia_titles, + thai_words, + ttc, +) + + +class BuiltinCorpusIntegrityTestCase(unittest.TestCase): + """Test integrity of built-in corpus files.""" + + def test_negations(self): + """Test thai_negations corpus can be loaded and is not empty.""" + result = thai_negations() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_stopwords(self): + """Test thai_stopwords corpus can be loaded and is not empty.""" + result = thai_stopwords() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_syllables(self): + """Test thai_syllables corpus can be loaded and is not empty.""" + result = thai_syllables() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_words(self): + """Test thai_words corpus can be loaded and is not empty.""" + result = thai_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify it contains actual Thai content + self.assertTrue(any('\u0e00' <= char <= '\u0e7f' for item in result for char in item)) + + def test_synonyms(self): + """Test thai_synonyms corpus can be loaded and parsed correctly.""" + result = thai_synonyms() + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + # Test that find_synonyms works with the loaded data + synonyms = find_synonyms("หมู") + self.assertIsInstance(synonyms, list) + self.assertGreater(len(synonyms), 0) + + def test_icu_words(self): + """Test thai_icu_words corpus can be loaded and is not empty.""" + result = thai_icu_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_orst_words(self): + """Test thai_orst_words corpus can be loaded and is not empty.""" + result = thai_orst_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_volubilis_words(self): + """Test thai_volubilis_words corpus can be loaded and is not empty.""" + result = thai_volubilis_words() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_wikipedia_titles(self): + """Test thai_wikipedia_titles corpus can be loaded and is not empty.""" + result = thai_wikipedia_titles() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_countries(self): + """Test countries corpus can be loaded and is not empty.""" + result = countries() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_provinces(self): + """Test provinces corpus can be loaded and parsed correctly.""" + result = provinces() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + # Test with details + result_details = provinces(details=True) + self.assertIsInstance(result_details, list) + self.assertEqual(len(result), len(result_details)) + + def test_family_names(self): + """Test thai_family_names corpus can be loaded and is not empty.""" + result = thai_family_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_female_names(self): + """Test thai_female_names corpus can be loaded and is not empty.""" + result = thai_female_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_male_names(self): + """Test thai_male_names corpus can be loaded and is not empty.""" + result = thai_male_names() + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + + def test_ttc_freq(self): + """Test TTC frequency corpus can be loaded and parsed correctly.""" + result = ttc.word_freqs() + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + # Verify format: list of (word, frequency) tuples + for item in result[:10]: # Check first 10 items + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + + # Test unigram version + result_unigram = ttc.unigram_word_freqs() + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + def test_tnc_freq(self): + """Test TNC frequency corpus can be loaded and parsed correctly.""" + # Test unigram from built-in file + result = get_corpus("tnc_freq.txt") + self.assertIsInstance(result, frozenset) + self.assertGreater(len(result), 0) + # Verify format: tab-separated word and frequency + for line in list(result)[:10]: # Check first 10 lines + parts = line.split('\t') + self.assertGreaterEqual(len(parts), 2) diff --git a/tests/corpus/test_catalog.py b/tests/corpus/test_catalog.py new file mode 100644 index 000000000..0ac49ecd4 --- /dev/null +++ b/tests/corpus/test_catalog.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test corpus catalog download and query functionality. + +These tests verify that the corpus catalog can be downloaded from +the remote server and queried correctly. +""" + +import unittest + +from pythainlp.corpus import ( + corpus_db_path, + corpus_db_url, + get_corpus_db, + get_corpus_db_detail, +) + + +class CorpusCatalogTestCase(unittest.TestCase): + """Test corpus catalog functionality.""" + + def test_catalog_url(self): + """Test that corpus catalog URL is valid and accessible.""" + url = corpus_db_url() + self.assertIsNotNone(url) + self.assertIsInstance(url, str) + self.assertTrue(url.startswith("https://")) + self.assertIn("pythainlp", url.lower()) + + def test_catalog_path(self): + """Test that corpus catalog path is valid.""" + path = corpus_db_path() + self.assertIsNotNone(path) + self.assertIsInstance(path, str) + self.assertTrue(path.endswith("db.json")) + + def test_catalog_download(self): + """Test that corpus catalog can be downloaded from remote server.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog, "Catalog download should succeed") + self.assertTrue(hasattr(catalog, "json"), "Catalog should have json method") + self.assertTrue( + hasattr(catalog, "status_code"), "Catalog should have status_code" + ) + + def test_catalog_structure(self): + """Test that downloaded catalog has valid JSON structure.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + self.assertIsInstance(catalog_data, dict, "Catalog should be a dictionary") + self.assertGreater( + len(catalog_data), 0, "Catalog should contain at least one corpus" + ) + + # Check that catalog entries have expected structure + for corpus_name, corpus_info in list(catalog_data.items())[:5]: + self.assertIsInstance(corpus_name, str) + self.assertIsInstance(corpus_info, dict) + self.assertIn("latest_version", corpus_info) + self.assertIn("versions", corpus_info) + self.assertIsInstance(corpus_info["versions"], dict) + + def test_catalog_known_entries(self): + """Test that catalog contains known corpus entries.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + # Check for some known corpus entries + # "test" is a standard test corpus that should always exist + self.assertIn("test", catalog_data, "Catalog should contain 'test' corpus") + + # Verify the test corpus has required fields + test_corpus = catalog_data["test"] + self.assertIn("latest_version", test_corpus) + self.assertIn("versions", test_corpus) + + def test_corpus_detail_query(self): + """Test querying details for specific corpus from local database.""" + # This tests the local query function + # First, ensure we have a local database by attempting a download/query + url = corpus_db_url() + catalog = get_corpus_db(url) + self.assertIsNotNone(catalog) + + # Test querying a corpus that may exist locally + # get_corpus_db_detail reads from local db.json file + detail = get_corpus_db_detail("test") + # Detail could be empty if not downloaded, but should return a dict + self.assertIsInstance(detail, dict) + + # Test querying non-existent corpus + detail_nonexist = get_corpus_db_detail("NONEXISTENT_CORPUS_12345") + self.assertIsInstance(detail_nonexist, dict) + self.assertEqual(len(detail_nonexist), 0, "Non-existent corpus should return empty dict") + + def test_catalog_version_info(self): + """Test that catalog entries contain valid version information.""" + url = corpus_db_url() + catalog = get_corpus_db(url) + + self.assertIsNotNone(catalog) + catalog_data = catalog.json() + + # Check version information for test corpus + if "test" in catalog_data: + test_corpus = catalog_data["test"] + versions = test_corpus.get("versions", {}) + + self.assertIsInstance(versions, dict) + if versions: + # Check structure of version entries + for version_key, version_info in list(versions.items())[:1]: + self.assertIsInstance(version_key, str) + self.assertIsInstance(version_info, dict) + # Version info should have these fields + self.assertIn("filename", version_info) + self.assertIn("download_url", version_info) diff --git a/tests/corpus/test_downloadable_corpus.py b/tests/corpus/test_downloadable_corpus.py new file mode 100644 index 000000000..421656a76 --- /dev/null +++ b/tests/corpus/test_downloadable_corpus.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Test integrity and parseability of downloadable corpus files. + +These tests verify that corpus files that need to be downloaded +can be fetched and parsed correctly. These tests may take longer +to run due to network downloads. +""" + +import unittest + +from pythainlp.corpus import oscar, tnc + + +class DownloadableCorpusIntegrityTestCase(unittest.TestCase): + """Test integrity of downloadable corpus files.""" + + def test_oscar_corpus(self): + """Test OSCAR corpus can be downloaded and parsed correctly.""" + # Test word_freqs + result = oscar.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + # Verify format: list of (word, frequency) tuples + for item in result[:10]: # Check first 10 items + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + self.assertGreater(item[1], 0) + + # Test unigram_word_freqs + result_unigram = oscar.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + # Verify dict values are integers + for word, freq in list(result_unigram.items())[:10]: + self.assertIsInstance(word, str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) + + def test_tnc_unigram(self): + """Test TNC unigram corpus can be loaded and parsed correctly.""" + result = tnc.word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + # Verify format + for item in result[:10]: + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 2) + self.assertIsInstance(item[0], str) + self.assertIsInstance(item[1], int) + + # Test unigram version + result_unigram = tnc.unigram_word_freqs() + self.assertIsNotNone(result_unigram) + self.assertIsInstance(result_unigram, dict) + self.assertGreater(len(result_unigram), 0) + + def test_tnc_bigram(self): + """Test TNC bigram corpus can be downloaded and parsed correctly.""" + result = tnc.bigram_word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + + # Verify format: dict with tuple keys (word1, word2) -> frequency + for key, freq in list(result.items())[:10]: + self.assertIsInstance(key, tuple) + self.assertEqual(len(key), 2) + self.assertIsInstance(key[0], str) + self.assertIsInstance(key[1], str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) + + def test_tnc_trigram(self): + """Test TNC trigram corpus can be downloaded and parsed correctly.""" + result = tnc.trigram_word_freqs() + self.assertIsNotNone(result) + self.assertIsInstance(result, dict) + self.assertGreater(len(result), 0) + + # Verify format: dict with tuple keys (word1, word2, word3) -> frequency + for key, freq in list(result.items())[:10]: + self.assertIsInstance(key, tuple) + self.assertEqual(len(key), 3) + self.assertIsInstance(key[0], str) + self.assertIsInstance(key[1], str) + self.assertIsInstance(key[2], str) + self.assertIsInstance(freq, int) + self.assertGreater(freq, 0) diff --git a/tests/extra/testx_cli.py b/tests/extra/testx_cli.py index 7ec46d6bc..e53c55762 100644 --- a/tests/extra/testx_cli.py +++ b/tests/extra/testx_cli.py @@ -2,7 +2,9 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +import io import unittest +from contextlib import redirect_stderr, redirect_stdout from pythainlp import cli from pythainlp.cli.benchmark import App as BenchmarkApp @@ -14,39 +16,43 @@ class CliTestCaseX(unittest.TestCase): def test_cli_benchmark(self): self.assertTrue(hasattr(cli, "benchmark")) - with self.assertRaises(SystemExit) as ex: - DataApp(["thainlp", "benchmark"]) - self.assertEqual(ex.exception.code, 2) + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as ex: + DataApp(["thainlp", "benchmark"]) + self.assertEqual(ex.exception.code, 2) - self.assertIsNotNone( - BenchmarkApp( - [ - "thainlp", - "benchmark", - "word-tokenization", - "--input-file", - "./tests/data/input.txt", - "--test-file", - "./tests/data/test.txt", - "--save-details", - ] + self.assertIsNotNone( + BenchmarkApp( + [ + "thainlp", + "benchmark", + "word-tokenization", + "--input-file", + "./tests/data/input.txt", + "--test-file", + "./tests/data/test.txt", + "--save-details", + ] + ) ) - ) def test_cli_tokenize(self): - self.assertIsNotNone( - TokenizeApp( - [ - "thainlp", - "tokenize", - "sent", - "-s", - "|", - ( - "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้" - "กระสุนสำหรับสมองของคุณวันนี้" - "แต่คุณก็จะลืมมันไปทั้งหมดอีกครั้ง" - ), - ] + # Suppress output to keep test log clean + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertIsNotNone( + TokenizeApp( + [ + "thainlp", + "tokenize", + "sent", + "-s", + "|", + ( + "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้" + "กระสุนสำหรับสมองของคุณวันนี้" + "แต่คุณก็จะลืมมันไปทั้งหมดอีกครั้ง" + ), + ] + ) ) - )