diff --git a/docs/api/util.rst b/docs/api/util.rst index ef38bcd44..a5339cedb 100644 --- a/docs/api/util.rst +++ b/docs/api/util.rst @@ -27,16 +27,26 @@ Modules The `bahttext` function specializes in converting numerical values into Thai Baht text, an essential feature for rendering financial data or monetary amounts in a user-friendly Thai format. -.. autofunction:: convert_years +.. autofunction:: censor_profanity :noindex: - The `convert_years` function is designed to facilitate the conversion of Western calendar years into Thai Buddhist Era (BE) years. This is significant for presenting dates and years in a Thai context. + The `censor_profanity` function replaces profanity words in Thai text with a replacement character (default: "*"). Users can provide custom profanity words in addition to the built-in list for content moderation and filtering. .. autofunction:: collate :noindex: The `collate` function is a versatile tool for sorting Thai text in a locale-specific manner. It ensures that text data is sorted correctly, taking into account the Thai language's unique characteristics. +.. autofunction:: contains_profanity + :noindex: + + The `contains_profanity` function checks if Thai text contains profanity words. It returns True if profanity is detected and False otherwise. Users can provide custom profanity words for enhanced content moderation. + +.. autofunction:: convert_years + :noindex: + + The `convert_years` function is designed to facilitate the conversion of Western calendar years into Thai Buddhist Era (BE) years. This is significant for presenting dates and years in a Thai context. + .. autofunction:: count_thai_chars :noindex: @@ -77,6 +87,11 @@ Modules The `find_keyword` function is a powerful utility for identifying keywords and key phrases in text data. It is a fundamental component for text analysis and information extraction tasks. +.. autofunction:: find_profanity + :noindex: + + The `find_profanity` function identifies and returns a list of all profanity words found in Thai text. Users can provide custom profanity words to enhance detection capabilities for content moderation. + .. autofunction:: ipa_to_rtgs :noindex: diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 6d04c4593..db112cc4f 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -34,6 +34,7 @@ "thai_male_names", "thai_negations", "thai_orst_words", + "thai_profanity_words", "thai_stopwords", "thai_syllables", "thai_synonym", @@ -113,6 +114,7 @@ def corpus_db_path() -> str: thai_male_names, thai_negations, thai_orst_words, + thai_profanity_words, thai_stopwords, thai_syllables, thai_synonym, diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index cf467a754..fe63c43a8 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -49,6 +49,9 @@ _THAI_NEGATIONS: frozenset[str] = frozenset() _THAI_NEGATIONS_FILENAME = "negations_th.txt" +_THAI_PROFANITY_WORDS: frozenset[str] = frozenset() +_THAI_PROFANITY_WORDS_FILENAME = "profanity_th.txt" + _THAI_FAMLIY_NAMES: frozenset[str] = frozenset() _THAI_FAMLIY_NAMES_FILENAME = "family_names_th.txt" _THAI_FEMALE_NAMES: frozenset[str] = frozenset() @@ -205,6 +208,21 @@ def thai_negations() -> frozenset[str]: return _THAI_NEGATIONS +def thai_profanity_words() -> frozenset[str]: + """Return a frozenset of Thai profanity words for content filtering. + \n(See: `dev/pythainlp/corpus/profanity_th.txt\ + `_) + + :return: :class:`frozenset` containing profanity words in the Thai language. + :rtype: :class:`frozenset` + """ + global _THAI_PROFANITY_WORDS + if not _THAI_PROFANITY_WORDS: + _THAI_PROFANITY_WORDS = get_corpus(_THAI_PROFANITY_WORDS_FILENAME, comments=False) + + return _THAI_PROFANITY_WORDS + + def thai_family_names() -> frozenset[str]: """Return a frozenset of Thai family names \n(See: `dev/pythainlp/corpus/family_names_th.txt\ diff --git a/pythainlp/corpus/profanity_th.txt b/pythainlp/corpus/profanity_th.txt new file mode 100644 index 000000000..ddb35559d --- /dev/null +++ b/pythainlp/corpus/profanity_th.txt @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: OTHER +# SPDX-FileType: TEXT +# SPDX-License-Identifier: Apache-2.0 +# +# Thai profanity words +# +# This list contains common Thai profanity words for filtering/detection purposes +# Words are stored without tone marks or variations for broader matching +ควย +สัส +เหี้ย +ไอ้เหี้ย +ไอ้สัส +ระยำ +เย็ด +มึง +ไอ้มึง +กู +ไอ้กู +เชี่ย +ดอกทอง +สัตว์ +ไอ้สัตว์ +ชาติหมา +ตาย +ไอ้ตาย +บ้า +ไอ้บ้า +เวร +ไอ้เวร +เหี้ยน \ No newline at end of file diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index 3f1eb8520..11454c160 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -9,7 +9,9 @@ "abbreviation_to_full_text", "arabic_digit_to_thai_digit", "bahttext", + "censor_profanity", "collate", + "contains_profanity", "convert_years", "count_thai_chars", "countthai", @@ -20,6 +22,7 @@ "eng_to_thai", "expand_maiyamok", "find_keyword", + "find_profanity", "ipa_to_rtgs", "is_native_thai", "isthai", @@ -109,6 +112,11 @@ ) from pythainlp.util.numtoword import bahttext, num_to_thaiword from pythainlp.util.phoneme import ipa_to_rtgs, nectec_to_ipa, remove_tone_ipa +from pythainlp.util.profanity import ( + censor_profanity, + contains_profanity, + find_profanity, +) from pythainlp.util.remove_trailing_repeat_consonants import ( remove_trailing_repeat_consonants, ) diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py new file mode 100644 index 000000000..883116c57 --- /dev/null +++ b/pythainlp/util/profanity.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Profanity detection for Thai language +""" +from __future__ import annotations + +from pythainlp.corpus.common import thai_profanity_words, thai_words +from pythainlp.tokenize import word_tokenize +from pythainlp.util.trie import dict_trie + + +def contains_profanity( + text: str, custom_words: set[str] | None = None, engine: str = "newmm" +) -> bool: + """ + Check if the given text contains profanity words. + + :param str text: Thai text to check + :param set custom_words: additional profanity words to check (default: None) + :param str engine: tokenization engine (default: "newmm") + :return: True if text contains profanity, False otherwise + :rtype: bool + + :Example: + :: + + from pythainlp.util import contains_profanity + + print(contains_profanity("สวัสดีครับ")) + # output: False + + print(contains_profanity("คำหยาบคาย")) + # output: True if the word is in the profanity list + + # Add custom profanity words + print(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: True + """ + if not text: + return False + + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) + + for token in tokens: + if token in profanity_set: + return True + + return False + + +def find_profanity( + text: str, custom_words: set[str] | None = None, engine: str = "newmm" +) -> list[str]: + """ + Find all profanity words in the given text. + + :param str text: Thai text to check + :param set custom_words: additional profanity words to check (default: None) + :param str engine: tokenization engine (default: "newmm") + :return: list of profanity words found in the text + :rtype: list[str] + + :Example: + :: + + from pythainlp.util import find_profanity + + print(find_profanity("สวัสดีครับ")) + # output: [] + + print(find_profanity("text with profanity words")) + # output: ['profanity_word1', 'profanity_word2'] + + # Add custom profanity words + print(find_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: ['คำใหม่'] + """ + if not text: + return [] + + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize(text, custom_dict=custom_dict, engine=engine) + + found_profanity = [] + for token in tokens: + if token in profanity_set: + found_profanity.append(token) + + return found_profanity + + +def censor_profanity( + text: str, + replacement: str = "*", + custom_words: set[str] | None = None, + engine: str = "newmm", +) -> str: + """ + Replace profanity words in the text with a replacement character. + + :param str text: Thai text to censor + :param str replacement: character to replace profanity with (default: "*") + :param set custom_words: additional profanity words to censor (default: None) + :param str engine: tokenization engine (default: "newmm") + :return: Text with profanity words censored + :rtype: str + + :Example: + :: + + from pythainlp.util import censor_profanity + + print(censor_profanity("สวัสดีครับ")) + # output: สวัสดีครับ + + print(censor_profanity("text with profanity word")) + # output: text with *** word + + # Add custom profanity words + print(censor_profanity("คำใหม่", custom_words={"คำใหม่"})) + # output: ****** + """ + if not text: + return text + + profanity_set = set(thai_profanity_words()) + if custom_words: + profanity_set.update(custom_words) + + # Create custom dictionary that merges thai_words and profanity_set + # for better tokenization + custom_dict_set = set(thai_words()) + custom_dict_set.update(profanity_set) + custom_dict = dict_trie(dict_source=custom_dict_set) + + tokens = word_tokenize( + text, custom_dict=custom_dict, engine=engine, keep_whitespace=True + ) + + censored_tokens = [] + for token in tokens: + if token in profanity_set: + censored_tokens.append(replacement * len(token)) + else: + censored_tokens.append(token) + + return "".join(censored_tokens) diff --git a/tests/core/test_profanity.py b/tests/core/test_profanity.py new file mode 100644 index 000000000..6b1bde686 --- /dev/null +++ b/tests/core/test_profanity.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +""" +Unit tests for profanity detection functions +""" + +import unittest + +from pythainlp.corpus import thai_profanity_words +from pythainlp.util import ( + censor_profanity, + contains_profanity, + find_profanity, +) + + +class TestProfanity(unittest.TestCase): + def test_thai_profanity_words(self): + """Test loading profanity word list""" + words = thai_profanity_words() + self.assertIsInstance(words, frozenset) + self.assertGreater(len(words), 0) + + def test_contains_profanity_clean_text(self): + """Test clean text without profanity""" + self.assertFalse(contains_profanity("สวัสดีครับ")) + self.assertFalse(contains_profanity("วันนี้อากาศดีมาก")) + self.assertFalse(contains_profanity("")) + + def test_contains_profanity_with_profanity(self): + """Test text containing profanity""" + # Test with actual profanity words from the list + self.assertTrue(contains_profanity("ควย")) + self.assertTrue(contains_profanity("สัส")) + self.assertTrue(contains_profanity("สวัสดี ควย ครับ")) + + def test_find_profanity_clean_text(self): + """Test finding profanity in clean text""" + self.assertEqual(find_profanity("สวัสดีครับ"), []) + self.assertEqual(find_profanity(""), []) + + def test_find_profanity_with_profanity(self): + """Test finding profanity words""" + result = find_profanity("ควย") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + result = find_profanity("สัส") + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + def test_censor_profanity_clean_text(self): + """Test censoring clean text""" + text = "สวัสดีครับ" + self.assertEqual(censor_profanity(text), text) + self.assertEqual(censor_profanity(""), "") + + def test_censor_profanity_with_profanity(self): + """Test censoring profanity words""" + # Test that profanity is replaced with stars + result = censor_profanity("ควย") + self.assertNotEqual(result, "ควย") + self.assertIn("*", result) + + result = censor_profanity("สัส") + self.assertNotEqual(result, "สัส") + self.assertIn("*", result) + + def test_censor_profanity_custom_replacement(self): + """Test censoring with custom replacement character""" + result = censor_profanity("ควย", replacement="#") + self.assertIn("#", result) + self.assertNotIn("*", result) + + def test_contains_profanity_with_custom_words(self): + """Test detection with custom profanity words""" + # Clean text shouldn't be detected + self.assertFalse(contains_profanity("สวัสดีครับ", custom_words={"คำใหม่"})) + + # Custom word should be detected + self.assertTrue(contains_profanity("คำใหม่", custom_words={"คำใหม่"})) + + # Mix of default and custom words + self.assertTrue(contains_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"})) + + def test_find_profanity_with_custom_words(self): + """Test finding profanity with custom words""" + # Should find custom words + result = find_profanity("คำใหม่", custom_words={"คำใหม่"}) + self.assertIn("คำใหม่", result) + + # Should find both default and custom words + result = find_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) + self.assertGreater(len(result), 1) + + def test_censor_profanity_with_custom_words(self): + """Test censoring with custom profanity words""" + # Should censor custom words + result = censor_profanity("คำใหม่", custom_words={"คำใหม่"}) + self.assertNotEqual(result, "คำใหม่") + self.assertIn("*", result) + + # Should censor both default and custom words + result = censor_profanity("ควย และ คำใหม่", custom_words={"คำใหม่"}) + self.assertNotIn("ควย", result) + self.assertNotIn("คำใหม่", result) + self.assertIn("*", result) + + +if __name__ == "__main__": + unittest.main()