|
5 | 5 | Example: Using PyThaiNLP in PySpark Distributed Environment |
6 | 6 |
|
7 | 7 | This example demonstrates how to use PyThaiNLP in a distributed environment |
8 | | -like Apache Spark. The key is to set the PYTHAINLP_DATA_DIR environment |
| 8 | +like Apache Spark. The key is to set the PYTHAINLP_DATA environment |
9 | 9 | variable inside the function that will be distributed to executor nodes. |
10 | 10 |
|
11 | 11 | PyThaiNLP's core tokenization engines are thread-safe, making them suitable |
@@ -45,9 +45,9 @@ def tokenize_thai(text): |
45 | 45 | """ |
46 | 46 | import os |
47 | 47 |
|
48 | | - # Set PYTHAINLP_DATA_DIR before importing pythainlp |
| 48 | + # Set PYTHAINLP_DATA before importing pythainlp |
49 | 49 | # Use './pythainlp-data' to store data in current working directory |
50 | | - os.environ["PYTHAINLP_DATA_DIR"] = "./pythainlp-data" |
| 50 | + os.environ["PYTHAINLP_DATA"] = "./pythainlp-data" |
51 | 51 |
|
52 | 52 | # Now import pythainlp modules |
53 | 53 | from pythainlp.tokenize import word_tokenize |
@@ -96,7 +96,7 @@ def tokenize_udf(text): |
96 | 96 | """ |
97 | 97 | import os |
98 | 98 |
|
99 | | - os.environ["PYTHAINLP_DATA_DIR"] = "./pythainlp-data" |
| 99 | + os.environ["PYTHAINLP_DATA"] = "./pythainlp-data" |
100 | 100 |
|
101 | 101 | from pythainlp.tokenize import word_tokenize |
102 | 102 |
|
@@ -134,7 +134,7 @@ def process_thai_text(text): |
134 | 134 | import os |
135 | 135 |
|
136 | 136 | # Configure data directory |
137 | | - os.environ["PYTHAINLP_DATA_DIR"] = "./pythainlp-data" |
| 137 | + os.environ["PYTHAINLP_DATA"] = "./pythainlp-data" |
138 | 138 |
|
139 | 139 | # Import required modules |
140 | 140 | from pythainlp.tag import pos_tag |
@@ -197,7 +197,7 @@ def safe_tokenize(text): |
197 | 197 |
|
198 | 198 | try: |
199 | 199 | # Set environment variables |
200 | | - os.environ["PYTHAINLP_DATA_DIR"] = "./pythainlp-data" |
| 200 | + os.environ["PYTHAINLP_DATA"] = "./pythainlp-data" |
201 | 201 |
|
202 | 202 | # Import modules |
203 | 203 | from pythainlp.tokenize import word_tokenize |
|
0 commit comments