Drop nltk

.pre-commit-config.yaml

@@ -13,5 +13,5 @@ repos:
         name: Check spelling
         args:
           - --write-changes
-          - --skip=*.pyc,*.pyo,*.lock,*.git,*.mypy_cache,__pycache__,*.egg-info,.pytest_cache,docs/_build,env,venv,.venv
+          - --skip=*.pyc,*.pyo,*.lock,*.git,*.mypy_cache,__pycache__,*.egg-info,.pytest_cache,docs/_build,env,venv,.venv,redisvl/utils/stopwords.json
           - --ignore-words-list=enginee

pyproject.toml

@@ -40,7 +40,6 @@ mcp = [
 ]
 mistralai = ["mistralai>=1.0.0,<2"]
 openai = ["openai>=1.1.0"]
-nltk = ["nltk>=3.8.1,<4"]
 cohere = ["cohere>=4.44"]
 voyageai = ["voyageai>=0.2.2"]
 sentence-transformers = ["sentence-transformers>=3.4.0,<4"]
@@ -62,7 +61,6 @@ sql-redis = [
 all = [
     "mistralai>=1.0.0,<2",
     "openai>=1.1.0",
-    "nltk>=3.8.1,<4",
     "cohere>=4.44",
     "voyageai>=0.2.2",
     "sentence-transformers>=3.4.0,<4",

redisvl/query/aggregate.py

@@ -8,10 +8,6 @@
 from redisvl.redis.utils import array_to_buffer
 from redisvl.schema.fields import VectorDataType
 from redisvl.utils.full_text_query_helper import FullTextQueryHelper
-from redisvl.utils.utils import lazy_import
-
-nltk = lazy_import("nltk")
-nltk_stopwords = lazy_import("nltk.corpus.stopwords")
 
 
 class Vector(BaseModel):

redisvl/query/query.py

@@ -6,14 +6,12 @@
 from redisvl.query.filter import FilterExpression
 from redisvl.redis.utils import array_to_buffer
 from redisvl.utils.log import get_logger
+from redisvl.utils.stopwords import get_stopwords
 from redisvl.utils.token_escaper import TokenEscaper
-from redisvl.utils.utils import denorm_cosine_distance, lazy_import
+from redisvl.utils.utils import denorm_cosine_distance
 
 logger = get_logger(__name__)
 
-nltk = lazy_import("nltk")
-nltk_stopwords = lazy_import("nltk.corpus.stopwords")
-
 # Type alias for sort specification
 # Can be:
 # - str: single field name (ASC by default)
@@ -1406,21 +1404,7 @@ def _set_stopwords(self, stopwords: str | set[str] | None = "english"):
         if not stopwords:
             self._stopwords = set()
         elif isinstance(stopwords, str):
-            try:
-                # Try loading first; only download if not already present.
-                # This avoids race conditions when parallel workers (e.g.
-                # pytest-xdist) call nltk.download() concurrently.
-                try:
-                    self._stopwords = set(nltk_stopwords.words(stopwords))
-                except LookupError:
-                    nltk.download("stopwords", quiet=True)
-                    self._stopwords = set(nltk_stopwords.words(stopwords))
-            except ImportError:
-                raise ValueError(
-                    f"Loading stopwords for {stopwords} failed: nltk is not installed."
-                )
-            except Exception as e:
-                raise ValueError(f"Error trying to load {stopwords} from nltk. {e}")
+            self._stopwords = get_stopwords(stopwords)
         elif isinstance(stopwords, (set, list, tuple)) and all(
             isinstance(word, str) for word in stopwords
         ):

redisvl/utils/full_text_query_helper.py

@@ -1,9 +1,6 @@
 from redisvl.query.filter import FilterExpression
+from redisvl.utils.stopwords import get_stopwords
 from redisvl.utils.token_escaper import TokenEscaper
-from redisvl.utils.utils import lazy_import
-
-nltk = lazy_import("nltk")
-nltk_stopwords = lazy_import("nltk.corpus.stopwords")
 
 
 def _parse_text_weights(weights: dict[str, float] | None) -> dict[str, float]:
@@ -88,21 +85,7 @@ def _get_stopwords(self, stopwords: str | set[str] | None = "english") -> set[st
         if not stopwords:
             return set()
         elif isinstance(stopwords, str):
-            try:
-                # Try loading first; only download if not already present.
-                # This avoids race conditions when parallel workers (e.g.
-                # pytest-xdist) call nltk.download() concurrently.
-                try:
-                    return set(nltk_stopwords.words(stopwords))
-                except LookupError:
-                    nltk.download("stopwords", quiet=True)
-                    return set(nltk_stopwords.words(stopwords))
-            except ImportError:
-                raise ValueError(
-                    f"Loading stopwords for {stopwords} failed: nltk is not installed."
-                )
-            except Exception as e:
-                raise ValueError(f"Error trying to load {stopwords} from nltk. {e}")
+            return get_stopwords(stopwords)
         elif isinstance(stopwords, (set, list, tuple)) and all(
             isinstance(word, str) for word in stopwords
         ):