tests: eliminate NLTK stopwords download race under pytest-xdist

pyproject.toml

@@ -105,6 +105,7 @@ dev = [
     "types-pyyaml",
     "types-pyopenssl",
     "testcontainers>=4.3.1,<5",
+    "filelock>=3.12,<4",
     "cryptography>=44.0.1",
     "codespell>=2.4.1,<3",
     "langcache>=0.9.0",

tests/conftest.py

@@ -53,6 +53,43 @@ def set_tokenizers_parallelism():
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 
+@pytest.fixture(scope="session", autouse=True)
+def ensure_nltk_stopwords(tmp_path_factory, worker_id):
+    """Pre-download the NLTK ``stopwords`` corpus once, before tests run.
+
+    Query classes that take a string ``stopwords`` value ("english" by
+    default) lazily download the corpus on first use. Under pytest-xdist
+    several workers can hit the missing corpus simultaneously and call
+    ``nltk.download()`` concurrently, corrupting the shared ``nltk_data``
+    directory and producing intermittent "stopwords/english not found"
+    failures. Fetch it here once, serializing across workers with a file lock
+    so exactly one process performs the download.
+    """
+    try:
+        import nltk
+    except ImportError:
+        return
+
+    def _ensure() -> None:
+        try:
+            nltk.data.find("corpora/stopwords")
+        except LookupError:
+            nltk.download("stopwords", quiet=True)
+
+    # Not running under xdist: download in-process.
+    if worker_id == "master":
+        _ensure()
+        return
+
+    # Under xdist: the first worker to grab the lock downloads; the others
+    # wait, then find the corpus already present.
+    from filelock import FileLock
+
+    root_tmp = tmp_path_factory.getbasetemp().parent
+    with FileLock(str(root_tmp / "nltk_stopwords.lock")):
+        _ensure()
+
+
 @pytest.fixture(scope="session", autouse=True)
 def redis_container(worker_id):
     """

tests/integration/test_aggregation.py

@@ -96,6 +96,7 @@ def test_hybrid_query(index):
         vector=vector,
         vector_field_name=vector_field,
         return_fields=return_fields,
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -122,6 +123,7 @@ def test_hybrid_query(index):
         vector=vector,
         vector_field_name=vector_field,
         num_results=3,
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -177,6 +179,7 @@ def test_hybrid_query_with_filter(index):
         vector_field_name=vector_field,
         filter_expression=filter_expression,
         return_fields=return_fields,
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -203,6 +206,7 @@ def test_hybrid_query_with_geo_filter(index):
         vector_field_name=vector_field,
         filter_expression=filter_expression,
         return_fields=return_fields,
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -226,6 +230,7 @@ def test_hybrid_query_alpha(index, alpha):
         vector=vector,
         vector_field_name=vector_field,
         alpha=alpha,
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -291,6 +296,7 @@ def test_hybrid_query_with_text_filter(index):
         alpha=0.5,
         filter_expression=filter_expression,
         return_fields=["job", "description"],
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -309,6 +315,7 @@ def test_hybrid_query_with_text_filter(index):
         alpha=0.5,
         filter_expression=filter_expression,
         return_fields=["description"],
+        stopwords=None,
     )
 
     results = index.query(hybrid_query)
@@ -339,6 +346,7 @@ def test_hybrid_query_word_weights(index, scorer):
         return_fields=return_fields,
         text_scorer=scorer,
         text_weights=weights,
+        stopwords=None,
     )
 
     weighted_results = index.query(weighted_query)
@@ -353,6 +361,7 @@ def test_hybrid_query_word_weights(index, scorer):
         return_fields=return_fields,
         text_scorer=scorer,
         text_weights={},
+        stopwords=None,
     )
 
     unweighted_results = index.query(unweighted_query)
@@ -372,6 +381,7 @@ def test_hybrid_query_word_weights(index, scorer):
         return_fields=return_fields,
         text_scorer=scorer,
         text_weights=weights,
+        stopwords=None,
     )
 
     weighted_results = index.query(weighted_query)
@@ -386,6 +396,7 @@ def test_hybrid_query_word_weights(index, scorer):
         return_fields=return_fields,
         text_scorer=scorer,
         text_weights=None,
+        stopwords=None,
     )
 
     new_query.set_text_weights(weights)