fix: keep parser for accurate sentence boundaries

Per review feedback, removing parser and using sentencizer causes
sentence splitting regressions. Keep parser loaded, only exclude
ner, lemmatizer, and attribute_ruler.

Author: KRRT7

CHANGELOG.md

@@ -29,16 +29,13 @@
 ### Enhancements
 - **`partition_md` Markdown `extensions`**: Optional `extensions` list is passed to `markdown.markdown()`; entries may be registered names (`str`) or `markdown.extensions.Extension` instances. Defaults to `["tables", "fenced_code"]`. Invalid values raise `ValueError`.
 
-<<<<<<< HEAD
 ## 0.22.2
 
 ### Enhancements
 - Store routing in ElementMetadata
 
 ## 0.22.1
 
-=======
->>>>>>> 2291c234 (Bump version to 0.22.3 for changelog CI check)
 ### Fixes
 - **Security update**: Bumped dependencies to address security vulnerabilities
 

unstructured/nlp/tokenize.py

@@ -109,21 +109,15 @@ def _install_spacy_model() -> None:
     logger.info("Installed %s %s", _SPACY_MODEL_NAME, _SPACY_MODEL_VERSION)
 
 
-# Only tok2vec, tagger, and sentence splitting are used (pos_tag and sent_tokenize).
-# Excluding the remaining components saves ~12 MiB of model weights per process.
-_SPACY_EXCLUDE = ["ner", "parser", "lemmatizer", "attribute_ruler"]
-
-
-def _load_and_configure() -> spacy.language.Language:
-    """Load the model excluding unused components, add a lightweight sentencizer."""
-    nlp = spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
-    nlp.add_pipe("sentencizer")
-    return nlp
+# Only tok2vec, tagger, parser (sentence boundaries), and sentencizer are used
+# (pos_tag and sent_tokenize). Excluding the remaining components saves ~7 MiB
+# of model weights per process.
+_SPACY_EXCLUDE = ["ner", "lemmatizer", "attribute_ruler"]
 
 
 def _load_spacy_model() -> spacy.language.Language:
     try:
-        return _load_and_configure()
+        return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
     except OSError:
         pass
 
@@ -134,13 +128,13 @@ def _load_spacy_model() -> spacy.language.Language:
         # Double-check: another process may have installed while we waited.
         importlib.invalidate_caches()
         try:
-            return _load_and_configure()
+            return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
         except OSError:
             pass
         _install_spacy_model()
         importlib.invalidate_caches()
         try:
-            return _load_and_configure()
+            return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
         except OSError as exc:
             raise RuntimeError(
                 f"Installed {_SPACY_MODEL_NAME} but spacy.load() still failed. "