Move property back to tokenization

Author: dirkkul

integration/test_tokenize.py

@@ -288,7 +288,9 @@ def test_property_and_generic_endpoints_agree(
         stopwords = config.inverted_index_config.stopwords
 
         text = "the quick brown fox"
-        via_property = recipe_collection.config.tokenize_property(property_name="recipe", text=text)
+        via_property = client.tokenization.for_property(
+            collection=recipe_collection.name, property_name="recipe", text=text
+        )
         via_generic = client.tokenization.text(
             text=text,
             tokenization=recipe.tokenization,
@@ -325,8 +327,9 @@ def test_property_result_shape(self, client: weaviate.WeaviateClient) -> None:
                     ],
                 }
             )
-            col = client.collections.get("TestDeserPropTypes")
-            result = col.config.tokenize_property(property_name="tag", text="  Hello World  ")
+            result = client.tokenization.for_property(
+                collection="TestDeserPropTypes", property_name="tag", text="  Hello World  "
+            )
             assert isinstance(result, TokenizeResult)
             assert result.indexed == ["Hello World"]
         finally:
@@ -442,9 +445,8 @@ def test_text_raises_on_old_server(self, client: weaviate.WeaviateClient) -> Non
     def test_tokenize_property_raises_on_old_server(self, client: weaviate.WeaviateClient) -> None:
         if client._connection._weaviate_version.is_at_least(1, 37, 0):
             pytest.skip("Version gate only applies to Weaviate < 1.37.0")
-        col = client.collections.get("Any")
         with pytest.raises(WeaviateUnsupportedFeatureError):
-            col.config.tokenize_property(property_name="title", text="hello")
+            client.tokenization.for_property(collection="Any", property_name="title", text="hello")
 
 
 # ---------------------------------------------------------------------------
@@ -454,7 +456,7 @@ def test_tokenize_property_raises_on_old_server(self, client: weaviate.WeaviateC
 
 @pytest.mark.usefixtures("require_1_37")
 class TestAsyncClient:
-    """Verify text() and tokenize_property() work through the async client."""
+    """Verify tokenization.text() and tokenization.for_property() work through the async client."""
 
     @pytest.mark.asyncio
     async def test_text_tokenize(self, async_client: weaviate.WeaviateAsyncClient) -> None:
@@ -498,8 +500,8 @@ async def test_property_tokenize(self, async_client: weaviate.WeaviateAsyncClien
                     ],
                 }
             )
-            col = async_client.collections.get("TestAsyncPropTokenize")
-            result = await col.config.tokenize_property(
+            result = await async_client.tokenization.for_property(
+                collection="TestAsyncPropTokenize",
                 property_name="title",
                 text="The quick brown fox",
             )

weaviate/collections/config/async_.pyi

@@ -27,7 +27,6 @@ from weaviate.collections.classes.config import (
 from weaviate.collections.classes.config_object_ttl import _ObjectTTLConfigUpdate
 from weaviate.collections.classes.config_vector_index import _VectorIndexConfigDynamicUpdate
 from weaviate.connect.v4 import ConnectionAsync
-from weaviate.tokenization.models import TokenizeResult
 
 from .executor import _ConfigCollectionExecutor
 
@@ -91,4 +90,3 @@ class _ConfigCollectionAsync(_ConfigCollectionExecutor[ConnectionAsync]):
         self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]]
     ) -> None: ...
     async def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ...
-    async def tokenize_property(self, property_name: str, text: str) -> TokenizeResult: ...

weaviate/collections/config/executor.py

@@ -56,7 +56,6 @@
     WeaviateInvalidInputError,
     WeaviateUnsupportedFeatureError,
 )
-from weaviate.tokenization.models import TokenizeResult
 from weaviate.util import (
     _capitalize_first_letter,
     _decode_json_response_dict,
@@ -667,42 +666,3 @@ def resp(res: Response) -> bool:
             error_msg="Property may not exist",
             status_codes=_ExpectedStatusCodes(ok_in=[200], error="property exists"),
         )
-
-    def tokenize_property(
-        self,
-        property_name: str,
-        text: str,
-    ) -> executor.Result[TokenizeResult]:
-        """Tokenize text using a property's configured tokenization settings.
-
-        Args:
-            property_name: The property name whose tokenization config to use.
-            text: The text to tokenize.
-
-        Returns:
-            A TokenizeResult with indexed and query token lists.
-
-        Raises:
-            WeaviateUnsupportedFeatureError: If the server version is below 1.37.0.
-        """
-        if self._connection._weaviate_version.is_lower_than(1, 37, 0):
-            raise WeaviateUnsupportedFeatureError(
-                "Tokenization",
-                str(self._connection._weaviate_version),
-                "1.37.0",
-            )
-
-        path = f"/schema/{self._name}/properties/{property_name}/tokenize"
-        payload: Dict[str, Any] = {"text": text}
-
-        def resp(response: Response) -> TokenizeResult:
-            return TokenizeResult.model_validate(response.json())
-
-        return executor.execute(
-            response_callback=resp,
-            method=self._connection.post,
-            path=path,
-            weaviate_object=payload,
-            error_msg="Property tokenization failed",
-            status_codes=_ExpectedStatusCodes(ok_in=[200], error="tokenize property text"),
-        )

weaviate/collections/config/sync.pyi

@@ -27,7 +27,6 @@ from weaviate.collections.classes.config import (
 from weaviate.collections.classes.config_object_ttl import _ObjectTTLConfigUpdate
 from weaviate.collections.classes.config_vector_index import _VectorIndexConfigDynamicUpdate
 from weaviate.connect.v4 import ConnectionSync
-from weaviate.tokenization.models import TokenizeResult
 
 from .executor import _ConfigCollectionExecutor
 
@@ -89,4 +88,3 @@ class _ConfigCollection(_ConfigCollectionExecutor[ConnectionSync]):
         self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]]
     ) -> None: ...
     def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ...
-    def tokenize_property(self, property_name: str, text: str) -> TokenizeResult: ...

weaviate/tokenization/async_.pyi

@@ -30,3 +30,6 @@ class _TokenizationAsync(_TokenizationExecutor[ConnectionAsync]):
         analyzer_config: Optional[TextAnalyzerConfigCreate] = ...,
         stopword_presets: Optional[Dict[str, List[str]]] = ...,
     ) -> TokenizeResult: ...
+    async def for_property(
+        self, collection: str, property_name: str, text: str
+    ) -> TokenizeResult: ...

weaviate/tokenization/executor.py

@@ -14,6 +14,7 @@
 from weaviate.connect.v4 import ConnectionType, _ExpectedStatusCodes
 from weaviate.exceptions import WeaviateUnsupportedFeatureError
 from weaviate.tokenization.models import TokenizeResult
+from weaviate.util import _capitalize_first_letter
 
 
 class _TokenizationExecutor(Generic[ConnectionType]):
@@ -189,3 +190,42 @@ def resp(response: Response) -> TokenizeResult:
             error_msg="Tokenization failed",
             status_codes=_ExpectedStatusCodes(ok_in=[200], error="tokenize text"),
         )
+
+    def for_property(
+        self,
+        collection: str,
+        property_name: str,
+        text: str,
+    ) -> executor.Result[TokenizeResult]:
+        """Tokenize text using a property's configured tokenization settings.
+
+        The server resolves the tokenization and analyzer configuration from
+        the property's schema, so callers only supply the text.
+
+        Args:
+            collection: The collection that owns the property.
+            property_name: The property name whose tokenization config to use.
+            text: The text to tokenize.
+
+        Returns:
+            A TokenizeResult with indexed and query token lists.
+
+        Raises:
+            WeaviateUnsupportedFeatureError: If the server version is below 1.37.0.
+        """
+        self.__check_version()
+
+        path = f"/schema/{_capitalize_first_letter(collection)}/properties/{property_name}/tokenize"
+        payload: Dict[str, Any] = {"text": text}
+
+        def resp(response: Response) -> TokenizeResult:
+            return TokenizeResult.model_validate(response.json())
+
+        return executor.execute(
+            response_callback=resp,
+            method=self._connection.post,
+            path=path,
+            weaviate_object=payload,
+            error_msg="Property tokenization failed",
+            status_codes=_ExpectedStatusCodes(ok_in=[200], error="tokenize property text"),
+        )

weaviate/tokenization/sync.pyi

@@ -30,3 +30,4 @@ class _Tokenization(_TokenizationExecutor[ConnectionSync]):
         analyzer_config: Optional[TextAnalyzerConfigCreate] = ...,
         stopword_presets: Optional[Dict[str, List[str]]] = ...,
     ) -> TokenizeResult: ...
+    def for_property(self, collection: str, property_name: str, text: str) -> TokenizeResult: ...