feat: support per-property tokenization config

Author: bevzzz

src/main/java/io/weaviate/client6/v1/api/collections/Property.java

@@ -16,6 +16,7 @@ public record Property(
     @SerializedName("indexFilterable") Boolean indexFilterable,
     @SerializedName("indexRangeFilters") Boolean indexRangeFilters,
     @SerializedName("indexSearchable") Boolean indexSearchable,
+    @SerializedName("tokenization") Tokenization tokenization,
     @SerializedName("skipVectorization") Boolean skipVectorization,
     @SerializedName("vectorizePropertyName") Boolean vectorizePropertyName) {
 
@@ -142,6 +143,7 @@ public Builder edit() {
         .indexFilterable(indexFilterable)
         .indexRangeFilters(indexRangeFilters)
         .indexSearchable(indexSearchable)
+        .tokenization(tokenization)
         .skipVectorization(skipVectorization)
         .vectorizePropertyName(vectorizePropertyName);
   }
@@ -159,6 +161,7 @@ public Property(Builder builder) {
         builder.indexFilterable,
         builder.indexRangeFilters,
         builder.indexSearchable,
+        builder.tokenization,
         builder.skipVectorization,
         builder.vectorizePropertyName);
   }
@@ -174,6 +177,7 @@ public static class Builder implements ObjectBuilder<Property> {
     private Boolean indexFilterable;
     private Boolean indexRangeFilters;
     private Boolean indexSearchable;
+    private Tokenization tokenization;
     private Boolean skipVectorization;
     private Boolean vectorizePropertyName;
 
@@ -221,6 +225,11 @@ public Builder indexSearchable(Boolean indexSearchable) {
       return this;
     }
 
+    public Builder tokenization(Tokenization tokenization) {
+      this.tokenization = tokenization;
+      return this;
+    }
+
     public Builder skipVectorization(Boolean skipVectorization) {
       this.skipVectorization = skipVectorization;
       return this;

src/main/java/io/weaviate/client6/v1/api/collections/Tokenization.java

@@ -0,0 +1,22 @@
+package io.weaviate.client6.v1.api.collections;
+
+import com.google.gson.annotations.SerializedName;
+
+public enum Tokenization {
+  @SerializedName("word")
+  WORD,
+  @SerializedName("whitespace")
+  WHITESPACE,
+  @SerializedName("lowercase")
+  LOWERCASE,
+  @SerializedName("field")
+  FIELD,
+  @SerializedName("gse")
+  GSE,
+  @SerializedName("trigram")
+  TRIGRAM,
+  @SerializedName("kagome_ja")
+  KAGOME_JA,
+  @SerializedName("kagome_kr")
+  KAGOME_KR;
+}

src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java

@@ -19,6 +19,7 @@
 import io.weaviate.client6.v1.api.collections.ObjectMetadata;
 import io.weaviate.client6.v1.api.collections.Property;
 import io.weaviate.client6.v1.api.collections.Reranker;
+import io.weaviate.client6.v1.api.collections.Tokenization;
 import io.weaviate.client6.v1.api.collections.Vectorizer;
 import io.weaviate.client6.v1.api.collections.Vectorizers;
 import io.weaviate.client6.v1.api.collections.Vectors;
@@ -224,6 +225,7 @@ public static Object[][] testCases() {
                 .description("A collection of things")
                 .properties(
                     Property.text("shape"),
+                    Property.text("custom_id", p -> p.tokenization(Tokenization.WORD)),
                     Property.integer("size"))
                 .references(
                     Property.reference("owner", "Person", "Company"))
@@ -237,6 +239,7 @@ public static Object[][] testCases() {
                   "properties": [
                     {"name": "shape", "dataType": ["text"]},
                     {"name": "size", "dataType": ["int"]},
+                    {"name": "custom_id", "dataType": ["text"], tokenization: "word"},
                     {"name": "owner", "dataType": ["Person", "Company"]}
                   ],
                   "vectorConfig": {