fix: separate loading configuration handling for jieba and lindera (#3932)

KazuhitoT · web-flow · commit b9363296511d · 2025-06-18T21:22:24.000-07:00
This is a pull request for #3891, which implements the following changes: - Changed the Lindera configuration file format to YAML (see: https://github.com/lindera/lindera?tab=readme-ov-file#configuration-file). - Replaced all lindera `config.json` files with `config.yml`. - Updated the documentation to reflect the new YAML-based configuration. - Introduced the `LINDERA_CONFIG_PATH` environment variable to allow specifying custom config file. Please let me know if backward compatibility needs to be preserved.
diff --git a/docs/tokenizer.rst b/docs/tokenizer.rst
@@ -70,18 +70,17 @@ Using the Model
 User Dictionaries
 ~~~~~~~~~~~~~~~~~
 
-Create a file named config.json in the root directory of the current model.
+Create a file named config.yml in the root directory of your model, or specify a custom YAML file using the `LINDERA_CONFIG_PATH` environment variable.
+If both are provided, the config.yml in the root directory will be used.
+For more detailed configuration methods, see the lindera documentation at https://github.com/lindera/lindera/.
 
-.. code-block::json
-    {
-        "main": "main",
-        "users": "path/to/user/dict.bin",
-        "user_kind": "ipadic|ko-dic|unidic"
-    }
+.. code-block:: yaml
 
-- The "main" field is optional. If not filled, the default is the "main" directory.
-- "user" is the path of the user dictionary. The user dictionary can be passed as a CSV file or as a binary file compiled by lindera-cli.
-- The "user_kind" field can be left blank if the user dictionary is in binary format. If it's in CSV format, you need to specify the type of the language model.
+    segmenter:
+        mode: "normal"
+        dictionary:
+            # Note: in lance, the `kind` field is not supported. You need to specify the model path using the `path` field instead.
+            path: /path/to/lindera/ipadic/main
 
 
 Create your own language model
diff --git a/python/python/tests/models/lindera/invalid_dict/config.json b/python/python/tests/models/lindera/invalid_dict/config.json
diff --git a/python/python/tests/models/lindera/invalid_dict/config.yml b/python/python/tests/models/lindera/invalid_dict/config.yml
@@ -0,0 +1,7 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
+  user_dictionary:
+    path: "./python/tests/models/lindera/invalid_dict/invalid.bin"
+    kind: "ipadic"
diff --git a/python/python/tests/models/lindera/invalid_dict2/config.json b/python/python/tests/models/lindera/invalid_dict2/config.json
diff --git a/python/python/tests/models/lindera/invalid_dict2/config.yml b/python/python/tests/models/lindera/invalid_dict2/config.yml
@@ -0,0 +1,7 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
+  user_dictionary:
+    path: "./python/tests/models/lindera/invalid_dict2/ipadic_simple_userdic.bin"
+    kind: "ipadic"
diff --git a/python/python/tests/models/lindera/ipadic/config.yml b/python/python/tests/models/lindera/ipadic/config.yml
@@ -0,0 +1,4 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
diff --git a/python/python/tests/models/lindera/load_config_fallback/config_env.yml b/python/python/tests/models/lindera/load_config_fallback/config_env.yml
@@ -0,0 +1,4 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
diff --git a/python/python/tests/models/lindera/load_config_priority/config.yml b/python/python/tests/models/lindera/load_config_priority/config.yml
@@ -0,0 +1,10 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
+
+character_filters:
+  - kind: mapping
+    args:
+       mapping:
+          成田: ほげほげ
diff --git a/python/python/tests/models/lindera/load_config_priority/config_env.yml b/python/python/tests/models/lindera/load_config_priority/config_env.yml
@@ -0,0 +1,4 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
diff --git a/python/python/tests/models/lindera/user_dict/config.json b/python/python/tests/models/lindera/user_dict/config.json
diff --git a/python/python/tests/models/lindera/user_dict/config.yml b/python/python/tests/models/lindera/user_dict/config.yml
@@ -0,0 +1,7 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
+  user_dictionary:
+    path: "./python/tests/models/lindera/user_dict/userdic.csv"
+    kind: "ipadic"
diff --git a/python/python/tests/models/lindera/user_dict2/config.json b/python/python/tests/models/lindera/user_dict2/config.json
diff --git a/python/python/tests/models/lindera/user_dict2/config.yml b/python/python/tests/models/lindera/user_dict2/config.yml
@@ -0,0 +1,7 @@
+segmenter:
+  mode: "normal"
+  dictionary:
+    path: "./python/tests/models/lindera/ipadic/main"
+  user_dictionary:
+    path: "./python/tests/models/lindera/user_dict2/userdic.bin"
+    kind: "ipadic"
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
@@ -892,12 +892,83 @@ def test_index_after_merge_insert(tmp_path):
     assert dataset.to_table(filter="payload >= 30").num_rows == 20
 
 
+def test_lindera_load_config_fallback(tmp_path, lindera_ipadic, monkeypatch):
+    data = pa.table(
+        {
+            "text": [
+                "成田国際空港",
+                "東京国際空港",
+                "羽田空港",
+            ],
+        }
+    )
+    ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+    with pytest.raises(OSError):
+        ds.create_scalar_index(
+            "text", "INVERTED", base_tokenizer="lindera/load_config_fallback"
+        )
+
+    config_path = os.path.join(
+        os.path.dirname(__file__),
+        "models/lindera/load_config_fallback/config_not_exists.yml",
+    )
+    monkeypatch.setenv("LINDERA_CONFIG_PATH", config_path)
+    with pytest.raises(OSError):
+        ds.create_scalar_index(
+            "text", "INVERTED", base_tokenizer="lindera/load_config_fallback"
+        )
+
+    config_path = os.path.join(
+        os.path.dirname(__file__), "models/lindera/load_config_fallback/config_env.yml"
+    )
+    monkeypatch.setenv("LINDERA_CONFIG_PATH", config_path)
+    ds.create_scalar_index(
+        "text", "INVERTED", base_tokenizer="lindera/load_config_fallback"
+    )
+    results = ds.to_table(
+        full_text_query="成田",
+        prefilter=True,
+        with_row_id=True,
+    )
+    assert results["_rowid"].to_pylist() == [0]
+
+
+def test_lindera_load_config_priority(tmp_path, lindera_ipadic, monkeypatch):
+    data = pa.table(
+        {
+            "text": [
+                "成田国際空港",
+                "東京国際空港",
+                "羽田空港",
+            ],
+        }
+    )
+    config_path = os.path.join(
+        os.path.dirname(__file__), "models/lindera/load_config_priority/config_env.yml"
+    )
+    monkeypatch.setenv("LINDERA_CONFIG_PATH", config_path)
+    ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+    ds.create_scalar_index(
+        "text", "INVERTED", base_tokenizer="lindera/load_config_priority"
+    )
+    results = ds.to_table(
+        full_text_query="成田",
+        prefilter=True,
+        with_row_id=True,
+    )
+    assert results["_rowid"].to_pylist() == [0]
+
+    results = ds.to_table(
+        full_text_query="ほげほげ",
+        prefilter=True,
+        with_row_id=True,
+    )
+    assert results["_rowid"].to_pylist() == [0]
+
+
 def test_indexed_filter_with_fts_index_with_lindera_ipadic_jp_tokenizer(
     tmp_path, lindera_ipadic
 ):
-    os.environ["LANCE_LANGUAGE_MODEL_HOME"] = os.path.join(
-        os.path.dirname(__file__), "models"
-    )
     data = pa.table(
         {
             "text": [
@@ -986,6 +1057,18 @@ def test_lindera_ipadic_jp_tokenizer_bin_user_dict(tmp_path, lindera_ipadic):
     )
     ds = lance.write_dataset(data, tmp_path, mode="overwrite")
     ds.create_scalar_index("text", "INVERTED", base_tokenizer="lindera/user_dict2")
+    results = ds.to_table(
+        full_text_query="成田",
+        prefilter=True,
+        with_row_id=True,
+    )
+    assert len(results) == 0
+    results = ds.to_table(
+        full_text_query="成田国際空港",
+        prefilter=True,
+        with_row_id=True,
+    )
+    assert results["_rowid"].to_pylist() == [0]
 
 
 def test_jieba_tokenizer(tmp_path):
diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml
@@ -79,9 +79,8 @@ rstest.workspace = true
 
 [features]
 protoc = ["dep:protobuf-src"]
-tokenizer-lindera = ["lindera", "lindera-tantivy", "tokenizer-common"]
-tokenizer-jieba = ["jieba-rs", "tokenizer-common"]
-tokenizer-common = []
+tokenizer-lindera = ["lindera", "lindera-tantivy"]
+tokenizer-jieba = ["jieba-rs"]
 
 [build-dependencies]
 prost-build.workspace = true
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -7,11 +7,17 @@ use lance_core::{Error, Result};
 use serde::{Deserialize, Serialize};
 use snafu::location;
 
+#[cfg(feature = "tokenizer-jieba")]
+mod jieba;
+
 #[cfg(feature = "tokenizer-lindera")]
 mod lindera;
 
 #[cfg(feature = "tokenizer-jieba")]
-mod jieba;
+use jieba::JiebaTokenizerBuilder;
+
+#[cfg(feature = "tokenizer-lindera")]
+use lindera::LinderaTokenizerBuilder;
 
 /// Tokenizer configs
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -204,38 +210,9 @@ pub const LANCE_LANGUAGE_MODEL_HOME_ENV_KEY: &str = "LANCE_LANGUAGE_MODEL_HOME";
 
 pub const LANCE_LANGUAGE_MODEL_DEFAULT_DIRECTORY: &str = "lance/language_models";
 
-pub const LANCE_LANGUAGE_MODEL_CONFIG_FILE: &str = "config.json";
-
 pub fn language_model_home() -> Option<PathBuf> {
     match env::var(LANCE_LANGUAGE_MODEL_HOME_ENV_KEY) {
         Ok(p) => Some(PathBuf::from(p)),
         Err(_) => dirs::data_local_dir().map(|p| p.join(LANCE_LANGUAGE_MODEL_DEFAULT_DIRECTORY)),
     }
 }
-
-#[cfg(feature = "tokenizer-common")]
-trait TokenizerBuilder: Sized {
-    type Config: serde::de::DeserializeOwned + Default;
-    fn load(p: &std::path::Path) -> Result<Self> {
-        if !p.is_dir() {
-            return Err(Error::io(
-                format!("{} is not a valid directory", p.display()),
-                location!(),
-            ));
-        }
-        use std::{fs::File, io::BufReader};
-        let config_path = p.join(LANCE_LANGUAGE_MODEL_CONFIG_FILE);
-        let config = if config_path.exists() {
-            let file = File::open(config_path)?;
-            let reader = BufReader::new(file);
-            serde_json::from_reader::<BufReader<File>, Self::Config>(reader)?
-        } else {
-            Self::Config::default()
-        };
-        Self::new(config, p)
-    }
-
-    fn new(config: Self::Config, root: &std::path::Path) -> Result<Self>;
-
-    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder>;
-}
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs b/rust/lance-index/src/scalar/inverted/tokenizer/jieba.rs
@@ -1,11 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::path::{Path, PathBuf};
+use std::{fs::File, io::BufReader, path::Path, path::PathBuf};
 
-use super::TokenizerBuilder;
 use lance_core::{Error, Result};
-use serde::{Deserialize, Serialize};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use snafu::location;
 
 #[derive(Serialize, Deserialize, Default)]
@@ -14,6 +13,34 @@ pub struct JiebaConfig {
     users: Option<Vec<String>>,
 }
 
+pub const JIEBA_LANGUAGE_MODEL_CONFIG_FILE: &str = "config.json";
+
+pub trait JiebaTokenizerBuilder: Sized {
+    type Config: DeserializeOwned + Default;
+
+    fn load(p: &Path) -> Result<Self> {
+        if !p.is_dir() {
+            return Err(Error::io(
+                format!("{} is not a valid directory", p.display()),
+                snafu::location!(),
+            ));
+        }
+        let config_path = p.join(JIEBA_LANGUAGE_MODEL_CONFIG_FILE);
+        let config = if config_path.exists() {
+            let file = File::open(config_path)?;
+            let reader = BufReader::new(file);
+            serde_json::from_reader::<BufReader<File>, Self::Config>(reader)?
+        } else {
+            Self::Config::default()
+        };
+        Self::new(config, p)
+    }
+
+    fn new(config: Self::Config, root: &Path) -> Result<Self>;
+
+    fn build(&self) -> Result<tantivy::tokenizer::TextAnalyzerBuilder>;
+}
+
 pub struct JiebaBuilder {
     root: PathBuf,
     config: JiebaConfig,
@@ -35,7 +62,7 @@ impl JiebaBuilder {
     }
 }
 
-impl TokenizerBuilder for JiebaBuilder {
+impl JiebaTokenizerBuilder for JiebaBuilder {
     type Config = JiebaConfig;
 
     fn new(config: Self::Config, root: &Path) -> Result<Self> {
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs b/rust/lance-index/src/scalar/inverted/tokenizer/lindera.rs
