fix

Author: lxy-9602

CMakeLists.txt

@@ -303,7 +303,7 @@ if(PAIMON_ENABLE_LUCENE)
             PATTERN "user.dict.utf8"
             PATTERN "pos_dict"
             PATTERN ".git*" EXCLUDE
-            PATTERN "*.md"  EXCLUDE)
+            PATTERN "*.md" EXCLUDE)
 endif()
 
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/

cmake_modules/ThirdpartyToolchain.cmake

@@ -352,12 +352,10 @@ macro(build_jieba)
     file(MAKE_DIRECTORY ${JIEBA_DICT_DIR})
 
     set(JIEBA_CMAKE_ARGS
-        ${EP_COMMON_CMAKE_ARGS}
-        "-DENABLE_TEST=OFF"
-        "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF"
+        ${EP_COMMON_CMAKE_ARGS} "-DENABLE_TEST=OFF" "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF"
         "-DCMAKE_INSTALL_PREFIX=${JIEBA_INSTALL}")
 
-    set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff")    
+    set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff")
     externalproject_add(jieba_ep
                         ${EP_COMMON_OPTIONS}
                         GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git
@@ -370,13 +368,14 @@ macro(build_jieba)
                         PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> bash -c
                                       "[ -f .patched ] && echo '<SOURCE_DIR> patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched"
                         INSTALL_COMMAND bash -c
-                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/"
+                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/"
                         )
 
     # The include directory must exist before it is referenced by a target.
     include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR})
     add_library(jieba INTERFACE IMPORTED)
-    target_include_directories(jieba SYSTEM INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}")
+    target_include_directories(jieba SYSTEM
+                               INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}")
     add_dependencies(jieba jieba_ep)
 endmacro()
 

cmake_modules/jieba.diff

@@ -10,7 +10,7 @@ index 24b2c40..c7c6a94 100644
 +  const std::unordered_set<std::string>& GetStopWords() const {
 +    return stopWords_;
 +  }
-+    
++
   private:
    void LoadIdfDict(const std::string& idfPath) {
      std::ifstream ifs(idfPath.c_str());

src/paimon/global_index/lucene/CMakeLists.txt

@@ -13,12 +13,13 @@
 # limitations under the License.
 
 if(PAIMON_ENABLE_LUCENE)
-    set(PAIMON_LUCENE lucene_global_index.cpp
-                      lucene_directory.cpp
-                      jieba_analyzer.cpp
-                      lucene_global_index_writer.cpp
-                      lucene_global_index_reader.cpp
-                      lucene_global_index_factory.cpp)
+    set(PAIMON_LUCENE
+        lucene_global_index.cpp
+        lucene_directory.cpp
+        jieba_analyzer.cpp
+        lucene_global_index_writer.cpp
+        lucene_global_index_reader.cpp
+        lucene_global_index_factory.cpp)
 
     add_paimon_lib(paimon_lucene_index
                    SOURCES
@@ -61,7 +62,7 @@ if(PAIMON_ENABLE_LUCENE)
                         "-Wl,--no-whole-archive"
                         ${GTEST_LINK_TOOLCHAIN})
 
-       target_compile_definitions(paimon-lucene-index-test PRIVATE
-                                        JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")                                       
+        target_compile_definitions(paimon-lucene-index-test
+                                   PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")        
     endif()
 endif()

src/paimon/global_index/lucene/jieba_analyzer.cpp

@@ -29,19 +29,18 @@ JiebaTokenizerContext::JiebaTokenizerContext(const std::string& _tokenize_mode,
       jieba(_jieba) {}
 
 JiebaTokenizer::JiebaTokenizer(const JiebaTokenizerContext& context, const Lucene::ReaderPtr& input)
-    : Lucene::Tokenizer(), context_(context) {
-    this->input = input;
+    : Lucene::Tokenizer(input), context_(context) {
     term_att_ = addAttribute<Lucene::TermAttribute>();
     pos_att_ = addAttribute<Lucene::PositionIncrementAttribute>();
     buffer_ = static_cast<wchar_t*>(
-        context_.pool->Malloc(context_.buffer_size * sizeof(wchar_t), /*aligment=*/8));
+        context_.pool->Malloc(context_.buffer_size * sizeof(wchar_t), /*alignment=*/8));
 }
 
 JiebaTokenizer::~JiebaTokenizer() {
     if (buffer_) {
         context_.pool->Free(reinterpret_cast<void*>(buffer_),
                             context_.buffer_size * sizeof(wchar_t),
-                            /*aligment=*/8);
+                            /*alignment=*/8);
         buffer_ = nullptr;
     }
 }
@@ -98,7 +97,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words
 
         // to lower case
         bool is_alphanumeric = true;
-        for (const unsigned char& c : term) {
+        for (const auto& c : term) {
             if (!std::isalnum(c)) {
                 is_alphanumeric = false;
                 break;
@@ -113,6 +112,15 @@ void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words
 
 void JiebaTokenizer::reset() {
     Lucene::Tokenizer::reset();
+    InnerReset();
+}
+
+void JiebaTokenizer::reset(const Lucene::ReaderPtr& input) {
+    Lucene::Tokenizer::reset(input);
+    InnerReset();
+}
+
+void JiebaTokenizer::InnerReset() {
     terms_.clear();
     normalized_terms_.clear();
     term_index_ = 0;

src/paimon/global_index/lucene/jieba_analyzer.h

@@ -46,6 +46,8 @@ class JiebaTokenizer : public Lucene::Tokenizer {
 
     bool incrementToken() override;
 
+    void reset(const Lucene::ReaderPtr& input) override;
+
     void reset() override;
 
     static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba,
@@ -55,6 +57,9 @@ class JiebaTokenizer : public Lucene::Tokenizer {
     static void Normalize(const std::unordered_set<std::string>& stop_words,
                           std::vector<std::string>* input, std::vector<std::string_view>* output);
 
+ private:
+    void InnerReset();
+
  private:
     JiebaTokenizerContext context_;
     size_t term_index_ = 0;

src/paimon/global_index/lucene/lucene_global_index_reader.h

@@ -93,6 +93,10 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
         return CreateAllResult();
     }
 
+    Result<std::shared_ptr<GlobalIndexResult>> VisitLike(const Literal& literal) override {
+        return CreateAllResult();
+    }
+
     Result<std::shared_ptr<VectorSearchGlobalIndexResult>> VisitVectorSearch(
         const std::shared_ptr<VectorSearch>& vector_search) override {
         return Status::Invalid(