From a7157c213b91498707b4ce7018ad53363966ea2e Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 3 Feb 2026 19:14:18 +0800 Subject: [PATCH 01/12] feat: add Jieba tokenizer for full-text search --- CMakeLists.txt | 17 +- cmake_modules/ThirdpartyToolchain.cmake | 42 +- cmake_modules/jieba.diff | 16 + src/paimon/global_index/lucene/CMakeLists.txt | 13 +- .../global_index/lucene/jieba_analyzer.cpp | 138 +++++++ .../global_index/lucene/jieba_analyzer.h | 82 ++++ .../lucene/jieba_analyzer_test.cpp | 113 ++++++ .../global_index/lucene/jieba_api_test.cpp | 71 ++++ .../global_index/lucene/lucene_api_test.cpp | 227 +++++++---- src/paimon/global_index/lucene/lucene_defs.h | 6 + .../lucene/lucene_global_index.cpp | 358 +----------------- .../global_index/lucene/lucene_global_index.h | 150 -------- .../lucene/lucene_global_index_reader.cpp | 215 +++++++++++ .../lucene/lucene_global_index_reader.h | 137 +++++++ .../lucene/lucene_global_index_test.cpp | 120 ++++++ .../lucene/lucene_global_index_writer.cpp | 242 ++++++++++++ .../lucene/lucene_global_index_writer.h | 78 ++++ src/paimon/global_index/lucene/lucene_utils.h | 16 +- third_party/versions.txt | 2 + 19 files changed, 1459 insertions(+), 584 deletions(-) create mode 100644 cmake_modules/jieba.diff create mode 100644 src/paimon/global_index/lucene/jieba_analyzer.cpp create mode 100644 src/paimon/global_index/lucene/jieba_analyzer.h create mode 100644 src/paimon/global_index/lucene/jieba_analyzer_test.cpp create mode 100644 src/paimon/global_index/lucene/jieba_api_test.cpp create mode 100644 src/paimon/global_index/lucene/lucene_global_index_reader.cpp create mode 100644 src/paimon/global_index/lucene/lucene_global_index_reader.h create mode 100644 src/paimon/global_index/lucene/lucene_global_index_writer.cpp create mode 100644 src/paimon/global_index/lucene/lucene_global_index_writer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index a98fdb78e..8d9d26844 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,6 +290,22 @@ if(PAIMON_ENABLE_LUMINA) DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() +if(PAIMON_ENABLE_LUCENE) + set(PAIMON_DICT_DEST "share/paimon/dict") + + install(DIRECTORY ${JIEBA_DICT_DIR}/ + DESTINATION ${PAIMON_DICT_DEST} + FILES_MATCHING + PATTERN "jieba.dict.utf8" + PATTERN "hmm_model.utf8" + PATTERN "idf.utf8" + PATTERN "stop_words.utf8" + PATTERN "user.dict.utf8" + PATTERN "pos_dict" + PATTERN ".git*" EXCLUDE + PATTERN "*.md" EXCLUDE) +endif() + install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION "include" FILES_MATCHING @@ -389,7 +405,6 @@ if(PAIMON_BUILD_TESTS) list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared) list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") endif() - endif() include(CMakePackageConfigHelpers) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 511a69196..263e13b46 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -322,8 +322,7 @@ macro(build_lucene) set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include") # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}") - include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR} - ${BOOST_EXTRA_INCLUDE_DIR}) + include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR}) add_library(lucene INTERFACE IMPORTED) target_include_directories(lucene SYSTEM INTERFACE "${LUCENE_INCLUDE_DIR}") target_compile_options(lucene INTERFACE -pthread) @@ -343,6 +342,44 @@ macro(build_lucene) add_dependencies(lucene lucene_ep) endmacro() +macro(build_jieba) + message(STATUS "Building jieba from source") + set(JIEBA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-prefix") + set(JIEBA_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-install") + set(JIEBA_INCLUDE_DIR "${JIEBA_INSTALL}/include") + set(JIEBA_DICT_DIR "${JIEBA_INSTALL}/dict") + file(MAKE_DIRECTORY ${JIEBA_INCLUDE_DIR}) + file(MAKE_DIRECTORY ${JIEBA_DICT_DIR}) + + set(JIEBA_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DENABLE_TEST=OFF" + "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF" + "-DCMAKE_INSTALL_PREFIX=${JIEBA_INSTALL}") + + set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff") + externalproject_add(jieba_ep + ${EP_COMMON_OPTIONS} + GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git + GIT_TAG ${PAIMON_JIEBA_BUILD_VERSION} + GIT_SHALLOW FALSE + GIT_PROGRESS TRUE + GIT_SUBMODULES_RECURSE TRUE + CMAKE_ARGS ${JIEBA_CMAKE_ARGS} + LOG_PATCH ON + PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c + "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" + INSTALL_COMMAND bash -c + "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/" + ) + + # The include directory must exist before it is referenced by a target. + include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) + add_library(jieba INTERFACE IMPORTED) + target_include_directories(jieba SYSTEM INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}") + add_dependencies(jieba jieba_ep) +endmacro() + macro(build_rapidjson) message(STATUS "Building RapidJSON from source") set(RAPIDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rapidjson_ep-install") @@ -1272,4 +1309,5 @@ endif() if(PAIMON_ENABLE_LUCENE) build_boost() build_lucene() + build_jieba() endif() diff --git a/cmake_modules/jieba.diff b/cmake_modules/jieba.diff new file mode 100644 index 000000000..04478ff6a --- /dev/null +++ b/cmake_modules/jieba.diff @@ -0,0 +1,16 @@ +diff --git a/include/cppjieba/KeywordExtractor.hpp b/include/cppjieba/KeywordExtractor.hpp +index 24b2c40..c7c6a94 100644 +--- a/include/cppjieba/KeywordExtractor.hpp ++++ b/include/cppjieba/KeywordExtractor.hpp +@@ -89,6 +89,11 @@ class KeywordExtractor { + std::partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare); + keywords.resize(topN); + } ++ ++ const std::unordered_set& GetStopWords() const { ++ return stopWords_; ++ } ++ + private: + void LoadIdfDict(const std::string& idfPath) { + std::ifstream ifs(idfPath.c_str()); diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index cb556ae9a..a032f3a10 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -13,7 +13,11 @@ # limitations under the License. if(PAIMON_ENABLE_LUCENE) - set(PAIMON_LUCENE lucene_global_index.cpp lucene_directory.cpp + set(PAIMON_LUCENE lucene_global_index.cpp + lucene_directory.cpp + jieba_analyzer.cpp + lucene_global_index_writer.cpp + lucene_global_index_reader.cpp lucene_global_index_factory.cpp) add_paimon_lib(paimon_lucene_index @@ -21,9 +25,11 @@ if(PAIMON_ENABLE_LUCENE) ${PAIMON_LUCENE} EXTRA_INCLUDES ${LUCENE_INCLUDE_DIR} + ${JIEBA_INCLUDE_DIR} DEPENDENCIES paimon_shared lucene + jieba STATIC_LINK_LIBS lucene arrow @@ -38,7 +44,9 @@ if(PAIMON_ENABLE_LUCENE) if(PAIMON_BUILD_TESTS) add_paimon_test(lucene_index_test SOURCES + jieba_analyzer_test.cpp lucene_api_test.cpp + jieba_api_test.cpp lucene_directory_test.cpp lucene_global_index_test.cpp lucene_filter_test.cpp @@ -52,5 +60,8 @@ if(PAIMON_ENABLE_LUCENE) paimon_lucene_index_static "-Wl,--no-whole-archive" ${GTEST_LINK_TOOLCHAIN}) + + target_compile_definitions(paimon-lucene-index-test PRIVATE + JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() endif() diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp new file mode 100644 index 000000000..f09692921 --- /dev/null +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -0,0 +1,138 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "paimon/global_index/lucene/jieba_analyzer.h" + +#include "paimon/global_index/lucene/lucene_utils.h" + +namespace paimon::lucene { +JiebaTokenizerContext::JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position, + const std::shared_ptr _jieba, + const std::shared_ptr& _pool, + int32_t _buffer_size) + : pool(_pool), + tokenize_mode(_tokenize_mode), + with_position(_with_position), + buffer_size(_buffer_size), + jieba(_jieba) {} + +JiebaTokenizer::JiebaTokenizer(const JiebaTokenizerContext& context, const Lucene::ReaderPtr& input) + : Lucene::Tokenizer(), context_(context) { + this->input = input; + term_att_ = addAttribute(); + pos_att_ = addAttribute(); + buffer_ = static_cast( + context_.pool->Malloc(context_.buffer_size * sizeof(wchar_t), /*aligment=*/8)); +} + +JiebaTokenizer::~JiebaTokenizer() { + if (buffer_) { + context_.pool->Free(reinterpret_cast(buffer_), + context_.buffer_size * sizeof(wchar_t), + /*aligment=*/8); + buffer_ = nullptr; + } +} + +bool JiebaTokenizer::incrementToken() { + if (term_index_ >= normalized_terms_.size()) { + return false; + } + + const auto& term = normalized_terms_[term_index_++]; + clearAttributes(); + + term_att_->setTermBuffer(LuceneUtils::StringToWstring(term)); + + if (context_.with_position) { + pos_att_->setPositionIncrement(1); + } else { + pos_att_->setPositionIncrement(0); + } + return true; +} + +void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, + const std::string& str, std::vector* terms_ptr) { + auto& terms = *terms_ptr; + if (tokenize_mode == "mp") { + jieba->CutSmall(str, terms, /*max_word_len=*/JiebaTokenizerContext::kMaxWordLen); + } else if (tokenize_mode == "hmm") { + jieba->CutHMM(str, terms); + } else if (tokenize_mode == "mix") { + jieba->Cut(str, terms, /*hmm=*/true); + } else if (tokenize_mode == "full") { + jieba->CutAll(str, terms); + } else if (tokenize_mode == "query") { + jieba->CutForSearch(str, terms, /*hmm=*/true); + } else { + throw Lucene::IllegalArgumentException( + L"only support mp/hmm/mix/full/query in jieba tokenizer"); + } +} + +void JiebaTokenizer::Normalize(const std::unordered_set& stop_words, + std::vector* input_ptr, + std::vector* output_ptr) { + auto& input = *input_ptr; + auto& output = *output_ptr; + output.clear(); + output.reserve(input.size()); + for (auto& term : input) { + // remove stop words + if (stop_words.find(term) != stop_words.end()) { + continue; + } + + // to lower case + bool is_alphanumeric = true; + for (const unsigned char& c : term) { + if (!std::isalnum(c)) { + is_alphanumeric = false; + break; + } + } + if (is_alphanumeric && !term.empty()) { + std::transform(term.begin(), term.end(), term.begin(), ::tolower); + } + output.emplace_back(term.data(), term.length()); + } +} + +void JiebaTokenizer::reset() { + Lucene::Tokenizer::reset(); + terms_.clear(); + normalized_terms_.clear(); + term_index_ = 0; + + // read wchar from input + Lucene::String wstr; + wstr.reserve(context_.buffer_size); + while (true) { + int32_t length = input->read(buffer_, /*offset=*/0, context_.buffer_size); + if (length <= 0) { + break; + } + wstr.append(buffer_, length); + } + + // jieba tokenize + std::string doc_str = LuceneUtils::WstringToString(wstr); + // TODO(xinyu.lxy): support porter2 stemmer + CutWithMode(context_.tokenize_mode, context_.jieba.get(), doc_str, &terms_); + Normalize(context_.jieba->extractor.GetStopWords(), &terms_, &normalized_terms_); +} + +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h new file mode 100644 index 000000000..c74cb31a9 --- /dev/null +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -0,0 +1,82 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "cppjieba/Jieba.hpp" +#include "lucene++/LuceneHeaders.h" +#include "lucene++/MiscUtils.h" +#include "lucene++/PositionIncrementAttribute.h" +#include "lucene++/TermAttribute.h" +#include "paimon/global_index/lucene/lucene_utils.h" +#include "paimon/memory/memory_pool.h" +namespace paimon::lucene { +struct JiebaTokenizerContext { + JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position, + const std::shared_ptr _jieba, + const std::shared_ptr& _pool, + int32_t _buffer_size = kReadBufferSize); + + std::shared_ptr pool; + std::string tokenize_mode; + bool with_position; + int32_t buffer_size; + std::shared_ptr jieba; + + static inline const int32_t kReadBufferSize = 5 * 1024 * 1024; + static inline const int32_t kMaxWordLen = 1024; +}; + +class JiebaTokenizer : public Lucene::Tokenizer { + public: + JiebaTokenizer(const JiebaTokenizerContext& context, const Lucene::ReaderPtr& input); + + ~JiebaTokenizer() override; + + bool incrementToken() override; + + void reset() override; + + static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, + const std::string& str, std::vector* terms_ptr); + + // inplace to lower to avoid data copy + static void Normalize(const std::unordered_set& stop_words, + std::vector* input, std::vector* output); + + private: + JiebaTokenizerContext context_; + size_t term_index_ = 0; + std::vector terms_; + std::vector normalized_terms_; + wchar_t* buffer_; + Lucene::TermAttributePtr term_att_; + Lucene::PositionIncrementAttributePtr pos_att_; +}; + +class JiebaAnalyzer : public Lucene::Analyzer { + public: + JiebaAnalyzer(const JiebaTokenizerContext& context) : context_(context) {} + + ~JiebaAnalyzer() override = default; + + Lucene::TokenStreamPtr tokenStream(const Lucene::String& field_name, + const Lucene::ReaderPtr& reader) override { + return Lucene::newLucene(context_, reader); + } + + private: + JiebaTokenizerContext context_; +}; +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp new file mode 100644 index 000000000..8c3f74626 --- /dev/null +++ b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp @@ -0,0 +1,113 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "paimon/global_index/lucene/jieba_analyzer.h" + +#include "cppjieba/Jieba.hpp" +#include "gtest/gtest.h" +#include "lucene++/LuceneHeaders.h" +#include "paimon/global_index/lucene/lucene_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" +namespace paimon::lucene::test { +class JiebaAnalyzerTest : public ::testing::Test, public ::testing::WithParamInterface { + public: + void SetUp() override {} + void TearDown() override {} + Lucene::TokenStreamPtr CreateJiebaTokenizer(bool with_position) const { + return CreateJiebaTokenizer(with_position, L"我爱机器学习"); + } + + Lucene::TokenStreamPtr CreateJiebaTokenizer(bool with_position, + const Lucene::String& text) const { + auto pool = GetDefaultPool(); + std::string dictionary_dir = LuceneUtils::GetJiebaDictionaryDir().value(); + auto jieba = std::make_shared( + dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + auto reader = Lucene::newLucene(text); + int32_t buffer_size = GetParam(); + JiebaTokenizerContext context(/*tokenize_mode=*/"query", with_position, jieba, pool, + buffer_size); + auto analyzer = Lucene::newLucene(context); + return analyzer->tokenStream(/*field_name*/ L"f0", reader); + } +}; + +TEST_P(JiebaAnalyzerTest, TestSimple) { + auto tokenizer = CreateJiebaTokenizer(/*with_position=*/false); + + auto term_att = tokenizer->addAttribute(); + + tokenizer->reset(); + std::vector results; + while (tokenizer->incrementToken()) { + results.push_back(term_att->term()); + } + tokenizer->end(); + tokenizer->close(); + std::vector expected = {L"爱", L"机器", L"学习"}; + ASSERT_EQ(expected, results); +} + +TEST_P(JiebaAnalyzerTest, TestWithPosition) { + auto tokenizer = CreateJiebaTokenizer(/*with_position=*/true); + + auto term_att = tokenizer->addAttribute(); + auto pos_att = tokenizer->addAttribute(); + + tokenizer->reset(); + std::vector results; + std::vector result_pos; + int32_t pos = 0; + while (tokenizer->incrementToken()) { + pos += pos_att->getPositionIncrement(); + result_pos.push_back(pos); + results.push_back(term_att->term()); + } + tokenizer->end(); + tokenizer->close(); + + std::vector expected = {L"爱", L"机器", L"学习"}; + std::vector expected_pos = {1, 2, 3}; + ASSERT_EQ(expected, results); + ASSERT_EQ(expected_pos, result_pos); +} + +TEST_P(JiebaAnalyzerTest, TestNormalize) { + auto tokenizer = CreateJiebaTokenizer( + /*with_position=*/false, + L"由于购买了Iphone14，我越来越热爱网上学习了！Happy work, happy day!"); + + auto term_att = tokenizer->addAttribute(); + + tokenizer->reset(); + std::vector results; + while (tokenizer->incrementToken()) { + results.push_back(term_att->term()); + } + tokenizer->end(); + tokenizer->close(); + std::vector expected = {L"购买", L"iphone14", L"越来", L"越来越", + L"热爱", L"网上", L"学习", L"happy", + L"work", L"happy", L"day"}; + ASSERT_EQ(expected, results); +} + +INSTANTIATE_TEST_SUITE_P(ReadBufferSize, JiebaAnalyzerTest, + ::testing::ValuesIn(std::vector({2, 5, 10, 100}))); + +} // namespace paimon::lucene::test diff --git a/src/paimon/global_index/lucene/jieba_api_test.cpp b/src/paimon/global_index/lucene/jieba_api_test.cpp new file mode 100644 index 000000000..ae1b20567 --- /dev/null +++ b/src/paimon/global_index/lucene/jieba_api_test.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "cppjieba/Jieba.hpp" +#include "gtest/gtest.h" +#include "paimon/global_index/lucene/lucene_utils.h" +#include "paimon/testing/utils/testharness.h" +namespace paimon::lucene::test { +class JiebaInterfaceTest : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(JiebaInterfaceTest, TestSimple) { + ASSERT_OK_AND_ASSIGN(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); + cppjieba::Jieba jieba(dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + + { + std::vector words; + jieba.CutForSearch("我爱机器学习", words); + std::vector expected = {"我", "爱", "机器", "学习"}; + ASSERT_EQ(expected, words); + } + { + std::vector words; + jieba.CutForSearch("我爱机器学习 工作 good work nice Day，price12345，12345", words); + std::vector expected = {"我", "爱", "机器", "学习", " ", "工作", + " ", "good", " ", "work", " ", "nice", + " ", "Day", "，", "price12345", "，", "12345"}; + ASSERT_EQ(expected, words); + } +} + +TEST_F(JiebaInterfaceTest, TestMP) { + ASSERT_OK_AND_ASSIGN(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); + cppjieba::Jieba jieba(dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + + { + std::vector words; + jieba.CutSmall("我爱机器学习", words, /*max_word_len=*/5); + std::vector expected = {"我", "爱", "机器", "学习"}; + ASSERT_EQ(expected, words); + } + { + std::vector words; + jieba.CutSmall("我爱机器学习", words, /*max_word_len=*/1); + std::vector expected = {"我", "爱", "机", "器", "学", "习"}; + ASSERT_EQ(expected, words); + } +} + +} // namespace paimon::lucene::test diff --git a/src/paimon/global_index/lucene/lucene_api_test.cpp b/src/paimon/global_index/lucene/lucene_api_test.cpp index d6868f0f1..fd4f274f5 100644 --- a/src/paimon/global_index/lucene/lucene_api_test.cpp +++ b/src/paimon/global_index/lucene/lucene_api_test.cpp @@ -17,10 +17,11 @@ #include "lucene++/FileUtils.h" #include "lucene++/LuceneHeaders.h" #include "lucene++/MiscUtils.h" +#include "paimon/global_index/lucene/jieba_analyzer.h" #include "paimon/global_index/lucene/lucene_directory.h" #include "paimon/global_index/lucene/lucene_utils.h" +#include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" - namespace paimon::lucene::test { class LuceneInterfaceTest : public ::testing::Test { public: @@ -80,104 +81,184 @@ class LuceneInterfaceTest : public ::testing::Test { private: std::vector ids_; }; -}; -TEST_F(LuceneInterfaceTest, TestSimple) { - auto dir = paimon::test::UniqueTestDirectory::Create("local"); - std::string index_path = dir->Str() + "/lucene_test"; - auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(index_path), - Lucene::NoLockFactory::getNoLockFactory()); - - Lucene::IndexWriterPtr writer = Lucene::newLucene( - lucene_dir, - Lucene::newLucene(Lucene::LuceneVersion::LUCENE_CURRENT), - /*create=*/true, Lucene::IndexWriter::MaxFieldLengthLIMITED); - - Lucene::DocumentPtr doc = Lucene::newLucene(); - auto field = Lucene::newLucene(L"content", L"", Lucene::Field::STORE_NO, - Lucene::Field::INDEX_ANALYZED_NO_NORMS); - auto doc_id_field = Lucene::newLucene( - L"id", L"", Lucene::Field::STORE_YES, Lucene::Field::INDEX_NOT_ANALYZED_NO_NORMS); - - field->setOmitTermFreqAndPositions(false); - doc_id_field->setOmitTermFreqAndPositions(true); - doc->add(field); - doc->add(doc_id_field); - - auto build = [&](const std::wstring& doc_str, int32_t doc_id) { - field->setValue(doc_str); - doc_id_field->setValue(LuceneUtils::StringToWstring(std::to_string(doc_id))); - writer->addDocument(doc); + struct WriteContext { + Lucene::IndexWriterPtr writer; + Lucene::DocumentPtr doc; + Lucene::FieldPtr field; + Lucene::FieldPtr doc_id_field; }; - build(L"This is an test document.", 0); - build(L"This is an new document document document.", 1); - build(L"Document document document document test.", 2); - build(L"unordered user-defined doc id", 5); - build(L"", 6); // add a null doc + Lucene::AnalyzerPtr CreateJiebaAnalyzer() const { + auto pool = GetDefaultPool(); + std::string dictionary_dir = LuceneUtils::GetJiebaDictionaryDir().value(); + auto jieba = std::make_shared( + dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + JiebaTokenizerContext context(/*tokenize_mode=*/"query", /*with_position=*/true, jieba, + pool); + return Lucene::newLucene(context); + } - writer->optimize(); - writer->close(); + WriteContext CreateWriteContext(const Lucene::DirectoryPtr& lucene_dir, + const Lucene::AnalyzerPtr& analyzer) const { + auto lucene_analyzer = analyzer ? analyzer + : Lucene::newLucene( + Lucene::LuceneVersion::LUCENE_CURRENT); + Lucene::IndexWriterPtr writer = Lucene::newLucene( + lucene_dir, lucene_analyzer, + /*create=*/true, Lucene::IndexWriter::MaxFieldLengthLIMITED); - // read - Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true); - Lucene::IndexSearcherPtr searcher = Lucene::newLucene(reader); - Lucene::QueryParserPtr parser = Lucene::newLucene( - Lucene::LuceneVersion::LUCENE_CURRENT, L"content", - Lucene::newLucene(Lucene::LuceneVersion::LUCENE_CURRENT)); - parser->setAllowLeadingWildcard(true); - - auto search = [&](const std::wstring& query_str, int32_t limit, - const std::optional> selected_id, - const std::vector& expected_doc_id_vec, - const std::vector& expected_doc_id_content_vec) { - Lucene::QueryPtr query = parser->parse(query_str); + Lucene::DocumentPtr doc = Lucene::newLucene(); + auto field = Lucene::newLucene(L"content", L"", Lucene::Field::STORE_NO, + Lucene::Field::INDEX_ANALYZED_NO_NORMS); + auto doc_id_field = Lucene::newLucene( + L"id", L"", Lucene::Field::STORE_YES, Lucene::Field::INDEX_NOT_ANALYZED_NO_NORMS); + + field->setOmitTermFreqAndPositions(false); + doc_id_field->setOmitTermFreqAndPositions(true); + doc->add(field); + doc->add(doc_id_field); + return {writer, doc, field, doc_id_field}; + } + + void AddDocument(const std::wstring& doc_str, int32_t doc_id, WriteContext* context) const { + context->field->setValue(doc_str); + context->doc_id_field->setValue(LuceneUtils::StringToWstring(std::to_string(doc_id))); + context->writer->addDocument(context->doc); + } + + struct ReadConext { + Lucene::IndexReaderPtr reader; + Lucene::IndexSearcherPtr searcher; + Lucene::QueryParserPtr parser; + }; + + ReadConext CreateReadContext(const Lucene::DirectoryPtr& lucene_dir, + const Lucene::AnalyzerPtr& analyzer) const { + auto lucene_analyzer = analyzer ? analyzer + : Lucene::newLucene( + Lucene::LuceneVersion::LUCENE_CURRENT); + Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true); + Lucene::IndexSearcherPtr searcher = Lucene::newLucene(reader); + Lucene::QueryParserPtr parser = Lucene::newLucene( + Lucene::LuceneVersion::LUCENE_CURRENT, L"content", lucene_analyzer); + parser->setAllowLeadingWildcard(true); + return {reader, searcher, parser}; + } + + void Search(const std::wstring& query_str, int32_t limit, + const std::optional> selected_id, + const std::vector& expected_doc_id_vec, + const std::vector& expected_doc_id_content_vec, + ReadConext* context) const { + Lucene::QueryPtr query = context->parser->parse(query_str); Lucene::TopDocsPtr results; if (selected_id) { Lucene::FilterPtr lucene_filter = Lucene::newLucene(selected_id.value()); - results = searcher->search(query, lucene_filter, limit); + results = context->searcher->search(query, lucene_filter, limit); } else { - results = searcher->search(query, limit); + results = context->searcher->search(query, limit); } ASSERT_EQ(expected_doc_id_vec.size(), results->scoreDocs.size()); std::vector resule_doc_id_vec; std::vector result_doc_id_content_vec; for (auto score_doc : results->scoreDocs) { - Lucene::DocumentPtr result_doc = searcher->doc(score_doc->doc); + Lucene::DocumentPtr result_doc = context->searcher->doc(score_doc->doc); resule_doc_id_vec.push_back(score_doc->doc); result_doc_id_content_vec.push_back(result_doc->get(L"id")); } ASSERT_EQ(resule_doc_id_vec, expected_doc_id_vec); ASSERT_EQ(result_doc_id_content_vec, expected_doc_id_content_vec); - }; + } +}; + +TEST_F(LuceneInterfaceTest, TestSimple) { + auto dir = paimon::test::UniqueTestDirectory::Create("local"); + std::string index_path = dir->Str() + "/lucene_test"; + auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(index_path), + Lucene::NoLockFactory::getNoLockFactory()); + // write + auto write_context = CreateWriteContext(lucene_dir, /*analyzer=*/nullptr); + + AddDocument(L"This is an test document.", 0, &write_context); + AddDocument(L"This is an new document document document.", 1, &write_context); + AddDocument(L"Document document document document test.", 2, &write_context); + AddDocument(L"unordered user-defined doc id", 5, &write_context); + AddDocument(L"", 6, &write_context); // add a null doc + + write_context.writer->optimize(); + write_context.writer->close(); + + // read + auto read_context = CreateReadContext(lucene_dir, /*analyzer=*/nullptr); // result is sorted by tf-idf score - search(L"document", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({2, 1, 0}), - std::vector({L"2", L"1", L"0"})); - search(L"document", /*limit=*/1, /*selected_id=*/std::nullopt, std::vector({2}), - std::vector({L"2"})); - search(L"test AND document", /*limit=*/10, /*selected_id=*/std::nullopt, - std::vector({2, 0}), std::vector({L"2", L"0"})); - search(L"test OR new", /*limit=*/10, /*selected_id=*/std::nullopt, - std::vector({1, 0, 2}), std::vector({L"1", L"0", L"2"})); - search(L"\"test document\"", /*limit=*/10, /*selected_id=*/std::nullopt, - std::vector({0}), std::vector({L"0"})); - search(L"unordered", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({3}), - std::vector({L"5"})); - search(L"*orDer*", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({3}), - std::vector({L"5"})); + Search(L"document", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({2, 1, 0}), + std::vector({L"2", L"1", L"0"}), &read_context); + Search(L"document", /*limit=*/1, /*selected_id=*/std::nullopt, std::vector({2}), + std::vector({L"2"}), &read_context); + Search(L"test AND document", /*limit=*/10, /*selected_id=*/std::nullopt, + std::vector({2, 0}), std::vector({L"2", L"0"}), &read_context); + Search(L"test OR new", /*limit=*/10, /*selected_id=*/std::nullopt, + std::vector({1, 0, 2}), std::vector({L"1", L"0", L"2"}), + &read_context); + Search(L"\"test document\"", /*limit=*/10, /*selected_id=*/std::nullopt, + std::vector({0}), std::vector({L"0"}), &read_context); + Search(L"unordered", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({3}), + std::vector({L"5"}), &read_context); + Search(L"*orDer*", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({3}), + std::vector({L"5"}), &read_context); // test filter - search(L"document", /*limit=*/10, /*selected_id=*/std::vector({0, 1}), - std::vector({1, 0}), std::vector({L"1", L"0"})); - search(L"document OR unordered", /*limit=*/10, + Search(L"document", /*limit=*/10, /*selected_id=*/std::vector({0, 1}), + std::vector({1, 0}), std::vector({L"1", L"0"}), &read_context); + Search(L"document OR unordered", /*limit=*/10, /*selected_id=*/std::vector({0, 1, 3}), std::vector({3, 1, 0}), - std::vector({L"5", L"1", L"0"})); - search(L"unordered", /*limit=*/10, /*selected_id=*/std::vector({0}), - std::vector(), std::vector()); + std::vector({L"5", L"1", L"0"}), &read_context); + Search(L"unordered", /*limit=*/10, /*selected_id=*/std::vector({0}), + std::vector(), std::vector(), &read_context); + + read_context.reader->close(); + lucene_dir->close(); +} + +TEST_F(LuceneInterfaceTest, TestWithAnalyzer) { + auto dir = paimon::test::UniqueTestDirectory::Create("local"); + std::string index_path = dir->Str() + "/lucene_test"; + auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(index_path), + Lucene::NoLockFactory::getNoLockFactory()); + // write + auto analyzer = CreateJiebaAnalyzer(); + auto write_context = CreateWriteContext(lucene_dir, analyzer); + + AddDocument(L"我爱机器学习", 0, &write_context); + AddDocument(L"机器会学习吗？", 1, &write_context); + AddDocument(L"我爱工作", 2, &write_context); + AddDocument(L"Have a nice day", 3, &write_context); + + write_context.writer->optimize(); + write_context.writer->close(); + + // read + auto read_context = CreateReadContext(lucene_dir, analyzer); + + // result is sorted by tf-idf score + Search(L"机器", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({0, 1}), + std::vector({L"0", L"1"}), &read_context); + Search(L"机器 AND 学习", /*limit=*/10, /*selected_id=*/std::nullopt, + std::vector({0, 1}), std::vector({L"0", L"1"}), &read_context); + Search(L"\"机器学习\"", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({0}), + std::vector({L"0"}), &read_context); + Search(L"我爱", /*limit=*/10, /*selected_id=*/std::nullopt, std::vector({0, 2}), + std::vector({L"0", L"2"}), &read_context); + Search(L"爱 OR nice", /*limit=*/10, /*selected_id=*/std::nullopt, + std::vector({3, 0, 2}), std::vector({L"3", L"0", L"2"}), + &read_context); - reader->close(); + read_context.reader->close(); lucene_dir->close(); } diff --git a/src/paimon/global_index/lucene/lucene_defs.h b/src/paimon/global_index/lucene/lucene_defs.h index db47037f5..71eaa9547 100644 --- a/src/paimon/global_index/lucene/lucene_defs.h +++ b/src/paimon/global_index/lucene/lucene_defs.h @@ -33,4 +33,10 @@ static inline const char kLuceneReadBufferSize[] = "read.buffer-size"; // default is false static inline const char kLuceneWriteOmitTermFreqAndPositions[] = "write.omit-term-freq-and-position"; + +static inline const char kJiebaDictDirEnv[] = "PAIMON_JIEBA_DICT_DIR"; + +static inline const char kDefaultJiebaTokenizeMode[] = "mix"; +// default is "mix". Values can be "mp", "hmm", "mix", "full", "query". +static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode"; } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index.cpp b/src/paimon/global_index/lucene/lucene_global_index.cpp index 5e4dc5f6a..bef013698 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index.cpp @@ -19,19 +19,11 @@ #include "arrow/c/bridge.h" #include "lucene++/FileUtils.h" -#include "paimon/common/io/data_output_stream.h" #include "paimon/common/utils/options_utils.h" -#include "paimon/common/utils/path_util.h" -#include "paimon/common/utils/rapidjson_util.h" -#include "paimon/common/utils/uuid.h" -#include "paimon/global_index/bitmap_vector_search_global_index_result.h" -#include "paimon/global_index/lucene/lucene_collector.h" #include "paimon/global_index/lucene/lucene_defs.h" -#include "paimon/global_index/lucene/lucene_directory.h" -#include "paimon/global_index/lucene/lucene_filter.h" +#include "paimon/global_index/lucene/lucene_global_index_reader.h" +#include "paimon/global_index/lucene/lucene_global_index_writer.h" #include "paimon/global_index/lucene/lucene_utils.h" -#include "paimon/io/data_input_stream.h" - namespace paimon::lucene { #define CHECK_NOT_NULL(pointer, error_msg) \ do { \ @@ -83,350 +75,4 @@ Result> LuceneGlobalIndex::CreateReader( pool); } -LuceneGlobalIndexWriter::LuceneWriteContext::LuceneWriteContext( - const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, const Lucene::DocumentPtr& _doc, - const Lucene::FieldPtr& _field, const Lucene::FieldPtr& _row_id_field) - : tmp_index_path(_tmp_index_path), - lucene_dir(_lucene_dir), - index_writer(_index_writer), - doc(_doc), - field(_field), - row_id_field(_row_id_field) {} - -Result> LuceneGlobalIndexWriter::Create( - const std::string& field_name, const std::shared_ptr& arrow_type, - const std::shared_ptr& file_writer, - const std::map& options, const std::shared_ptr& pool) { - try { - std::string uuid; - if (!UUID::Generate(&uuid)) { - return Status::Invalid("generate uuid for lucene tmp path failed."); - } - // create a local tmp path - std::string tmp_path = PathUtil::JoinPath(std::filesystem::temp_directory_path().string(), - "paimon-lucene-" + uuid); - auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), - Lucene::NoLockFactory::getNoLockFactory()); - // TODO(xinyu.lxy): support other tokenizer - // open lucene index writer - Lucene::IndexWriterPtr writer = Lucene::newLucene( - lucene_dir, - Lucene::newLucene(Lucene::LuceneVersion::LUCENE_CURRENT), - /*create=*/true, Lucene::IndexWriter::MaxFieldLengthLIMITED); - - // prepare field and document - Lucene::DocumentPtr doc = Lucene::newLucene(); - auto field = Lucene::newLucene(LuceneUtils::StringToWstring(field_name), - kEmptyWstring, Lucene::Field::STORE_NO, - Lucene::Field::INDEX_ANALYZED_NO_NORMS); - auto row_id_field = Lucene::newLucene( - kRowIdFieldWstring, kEmptyWstring, Lucene::Field::STORE_YES, - Lucene::Field::INDEX_NOT_ANALYZED_NO_NORMS); - PAIMON_ASSIGN_OR_RAISE( - bool omit_term_freq_and_positions, - OptionsUtils::GetValueFromMap(options, kLuceneWriteOmitTermFreqAndPositions, false)); - field->setOmitTermFreqAndPositions(omit_term_freq_and_positions); - row_id_field->setOmitTermFreqAndPositions(true); - doc->add(field); - doc->add(row_id_field); - return std::shared_ptr(new LuceneGlobalIndexWriter( - field_name, arrow_type, - LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field, row_id_field), file_writer, - options, pool)); - } catch (const std::exception& e) { - return Status::Invalid( - fmt::format("create lucene global index writer failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError( - "create lucene global index writer failed, with unknown error."); - } -} - -LuceneGlobalIndexWriter::LuceneGlobalIndexWriter( - const std::string& field_name, const std::shared_ptr& arrow_type, - LuceneWriteContext&& write_context, const std::shared_ptr& file_writer, - const std::map& options, const std::shared_ptr& pool) - : pool_(pool), - field_name_(field_name), - arrow_type_(arrow_type), - write_context_(std::move(write_context)), - file_writer_(file_writer), - options_(options) {} - -LuceneGlobalIndexWriter::~LuceneGlobalIndexWriter() { - try { - [[maybe_unused]] bool ec = Lucene::FileUtils::removeDirectory( - LuceneUtils::StringToWstring(write_context_.tmp_index_path)); - } catch (...) { - // do nothing - } -} - -Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, - arrow::ImportArray(arrow_array, arrow_type_)); - auto struct_array = std::dynamic_pointer_cast(array); - CHECK_NOT_NULL(struct_array, "invalid input array in LuceneIndexWriter, must be struct array"); - auto field_array = struct_array->GetFieldByName(field_name_); - CHECK_NOT_NULL( - field_array, - fmt::format("invalid input array in LuceneIndexWriter, field {} not in input array", - field_name_)); - auto string_array = std::dynamic_pointer_cast(field_array); - CHECK_NOT_NULL( - string_array, - fmt::format( - "invalid input array in LuceneIndexWriter, field array {} is not a string array", - field_name_)); - try { - for (int64_t i = 0; i < string_array->length(); i++) { - if (string_array->IsNull(i)) { - write_context_.field->setValue(kEmptyWstring); - } else { - auto view = string_array->Value(i); - write_context_.field->setValue(LuceneUtils::StringToWstring(view)); - } - write_context_.row_id_field->setValue( - LuceneUtils::StringToWstring(std::to_string(row_id_++))); - write_context_.index_writer->addDocument(write_context_.doc); - } - } catch (const std::exception& e) { - return Status::Invalid(fmt::format( - "add batch for lucene global index writer failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError( - "add batch for lucene global index writer failed, with unknown error."); - } - return Status::OK(); -} - -Result LuceneGlobalIndexWriter::FlushIndexToFinal() const { - try { - // flush index to tmp dir - write_context_.index_writer->optimize(); - write_context_.index_writer->close(); - - // list tmp dir - auto tmp_file_names = write_context_.lucene_dir->listAll(); - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer_->NewFileName(kIdentifier)); - // prepare output from file_writer - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_writer_->NewOutputStream(index_file_name)); - DataOutputStream data_output_stream(out); - PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(kVersion)); - PAIMON_RETURN_NOT_OK( - data_output_stream.WriteValue(static_cast(tmp_file_names.size()))); - // read all data from index files and write to target output - auto buffer = std::make_shared(kDefaultReadBufferSize, pool_.get()); - for (const auto& wfile_name : tmp_file_names) { - auto file_name = LuceneUtils::WstringToString(wfile_name); - PAIMON_RETURN_NOT_OK( - data_output_stream.WriteValue(static_cast(file_name.size()))); - PAIMON_RETURN_NOT_OK( - data_output_stream.WriteBytes(std::make_shared(file_name, pool_.get()))); - int64_t file_length = write_context_.lucene_dir->fileLength(wfile_name); - PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(file_length)); - - Lucene::IndexInputPtr input = write_context_.lucene_dir->openInput(wfile_name); - int64_t total_write_size = 0; - while (total_write_size < file_length) { - int64_t current_write_size = std::min(file_length - total_write_size, - static_cast(kDefaultReadBufferSize)); - input->readBytes(reinterpret_cast(buffer->data()), /*offset=*/0, - static_cast(current_write_size)); - PAIMON_ASSIGN_OR_RAISE( - int32_t actual_write_size, - out->Write(buffer->data(), static_cast(current_write_size))); - if (static_cast(actual_write_size) != current_write_size) { - return Status::Invalid( - fmt::format("invalid write, try to write {} while actual write {}", - current_write_size, actual_write_size)); - } - total_write_size += current_write_size; - } - input->close(); - } - PAIMON_RETURN_NOT_OK(out->Flush()); - PAIMON_RETURN_NOT_OK(out->Close()); - write_context_.lucene_dir->close(); - return index_file_name; - } catch (const std::exception& e) { - return Status::Invalid( - fmt::format("finish for lucene global index writer failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError( - "finish for lucene global index writer failed, with unknown error."); - } -} - -Result> LuceneGlobalIndexWriter::Finish() { - PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, FlushIndexToFinal()); - // prepare global index meta - PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name)); - std::string options_json; - PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); - auto meta_bytes = std::make_shared(options_json, pool_.get()); - GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, - /*range_end=*/static_cast(row_id_) - 1, - /*metadata=*/meta_bytes); - return std::vector({meta}); -} - -Result> LuceneGlobalIndexReader::Create( - const std::string& field_name, const GlobalIndexIOMeta& io_meta, - const std::shared_ptr& file_reader, - const std::map& options, const std::shared_ptr& pool) { - try { - std::map> file_name_to_offset_and_length; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr paimon_input, - file_reader->GetInputStream(io_meta.file_path)); - DataInputStream data_input_stream(paimon_input); - PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input_stream.ReadValue()); - if (version != kVersion) { - return Status::Invalid(fmt::format("LuceneGlobalIndex not support version {}"), - kVersion); - } - PAIMON_ASSIGN_OR_RAISE(int32_t num_files, data_input_stream.ReadValue()); - for (int32_t i = 0; i < num_files; i++) { - PAIMON_ASSIGN_OR_RAISE(int32_t file_name_len, data_input_stream.ReadValue()); - auto file_name_bytes = std::make_shared(file_name_len, pool.get()); - PAIMON_RETURN_NOT_OK(data_input_stream.ReadBytes(file_name_bytes.get())); - std::string file_name(file_name_bytes->data(), file_name_bytes->size()); - PAIMON_ASSIGN_OR_RAISE(int64_t file_len, data_input_stream.ReadValue()); - PAIMON_ASSIGN_OR_RAISE(int64_t pos, data_input_stream.GetPos()); - file_name_to_offset_and_length[file_name] = {pos, file_len}; - pos += file_len; - if (i != num_files - 1) { - PAIMON_RETURN_NOT_OK(data_input_stream.Seek(pos)); - } - } - PAIMON_ASSIGN_OR_RAISE( - int32_t read_buffer_size, - OptionsUtils::GetValueFromMap(options, kLuceneReadBufferSize, kDefaultReadBufferSize)); - Lucene::DirectoryPtr lucene_dir = Lucene::newLucene( - PathUtil::GetParentDirPath(io_meta.file_path), file_name_to_offset_and_length, - paimon_input, read_buffer_size); - - Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true); - Lucene::IndexSearcherPtr searcher = Lucene::newLucene(reader); - return std::shared_ptr(new LuceneGlobalIndexReader( - LuceneUtils::StringToWstring(field_name), io_meta.range_end, searcher)); - } catch (const std::exception& e) { - return Status::Invalid( - fmt::format("create lucene global index reader failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError( - "create lucene global index reader failed, with unknown error."); - } -} - -std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::string& query) { - // TODO(xinyu.lxy): support jieba analyzer - std::vector terms = - StringUtils::Split(query, /*sep_str=*/" ", /*ignore_empty=*/true); - std::vector wterms; - wterms.reserve(terms.size()); - for (const auto& term : terms) { - wterms.push_back(LuceneUtils::StringToWstring(term)); - } - return wterms; -} - -Result> LuceneGlobalIndexReader::VisitFullTextSearch( - const std::shared_ptr& full_text_search) { - try { - Lucene::QueryPtr query; - switch (full_text_search->search_type) { - case FullTextSearch::SearchType::MATCH_ALL: - case FullTextSearch::SearchType::MATCH_ANY: { - Lucene::BooleanClause::Occur occur = - full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL - ? Lucene::BooleanClause::Occur::MUST - : Lucene::BooleanClause::Occur::SHOULD; - std::vector query_terms = TokenizeQuery(full_text_search->query); - if (query_terms.size() == 1) { - query = Lucene::newLucene( - Lucene::newLucene(wfield_name_, query_terms[0])); - } else { - auto typed_query = Lucene::newLucene(); - for (const auto& term : query_terms) { - typed_query->add(Lucene::newLucene( - Lucene::newLucene(wfield_name_, term)), - occur); - } - query = typed_query; - } - break; - } - case FullTextSearch::SearchType::PHRASE: { - std::vector query_terms = TokenizeQuery(full_text_search->query); - auto typed_query = Lucene::newLucene(); - for (const auto& term : query_terms) { - typed_query->add(Lucene::newLucene(wfield_name_, term)); - } - query = typed_query; - break; - } - case FullTextSearch::SearchType::PREFIX: { - query = Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); - break; - } - case FullTextSearch::SearchType::WILDCARD: { - query = Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); - break; - } - default: - return Status::Invalid( - fmt::format("Not support for FullTextSearch SearchType {}", - static_cast(full_text_search->search_type))); - } - Lucene::FilterPtr filter = - full_text_search->pre_filter - ? Lucene::newLucene(&(full_text_search->pre_filter.value())) - : Lucene::FilterPtr(); - - if (full_text_search->limit) { - Lucene::TopDocsPtr results = - searcher_->search(query, filter, full_text_search->limit.value()); - - // prepare BitmapVectorSearchGlobalIndexResult - std::map id_to_score; - for (auto score_doc : results->scoreDocs) { - Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); - std::string row_id_str = - LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); - std::optional row_id = StringUtils::StringToValue(row_id_str); - if (!row_id) { - return Status::Invalid( - fmt::format("parse row id str {} to int failed", row_id_str)); - } - id_to_score[static_cast(row_id.value())] = - static_cast(score_doc->score); - } - RoaringBitmap64 bitmap; - std::vector scores; - scores.reserve(id_to_score.size()); - for (const auto& [id, score] : id_to_score) { - bitmap.Add(id); - scores.push_back(score); - } - return std::make_shared(std::move(bitmap), - std::move(scores)); - } else { - // with no limit & no score - auto collector = Lucene::newLucene(); - searcher_->search(query, filter, collector); - return std::make_shared( - [collector]() -> Result { return collector->GetBitmap(); }); - } - } catch (const std::exception& e) { - return Status::Invalid(fmt::format("visit term query failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError("visit term query failed, with unknown error."); - } -} - } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index d5c607d84..61864c345 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -45,154 +45,4 @@ class LuceneGlobalIndex : public GlobalIndexer { std::map options_; }; -class LuceneGlobalIndexWriter : public GlobalIndexWriter { - public: - struct LuceneWriteContext { - LuceneWriteContext(const std::string& _tmp_index_path, - const Lucene::FSDirectoryPtr& _lucene_dir, - const Lucene::IndexWriterPtr& _index_writer, - const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field, - const Lucene::FieldPtr& _row_id_field); - - LuceneWriteContext(LuceneWriteContext&&) = default; - LuceneWriteContext& operator=(LuceneWriteContext&&) = default; - - std::string tmp_index_path; - Lucene::FSDirectoryPtr lucene_dir; - Lucene::IndexWriterPtr index_writer; - Lucene::DocumentPtr doc; - Lucene::FieldPtr field; - Lucene::FieldPtr row_id_field; - }; - - static Result> Create( - const std::string& field_name, const std::shared_ptr& arrow_type, - const std::shared_ptr& file_writer, - const std::map& options, const std::shared_ptr& pool); - - ~LuceneGlobalIndexWriter() override; - - Status AddBatch(::ArrowArray* c_arrow_array) override; - - Result> Finish() override; - - private: - LuceneGlobalIndexWriter(const std::string& field_name, - const std::shared_ptr& arrow_type, - LuceneWriteContext&& write_context, - const std::shared_ptr& file_writer, - const std::map& options, - const std::shared_ptr& pool); - - Result FlushIndexToFinal() const; - - private: - std::shared_ptr pool_; - int32_t row_id_ = 0; - std::string field_name_; - std::shared_ptr arrow_type_; - LuceneWriteContext write_context_; - std::shared_ptr file_writer_; - std::map options_; -}; - -class LuceneGlobalIndexReader : public GlobalIndexReader { - public: - static Result> Create( - const std::string& field_name, const GlobalIndexIOMeta& io_meta, - const std::shared_ptr& file_reader, - const std::map& options, const std::shared_ptr& pool); - - Result> VisitIsNotNull() override { - return CreateAllResult(); - } - - Result> VisitIsNull() override { - return CreateAllResult(); - } - - Result> VisitEqual(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitNotEqual(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitLessThan(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitLessOrEqual(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitGreaterThan(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitGreaterOrEqual( - const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitIn( - const std::vector& literals) override { - return CreateAllResult(); - } - - Result> VisitNotIn( - const std::vector& literals) override { - return CreateAllResult(); - } - - Result> VisitStartsWith(const Literal& prefix) override { - return CreateAllResult(); - } - - Result> VisitEndsWith(const Literal& suffix) override { - return CreateAllResult(); - } - - Result> VisitContains(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitLike(const Literal& literal) override { - return CreateAllResult(); - } - - Result> VisitVectorSearch( - const std::shared_ptr& vector_search) override { - return Status::Invalid( - "LuceneGlobalIndexReader is not supposed to handle vector search query"); - } - - Result> VisitFullTextSearch( - const std::shared_ptr& full_text_search); - - bool IsThreadSafe() const override { - return false; - } - - std::string GetIndexType() const override { - return kIdentifier; - } - - private: - LuceneGlobalIndexReader(const std::wstring& wfield_name, int64_t range_end, - const Lucene::IndexSearcherPtr& searcher) - : range_end_(range_end), wfield_name_(wfield_name), searcher_(searcher) {} - - static std::vector TokenizeQuery(const std::string& query); - - std::shared_ptr CreateAllResult() const { - return BitmapGlobalIndexResult::FromRanges({Range(0, range_end_)}); - } - - private: - int64_t range_end_; - std::wstring wfield_name_; - Lucene::IndexSearcherPtr searcher_; -}; } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp new file mode 100644 index 000000000..187f5c980 --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -0,0 +1,215 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "paimon/global_index/lucene/lucene_global_index_reader.h" +#include "arrow/c/bridge.h" +#include "lucene++/FileUtils.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/global_index/bitmap_vector_search_global_index_result.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/lucene/jieba_analyzer.h" +#include "paimon/global_index/lucene/lucene_collector.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/lucene/lucene_directory.h" +#include "paimon/global_index/lucene/lucene_filter.h" +#include "paimon/global_index/lucene/lucene_utils.h" +#include "paimon/io/data_input_stream.h" + +namespace paimon::lucene { +Result> LuceneGlobalIndexReader::Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool) { + try { + auto meta_bytes = io_meta.metadata; + if (!meta_bytes) { + return Status::Invalid("Lucene global index must have meta data"); + } + std::map write_options; + PAIMON_RETURN_NOT_OK(RapidJsonUtil::FromJsonString( + std::string(meta_bytes->data(), meta_bytes->size()), &write_options)); + + std::map> file_name_to_offset_and_length; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr paimon_input, + file_reader->GetInputStream(io_meta.file_path)); + DataInputStream data_input_stream(paimon_input); + PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input_stream.ReadValue()); + if (version != kVersion) { + return Status::Invalid(fmt::format("LuceneGlobalIndex not support version {}"), + kVersion); + } + PAIMON_ASSIGN_OR_RAISE(int32_t num_files, data_input_stream.ReadValue()); + for (int32_t i = 0; i < num_files; i++) { + PAIMON_ASSIGN_OR_RAISE(int32_t file_name_len, data_input_stream.ReadValue()); + auto file_name_bytes = std::make_shared(file_name_len, pool.get()); + PAIMON_RETURN_NOT_OK(data_input_stream.ReadBytes(file_name_bytes.get())); + std::string file_name(file_name_bytes->data(), file_name_bytes->size()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_len, data_input_stream.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t pos, data_input_stream.GetPos()); + file_name_to_offset_and_length[file_name] = {pos, file_len}; + pos += file_len; + if (i != num_files - 1) { + PAIMON_RETURN_NOT_OK(data_input_stream.Seek(pos)); + } + } + PAIMON_ASSIGN_OR_RAISE( + int32_t read_buffer_size, + OptionsUtils::GetValueFromMap(options, kLuceneReadBufferSize, kDefaultReadBufferSize)); + Lucene::DirectoryPtr lucene_dir = Lucene::newLucene( + PathUtil::GetParentDirPath(io_meta.file_path), file_name_to_offset_and_length, + paimon_input, read_buffer_size); + + Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true); + Lucene::IndexSearcherPtr searcher = Lucene::newLucene(reader); + + PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); + auto jieba = std::make_shared( + dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + + // priority: read options > write options > kDefaultJiebaTokenizeMode + PAIMON_ASSIGN_OR_RAISE( + std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, std::string(""))); + if (tokenize_mode.empty()) { + PAIMON_ASSIGN_OR_RAISE(tokenize_mode, OptionsUtils::GetValueFromMap( + write_options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + } + return std::shared_ptr( + new LuceneGlobalIndexReader(LuceneUtils::StringToWstring(field_name), io_meta.range_end, + searcher, tokenize_mode, jieba)); + } catch (const std::exception& e) { + return Status::Invalid( + fmt::format("create lucene global index reader failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError( + "create lucene global index reader failed, with unknown error."); + } +} + +std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::string& query) const { + std::vector terms; + JiebaTokenizer::CutWithMode(tokenize_mode_, jieba_.get(), query, &terms); + std::vector normalized_terms; + JiebaTokenizer::Normalize(jieba_->extractor.GetStopWords(), &terms, &normalized_terms); + std::vector wterms; + wterms.reserve(normalized_terms.size()); + for (const auto& term : normalized_terms) { + wterms.push_back(LuceneUtils::StringToWstring(term)); + } + return wterms; +} + +Result> LuceneGlobalIndexReader::VisitFullTextSearch( + const std::shared_ptr& full_text_search) { + try { + Lucene::QueryPtr query; + switch (full_text_search->search_type) { + case FullTextSearch::SearchType::MATCH_ALL: + case FullTextSearch::SearchType::MATCH_ANY: { + Lucene::BooleanClause::Occur occur = + full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL + ? Lucene::BooleanClause::Occur::MUST + : Lucene::BooleanClause::Occur::SHOULD; + std::vector query_terms = TokenizeQuery(full_text_search->query); + if (query_terms.size() == 1) { + query = Lucene::newLucene( + Lucene::newLucene(wfield_name_, query_terms[0])); + } else { + auto typed_query = Lucene::newLucene(); + for (const auto& term : query_terms) { + typed_query->add(Lucene::newLucene( + Lucene::newLucene(wfield_name_, term)), + occur); + } + query = typed_query; + } + break; + } + case FullTextSearch::SearchType::PHRASE: { + std::vector query_terms = TokenizeQuery(full_text_search->query); + auto typed_query = Lucene::newLucene(); + for (const auto& term : query_terms) { + typed_query->add(Lucene::newLucene(wfield_name_, term)); + } + query = typed_query; + break; + } + case FullTextSearch::SearchType::PREFIX: { + query = Lucene::newLucene(Lucene::newLucene( + wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + break; + } + case FullTextSearch::SearchType::WILDCARD: { + query = Lucene::newLucene(Lucene::newLucene( + wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + break; + } + default: + return Status::Invalid( + fmt::format("Not support for FullTextSearch SearchType {}", + static_cast(full_text_search->search_type))); + } + Lucene::FilterPtr filter = + full_text_search->pre_filter + ? Lucene::newLucene(&(full_text_search->pre_filter.value())) + : Lucene::FilterPtr(); + + if (full_text_search->limit) { + Lucene::TopDocsPtr results = + searcher_->search(query, filter, full_text_search->limit.value()); + + // prepare BitmapVectorSearchGlobalIndexResult + std::map id_to_score; + for (auto score_doc : results->scoreDocs) { + Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); + std::string row_id_str = + LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); + std::optional row_id = StringUtils::StringToValue(row_id_str); + if (!row_id) { + return Status::Invalid( + fmt::format("parse row id str {} to int failed", row_id_str)); + } + id_to_score[static_cast(row_id.value())] = + static_cast(score_doc->score); + } + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(id_to_score.size()); + for (const auto& [id, score] : id_to_score) { + bitmap.Add(id); + scores.push_back(score); + } + return std::make_shared(std::move(bitmap), + std::move(scores)); + } else { + // with no limit & no score + auto collector = Lucene::newLucene(); + searcher_->search(query, filter, collector); + return std::make_shared( + [collector]() -> Result { return collector->GetBitmap(); }); + } + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("visit term query failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError("visit term query failed, with unknown error."); + } +} + +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h new file mode 100644 index 000000000..57a9a9fce --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -0,0 +1,137 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "cppjieba/Jieba.hpp" +#include "lucene++/LuceneHeaders.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/predicate/full_text_search.h" + +namespace paimon::lucene { +class LuceneGlobalIndexReader : public GlobalIndexReader { + public: + static Result> Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool); + + Result> VisitIsNotNull() override { + return CreateAllResult(); + } + + Result> VisitIsNull() override { + return CreateAllResult(); + } + + Result> VisitEqual(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitNotEqual(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitLessThan(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitLessOrEqual(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitGreaterThan(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitIn( + const std::vector& literals) override { + return CreateAllResult(); + } + + Result> VisitNotIn( + const std::vector& literals) override { + return CreateAllResult(); + } + + Result> VisitStartsWith(const Literal& prefix) override { + return CreateAllResult(); + } + + Result> VisitEndsWith(const Literal& suffix) override { + return CreateAllResult(); + } + + Result> VisitContains(const Literal& literal) override { + return CreateAllResult(); + } + + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid( + "LuceneGlobalIndexReader is not supposed to handle vector search query"); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search); + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return kIdentifier; + } + + private: + LuceneGlobalIndexReader(const std::wstring& wfield_name, int64_t range_end, + const Lucene::IndexSearcherPtr& searcher, + const std::string& tokenize_mode, + const std::shared_ptr& jieba) + : range_end_(range_end), + wfield_name_(wfield_name), + searcher_(searcher), + tokenize_mode_(tokenize_mode), + jieba_(jieba) {} + + std::vector TokenizeQuery(const std::string& query) const; + + std::shared_ptr CreateAllResult() const { + return BitmapGlobalIndexResult::FromRanges({Range(0, range_end_)}); + } + + private: + int64_t range_end_; + std::wstring wfield_name_; + Lucene::IndexSearcherPtr searcher_; + std::string tokenize_mode_; + std::shared_ptr jieba_; +}; +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index f5274f7e2..42b595bb5 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -25,6 +25,8 @@ #include "paimon/core/index/index_path_factory.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/global_index/bitmap_vector_search_global_index_result.h" +#include "paimon/global_index/lucene/lucene_global_index_reader.h" +#include "paimon/global_index/lucene/lucene_global_index_writer.h" #include "paimon/testing/utils/testharness.h" namespace paimon::lucene::test { @@ -286,6 +288,124 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { } } +TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) { + int32_t read_buffer_size = GetParam(); + + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map options = { + {"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.read.buffer-size", std::to_string(read_buffer_size)}, + {"lucene-fts.jieba.tokenize-mode", "query"}, + }; + + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, + R"([ +["QianWen 是一个基于 AI 的智能助手，类似于 Siri 和 Alexa。我们正在用 Python 开发 QianWen 的 Natural Language Understanding 模块，该模块支持多轮对话和意图识别功能，是新一代智能助手的核心技术之一。"], +["最近开源了一个新项目叫ｑｉａｎｗｅｎ（全角字符），功能类似之前的 Qianwen，是一个面向 AI 应用的智能助手。它不仅支持 Machine Learning 和 NLP 技术，还提供了可扩展的开发框架，便于开发者构建自己的智能助手系统。"], +["我们在测试 qianwen-core v1.2 和 ai-engine-alpha 中的 bug，重点优化了 qianwen 的响应速度和稳定性。本次更新增强了核心模块的功能，提升了智能助手的开发效率，并修复了与 NLP 模块相关的多个问题。"], +["AI 助手开发中常用的技术包括 Speech Recognition、Natural Language Processing 和 Recommendation System。我们使用 TensorFlow 和 PyTorch 构建模型，开发了多个智能助手原型，支持语音交互和上下文理解功能，是当前热门的人工智能发展应用方向。"], +["新一代的 AI 助手代号为「千问」，内部命名为 QianwenX-2024，计划在 next quarter 发布。QianwenX 将集成更强的 multimodal 能力，支持图像和文本联合处理，进一步提升智能助手的理解能力和交互体验，是未来智能助手的重要发展方向。"] + ])") + .ValueOrDie(); + + // write index + ASSERT_OK_AND_ASSIGN(auto meta, + WriteGlobalIndex(test_root, data_type_, options, array, Range(0, 4))); + if (read_buffer_size == 10) { + ASSERT_EQ( + std::string(meta.metadata->data(), meta.metadata->size()), + R"({"jieba.tokenize-mode":"query","read.buffer-size":"10","write.omit-term-freq-and-position":"false"})"); + } + + // create reader + ASSERT_OK_AND_ASSIGN(auto reader, + CreateGlobalIndexReader(test_root, data_type_, options, meta)); + auto lucene_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(lucene_reader); + + // test visit + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "模块", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l, 2l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/1, "模块", FullTextSearch::SearchType::MATCH_ANY, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ANY, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l, 1l, 2l, 3l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "发展方向", FullTextSearch::SearchType::PHRASE, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {4l}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "发", FullTextSearch::SearchType::PREFIX, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {3l, 4l}); + } + // test wildcard query + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "*发*", FullTextSearch::SearchType::WILDCARD, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l, 1l, 2l, 3l, 4l}); + } + // test filter + { + ASSERT_OK_AND_ASSIGN(auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ANY, + /*pre_filter=*/RoaringBitmap64::From({1l, 3l, 4l})))); + + CheckResult(result, {1l, 3l}); + } + // test no limit + { + ASSERT_OK_AND_ASSIGN( + auto result, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", + /*limit=*/std::nullopt, "模块技术", FullTextSearch::SearchType::MATCH_ANY, + /*pre_filter=*/std::nullopt))); + CheckResult(result, {0l, 1l, 2l, 3l}); + } +} + INSTANTIATE_TEST_SUITE_P(ReadBufferSize, LuceneGlobalIndexTest, ::testing::ValuesIn(std::vector({10, 100, 1024}))); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp new file mode 100644 index 000000000..a8f6c9248 --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -0,0 +1,242 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/global_index/lucene/lucene_global_index_writer.h" + +#include + +#include "arrow/c/bridge.h" +#include "lucene++/FileUtils.h" +#include "lucene++/NoLockFactory.h" +#include "paimon/common/io/data_output_stream.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/uuid.h" +#include "paimon/global_index/lucene/jieba_analyzer.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/lucene/lucene_utils.h" +namespace paimon::lucene { +#define CHECK_NOT_NULL(pointer, error_msg) \ + do { \ + if (!(pointer)) { \ + return Status::Invalid(error_msg); \ + } \ + } while (0) + +LuceneGlobalIndexWriter::LuceneWriteContext::LuceneWriteContext( + const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, + const Lucene::IndexWriterPtr& _index_writer, const Lucene::DocumentPtr& _doc, + const Lucene::FieldPtr& _field, const Lucene::FieldPtr& _row_id_field) + : tmp_index_path(_tmp_index_path), + lucene_dir(_lucene_dir), + index_writer(_index_writer), + doc(_doc), + field(_field), + row_id_field(_row_id_field) {} + +Result> LuceneGlobalIndexWriter::Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) { + try { + std::string uuid; + if (!UUID::Generate(&uuid)) { + return Status::Invalid("generate uuid for lucene tmp path failed."); + } + // create a local tmp path + std::string tmp_path = PathUtil::JoinPath(std::filesystem::temp_directory_path().string(), + "paimon-lucene-" + uuid); + auto lucene_dir = Lucene::FSDirectory::open(LuceneUtils::StringToWstring(tmp_path), + Lucene::NoLockFactory::getNoLockFactory()); + // TODO(xinyu.lxy): support other tokenizer + // open lucene index writer + PAIMON_ASSIGN_OR_RAISE( + bool omit_term_freq_and_positions, + OptionsUtils::GetValueFromMap(options, kLuceneWriteOmitTermFreqAndPositions, false)); + PAIMON_ASSIGN_OR_RAISE( + std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); + auto jieba = std::make_shared( + dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + JiebaTokenizerContext jieba_context(tokenize_mode, + /*with_position=*/!omit_term_freq_and_positions, jieba, + pool); + auto analyzer = Lucene::newLucene(jieba_context); + Lucene::IndexWriterPtr writer = Lucene::newLucene( + lucene_dir, analyzer, + /*create=*/true, Lucene::IndexWriter::MaxFieldLengthLIMITED); + + // prepare field and document + Lucene::DocumentPtr doc = Lucene::newLucene(); + auto field = Lucene::newLucene(LuceneUtils::StringToWstring(field_name), + kEmptyWstring, Lucene::Field::STORE_NO, + Lucene::Field::INDEX_ANALYZED_NO_NORMS); + auto row_id_field = Lucene::newLucene( + kRowIdFieldWstring, kEmptyWstring, Lucene::Field::STORE_YES, + Lucene::Field::INDEX_NOT_ANALYZED_NO_NORMS); + field->setOmitTermFreqAndPositions(omit_term_freq_and_positions); + row_id_field->setOmitTermFreqAndPositions(true); + doc->add(field); + doc->add(row_id_field); + return std::shared_ptr(new LuceneGlobalIndexWriter( + field_name, arrow_type, + LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field, row_id_field), file_writer, + options, pool)); + } catch (const std::exception& e) { + return Status::Invalid( + fmt::format("create lucene global index writer failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError( + "create lucene global index writer failed, with unknown error."); + } +} + +LuceneGlobalIndexWriter::LuceneGlobalIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + LuceneWriteContext&& write_context, const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) + : pool_(pool), + field_name_(field_name), + arrow_type_(arrow_type), + write_context_(std::move(write_context)), + file_writer_(file_writer), + options_(options) {} + +LuceneGlobalIndexWriter::~LuceneGlobalIndexWriter() { + try { + [[maybe_unused]] bool ec = Lucene::FileUtils::removeDirectory( + LuceneUtils::StringToWstring(write_context_.tmp_index_path)); + } catch (...) { + // do nothing + } +} + +Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(arrow_array, arrow_type_)); + auto struct_array = std::dynamic_pointer_cast(array); + CHECK_NOT_NULL(struct_array, "invalid input array in LuceneIndexWriter, must be struct array"); + auto field_array = struct_array->GetFieldByName(field_name_); + CHECK_NOT_NULL( + field_array, + fmt::format("invalid input array in LuceneIndexWriter, field {} not in input array", + field_name_)); + auto string_array = std::dynamic_pointer_cast(field_array); + CHECK_NOT_NULL( + string_array, + fmt::format( + "invalid input array in LuceneIndexWriter, field array {} is not a string array", + field_name_)); + try { + for (int64_t i = 0; i < string_array->length(); i++) { + if (string_array->IsNull(i)) { + write_context_.field->setValue(kEmptyWstring); + } else { + auto view = string_array->Value(i); + write_context_.field->setValue(LuceneUtils::StringToWstring(view)); + } + write_context_.row_id_field->setValue( + LuceneUtils::StringToWstring(std::to_string(row_id_++))); + write_context_.index_writer->addDocument(write_context_.doc); + } + } catch (const std::exception& e) { + return Status::Invalid(fmt::format( + "add batch for lucene global index writer failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError( + "add batch for lucene global index writer failed, with unknown error."); + } + return Status::OK(); +} + +Result LuceneGlobalIndexWriter::FlushIndexToFinal() const { + try { + // flush index to tmp dir + write_context_.index_writer->optimize(); + write_context_.index_writer->close(); + + // list tmp dir + auto tmp_file_names = write_context_.lucene_dir->listAll(); + PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer_->NewFileName(kIdentifier)); + // prepare output from file_writer + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, + file_writer_->NewOutputStream(index_file_name)); + DataOutputStream data_output_stream(out); + PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(kVersion)); + PAIMON_RETURN_NOT_OK( + data_output_stream.WriteValue(static_cast(tmp_file_names.size()))); + // read all data from index files and write to target output + auto buffer = std::make_shared(kDefaultReadBufferSize, pool_.get()); + for (const auto& wfile_name : tmp_file_names) { + auto file_name = LuceneUtils::WstringToString(wfile_name); + PAIMON_RETURN_NOT_OK( + data_output_stream.WriteValue(static_cast(file_name.size()))); + PAIMON_RETURN_NOT_OK( + data_output_stream.WriteBytes(std::make_shared(file_name, pool_.get()))); + int64_t file_length = write_context_.lucene_dir->fileLength(wfile_name); + PAIMON_RETURN_NOT_OK(data_output_stream.WriteValue(file_length)); + + Lucene::IndexInputPtr input = write_context_.lucene_dir->openInput(wfile_name); + int64_t total_write_size = 0; + while (total_write_size < file_length) { + int64_t current_write_size = std::min(file_length - total_write_size, + static_cast(kDefaultReadBufferSize)); + input->readBytes(reinterpret_cast(buffer->data()), /*offset=*/0, + static_cast(current_write_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t actual_write_size, + out->Write(buffer->data(), static_cast(current_write_size))); + if (static_cast(actual_write_size) != current_write_size) { + return Status::Invalid( + fmt::format("invalid write, try to write {} while actual write {}", + current_write_size, actual_write_size)); + } + total_write_size += current_write_size; + } + input->close(); + } + PAIMON_RETURN_NOT_OK(out->Flush()); + PAIMON_RETURN_NOT_OK(out->Close()); + write_context_.lucene_dir->close(); + return index_file_name; + } catch (const std::exception& e) { + return Status::Invalid( + fmt::format("finish for lucene global index writer failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError( + "finish for lucene global index writer failed, with unknown error."); + } +} + +Result> LuceneGlobalIndexWriter::Finish() { + PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, FlushIndexToFinal()); + // prepare global index meta + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name)); + std::string options_json; + PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); + auto meta_bytes = std::make_shared(options_json, pool_.get()); + GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, + /*range_end=*/static_cast(row_id_) - 1, + /*metadata=*/meta_bytes); + return std::vector({meta}); +} + +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h new file mode 100644 index 000000000..798f25c4f --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -0,0 +1,78 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include "arrow/type.h" +#include "cppjieba/Jieba.hpp" +#include "lucene++/LuceneHeaders.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon//global_index/io/global_index_file_writer.h" + +namespace paimon::lucene { +class LuceneGlobalIndexWriter : public GlobalIndexWriter { + public: + struct LuceneWriteContext { + LuceneWriteContext(const std::string& _tmp_index_path, + const Lucene::FSDirectoryPtr& _lucene_dir, + const Lucene::IndexWriterPtr& _index_writer, + const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field, + const Lucene::FieldPtr& _row_id_field); + + LuceneWriteContext(LuceneWriteContext&&) = default; + LuceneWriteContext& operator=(LuceneWriteContext&&) = default; + + std::string tmp_index_path; + Lucene::FSDirectoryPtr lucene_dir; + Lucene::IndexWriterPtr index_writer; + Lucene::DocumentPtr doc; + Lucene::FieldPtr field; + Lucene::FieldPtr row_id_field; + }; + + static Result> Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool); + + ~LuceneGlobalIndexWriter() override; + + Status AddBatch(::ArrowArray* c_arrow_array) override; + + Result> Finish() override; + + private: + LuceneGlobalIndexWriter(const std::string& field_name, + const std::shared_ptr& arrow_type, + LuceneWriteContext&& write_context, + const std::shared_ptr& file_writer, + const std::map& options, + const std::shared_ptr& pool); + + Result FlushIndexToFinal() const; + + private: + std::shared_ptr pool_; + int32_t row_id_ = 0; + std::string field_name_; + std::shared_ptr arrow_type_; + LuceneWriteContext write_context_; + std::shared_ptr file_writer_; + std::map options_; +}; + +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_utils.h b/src/paimon/global_index/lucene/lucene_utils.h index 70a35dcc9..aa296f5e8 100644 --- a/src/paimon/global_index/lucene/lucene_utils.h +++ b/src/paimon/global_index/lucene/lucene_utils.h @@ -14,8 +14,10 @@ * limitations under the License. */ #pragma once - +#include "fmt/format.h" #include "lucene++/StringUtils.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/result.h" namespace paimon::lucene { class LuceneUtils { public: @@ -31,5 +33,17 @@ class LuceneUtils { static std::string WstringToString(const Lucene::String& wstr) { return Lucene::StringUtils::toUTF8(wstr); } + + static Result GetJiebaDictionaryDir() { + const char* env_dir = std::getenv(kJiebaDictDirEnv); + if (env_dir && *env_dir != '\0') { + return std::string(env_dir); + } +#ifdef JIEBA_TEST_DICT_DIR + return std::string(JIEBA_TEST_DICT_DIR); +#endif + return Status::Invalid( + fmt::format("cannot get dictionary dir for jieba, must set {}", kJiebaDictDirEnv)); + } }; } // namespace paimon::lucene diff --git a/third_party/versions.txt b/third_party/versions.txt index e834f9b88..76770d571 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -77,6 +77,8 @@ PAIMON_LUCENE_BUILD_VERSION=3.0.9 PAIMON_LUCENE_BUILD_SHA256_CHECKSUM=4e69e29d5d79a976498ef71eab70c9c88c7014708be4450a9fda7780fe93584e PAIMON_LUCENE_PKG_NAME=rel_${PAIMON_LUCENE_BUILD_VERSION}.tar.gz +PAIMON_JIEBA_BUILD_VERSION=v5.6.0 + # The first field is the name of the environment variable expected by cmake. # This _must_ match what is defined. The second field is the name of the # generated archive file. The third field is the url of the project for the From d124916c2bccc67d88df604ce31152845a664ac1 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 3 Feb 2026 21:24:37 +0800 Subject: [PATCH 02/12] fix --- CMakeLists.txt | 2 +- cmake_modules/ThirdpartyToolchain.cmake | 11 +++++------ cmake_modules/jieba.diff | 2 +- src/paimon/global_index/lucene/CMakeLists.txt | 17 +++++++++-------- .../global_index/lucene/jieba_analyzer.cpp | 18 +++++++++++++----- .../global_index/lucene/jieba_analyzer.h | 5 +++++ .../lucene/lucene_global_index_reader.h | 4 ++++ 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d9d26844..de2f16cfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,7 +303,7 @@ if(PAIMON_ENABLE_LUCENE) PATTERN "user.dict.utf8" PATTERN "pos_dict" PATTERN ".git*" EXCLUDE - PATTERN "*.md" EXCLUDE) + PATTERN "*.md" EXCLUDE) endif() install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 263e13b46..46ba2c451 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -352,12 +352,10 @@ macro(build_jieba) file(MAKE_DIRECTORY ${JIEBA_DICT_DIR}) set(JIEBA_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DENABLE_TEST=OFF" - "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF" + ${EP_COMMON_CMAKE_ARGS} "-DENABLE_TEST=OFF" "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF" "-DCMAKE_INSTALL_PREFIX=${JIEBA_INSTALL}") - set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff") + set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff") externalproject_add(jieba_ep ${EP_COMMON_OPTIONS} GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git @@ -370,13 +368,14 @@ macro(build_jieba) PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" INSTALL_COMMAND bash -c - "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/" + "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/" ) # The include directory must exist before it is referenced by a target. include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) add_library(jieba INTERFACE IMPORTED) - target_include_directories(jieba SYSTEM INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}") + target_include_directories(jieba SYSTEM + INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}") add_dependencies(jieba jieba_ep) endmacro() diff --git a/cmake_modules/jieba.diff b/cmake_modules/jieba.diff index 04478ff6a..d74c3f605 100644 --- a/cmake_modules/jieba.diff +++ b/cmake_modules/jieba.diff @@ -10,7 +10,7 @@ index 24b2c40..c7c6a94 100644 + const std::unordered_set& GetStopWords() const { + return stopWords_; + } -+ ++ private: void LoadIdfDict(const std::string& idfPath) { std::ifstream ifs(idfPath.c_str()); diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index a032f3a10..b4c701a3d 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -13,12 +13,13 @@ # limitations under the License. if(PAIMON_ENABLE_LUCENE) - set(PAIMON_LUCENE lucene_global_index.cpp - lucene_directory.cpp - jieba_analyzer.cpp - lucene_global_index_writer.cpp - lucene_global_index_reader.cpp - lucene_global_index_factory.cpp) + set(PAIMON_LUCENE + lucene_global_index.cpp + lucene_directory.cpp + jieba_analyzer.cpp + lucene_global_index_writer.cpp + lucene_global_index_reader.cpp + lucene_global_index_factory.cpp) add_paimon_lib(paimon_lucene_index SOURCES @@ -61,7 +62,7 @@ if(PAIMON_ENABLE_LUCENE) "-Wl,--no-whole-archive" ${GTEST_LINK_TOOLCHAIN}) - target_compile_definitions(paimon-lucene-index-test PRIVATE - JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + target_compile_definitions(paimon-lucene-index-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() endif() diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index f09692921..7a3606c4c 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -29,19 +29,18 @@ JiebaTokenizerContext::JiebaTokenizerContext(const std::string& _tokenize_mode, jieba(_jieba) {} JiebaTokenizer::JiebaTokenizer(const JiebaTokenizerContext& context, const Lucene::ReaderPtr& input) - : Lucene::Tokenizer(), context_(context) { - this->input = input; + : Lucene::Tokenizer(input), context_(context) { term_att_ = addAttribute(); pos_att_ = addAttribute(); buffer_ = static_cast( - context_.pool->Malloc(context_.buffer_size * sizeof(wchar_t), /*aligment=*/8)); + context_.pool->Malloc(context_.buffer_size * sizeof(wchar_t), /*alignment=*/8)); } JiebaTokenizer::~JiebaTokenizer() { if (buffer_) { context_.pool->Free(reinterpret_cast(buffer_), context_.buffer_size * sizeof(wchar_t), - /*aligment=*/8); + /*alignment=*/8); buffer_ = nullptr; } } @@ -98,7 +97,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words // to lower case bool is_alphanumeric = true; - for (const unsigned char& c : term) { + for (const auto& c : term) { if (!std::isalnum(c)) { is_alphanumeric = false; break; @@ -113,6 +112,15 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words void JiebaTokenizer::reset() { Lucene::Tokenizer::reset(); + InnerReset(); +} + +void JiebaTokenizer::reset(const Lucene::ReaderPtr& input) { + Lucene::Tokenizer::reset(input); + InnerReset(); +} + +void JiebaTokenizer::InnerReset() { terms_.clear(); normalized_terms_.clear(); term_index_ = 0; diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index c74cb31a9..40a3ff7d8 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -46,6 +46,8 @@ class JiebaTokenizer : public Lucene::Tokenizer { bool incrementToken() override; + void reset(const Lucene::ReaderPtr& input) override; + void reset() override; static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, @@ -55,6 +57,9 @@ class JiebaTokenizer : public Lucene::Tokenizer { static void Normalize(const std::unordered_set& stop_words, std::vector* input, std::vector* output); + private: + void InnerReset(); + private: JiebaTokenizerContext context_; size_t term_index_ = 0; diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index 57a9a9fce..11cafe113 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -93,6 +93,10 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { return CreateAllResult(); } + Result> VisitLike(const Literal& literal) override { + return CreateAllResult(); + } + Result> VisitVectorSearch( const std::shared_ptr& vector_search) override { return Status::Invalid( From 5a46a4a8259011bd816a328bba01871c0d01b1d7 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 3 Feb 2026 21:32:33 +0800 Subject: [PATCH 03/12] fix --- cmake_modules/ThirdpartyToolchain.cmake | 2 +- src/paimon/global_index/lucene/CMakeLists.txt | 2 +- src/paimon/global_index/lucene/lucene_defs.h | 6 +++--- .../global_index/lucene/lucene_global_index_reader.cpp | 3 ++- src/paimon/global_index/lucene/lucene_global_index_writer.h | 5 ++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 46ba2c451..edffa1d67 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -369,7 +369,7 @@ macro(build_jieba) "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" INSTALL_COMMAND bash -c "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/" - ) + ) # The include directory must exist before it is referenced by a target. include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index b4c701a3d..e8143e066 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -63,6 +63,6 @@ if(PAIMON_ENABLE_LUCENE) ${GTEST_LINK_TOOLCHAIN}) target_compile_definitions(paimon-lucene-index-test - PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() endif() diff --git a/src/paimon/global_index/lucene/lucene_defs.h b/src/paimon/global_index/lucene/lucene_defs.h index 71eaa9547..ed576a8de 100644 --- a/src/paimon/global_index/lucene/lucene_defs.h +++ b/src/paimon/global_index/lucene/lucene_defs.h @@ -36,7 +36,7 @@ static inline const char kLuceneWriteOmitTermFreqAndPositions[] = static inline const char kJiebaDictDirEnv[] = "PAIMON_JIEBA_DICT_DIR"; -static inline const char kDefaultJiebaTokenizeMode[] = "mix"; -// default is "mix". Values can be "mp", "hmm", "mix", "full", "query". -static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode"; +static inline const char kDefaultJiebaTokenizeMode[] = "mix"; +// default is "mix". Values can be "mp", "hmm", "mix", "full", "query". +static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode"; } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 187f5c980..06d9a69fc 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -14,13 +14,14 @@ * limitations under the License. */ #include "paimon/global_index/lucene/lucene_global_index_reader.h" + #include "arrow/c/bridge.h" #include "lucene++/FileUtils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/rapidjson_util.h" -#include "paimon/global_index/bitmap_vector_search_global_index_result.h" #include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_vector_search_global_index_result.h" #include "paimon/global_index/lucene/jieba_analyzer.h" #include "paimon/global_index/lucene/lucene_collector.h" #include "paimon/global_index/lucene/lucene_defs.h" diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index 798f25c4f..b7bfd7776 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -18,10 +18,9 @@ #include "arrow/type.h" #include "cppjieba/Jieba.hpp" #include "lucene++/LuceneHeaders.h" -#include "paimon/global_index/lucene/lucene_defs.h" -#include "paimon/global_index/global_index_writer.h" -#include "paimon/global_index/global_index_writer.h" #include "paimon//global_index/io/global_index_file_writer.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/lucene/lucene_defs.h" namespace paimon::lucene { class LuceneGlobalIndexWriter : public GlobalIndexWriter { From 5c7055dcb7164dd41fe67041ae110dc88e7a6fac Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 3 Feb 2026 22:02:48 +0800 Subject: [PATCH 04/12] fix --- src/paimon/global_index/lucene/jieba_analyzer.h | 2 +- src/paimon/global_index/lucene/lucene_api_test.cpp | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index 40a3ff7d8..fe7506f0e 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -72,7 +72,7 @@ class JiebaTokenizer : public Lucene::Tokenizer { class JiebaAnalyzer : public Lucene::Analyzer { public: - JiebaAnalyzer(const JiebaTokenizerContext& context) : context_(context) {} + explicit JiebaAnalyzer(const JiebaTokenizerContext& context) : context_(context) {} ~JiebaAnalyzer() override = default; diff --git a/src/paimon/global_index/lucene/lucene_api_test.cpp b/src/paimon/global_index/lucene/lucene_api_test.cpp index fd4f274f5..fd01b3cc7 100644 --- a/src/paimon/global_index/lucene/lucene_api_test.cpp +++ b/src/paimon/global_index/lucene/lucene_api_test.cpp @@ -41,10 +41,13 @@ class LuceneInterfaceTest : public ::testing::Test { return doc_id; } int32_t docID() override { + if (cursor_ >= ids_.size()) { + return Lucene::DocIdSetIterator::NO_MORE_DOCS; + } return ids_[cursor_]; } int32_t nextDoc() override { - if (cursor_ == ids_.size()) { + if (cursor_ >= ids_.size()) { return Lucene::DocIdSetIterator::NO_MORE_DOCS; } return ids_[cursor_++]; @@ -163,14 +166,14 @@ class LuceneInterfaceTest : public ::testing::Test { } ASSERT_EQ(expected_doc_id_vec.size(), results->scoreDocs.size()); - std::vector resule_doc_id_vec; + std::vector result_doc_id_vec; std::vector result_doc_id_content_vec; for (auto score_doc : results->scoreDocs) { Lucene::DocumentPtr result_doc = context->searcher->doc(score_doc->doc); - resule_doc_id_vec.push_back(score_doc->doc); + result_doc_id_vec.push_back(score_doc->doc); result_doc_id_content_vec.push_back(result_doc->get(L"id")); } - ASSERT_EQ(resule_doc_id_vec, expected_doc_id_vec); + ASSERT_EQ(result_doc_id_vec, expected_doc_id_vec); ASSERT_EQ(result_doc_id_content_vec, expected_doc_id_content_vec); } }; From 07326a406247f6a62de8756f275ef305fc158ff2 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 11:41:53 +0800 Subject: [PATCH 05/12] fix1141 --- src/paimon/global_index/lucene/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index e8143e066..295bc351d 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -42,6 +42,10 @@ if(PAIMON_ENABLE_LUCENE) SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) + if(PAIMON_BUILD_TESTS) + target_compile_definitions(paimon_lucene_index_objlib + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + endif() if(PAIMON_BUILD_TESTS) add_paimon_test(lucene_index_test SOURCES @@ -61,8 +65,5 @@ if(PAIMON_ENABLE_LUCENE) paimon_lucene_index_static "-Wl,--no-whole-archive" ${GTEST_LINK_TOOLCHAIN}) - - target_compile_definitions(paimon-lucene-index-test - PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() endif() From 3587aebcf221ee618df5570edb04c5970cdfd963 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 13:38:31 +0800 Subject: [PATCH 06/12] fix --- src/paimon/global_index/lucene/CMakeLists.txt | 7 +++---- src/paimon/global_index/lucene/jieba_analyzer.h | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 295bc351d..728564d6b 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -42,10 +42,6 @@ if(PAIMON_ENABLE_LUCENE) SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) - if(PAIMON_BUILD_TESTS) - target_compile_definitions(paimon_lucene_index_objlib - PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - endif() if(PAIMON_BUILD_TESTS) add_paimon_test(lucene_index_test SOURCES @@ -65,5 +61,8 @@ if(PAIMON_ENABLE_LUCENE) paimon_lucene_index_static "-Wl,--no-whole-archive" ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-lucene-index-test + PUBLIC JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + endif() endif() diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index fe7506f0e..022eadcd2 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -53,7 +53,7 @@ class JiebaTokenizer : public Lucene::Tokenizer { static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba, const std::string& str, std::vector* terms_ptr); - // inplace to lower to avoid data copy + // In-place converts each string in `input` to lowercase to avoid data copying. static void Normalize(const std::unordered_set& stop_words, std::vector* input, std::vector* output); From 1495b84cf748c5b7165c7b350e794fc43cb96ff2 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 13:51:33 +0800 Subject: [PATCH 07/12] fix --- src/paimon/global_index/lucene/lucene_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/global_index/lucene/lucene_utils.h b/src/paimon/global_index/lucene/lucene_utils.h index aa296f5e8..33b9996d1 100644 --- a/src/paimon/global_index/lucene/lucene_utils.h +++ b/src/paimon/global_index/lucene/lucene_utils.h @@ -43,7 +43,7 @@ class LuceneUtils { return std::string(JIEBA_TEST_DICT_DIR); #endif return Status::Invalid( - fmt::format("cannot get dictionary dir for jieba, must set {}", kJiebaDictDirEnv)); + fmt::format("cannot get dictionary dir for jieba, must set env {}", kJiebaDictDirEnv)); } }; } // namespace paimon::lucene From c4e9208af8c198bdb2fcd15715bae328423accab Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 14:53:38 +0800 Subject: [PATCH 08/12] fix0253 --- src/paimon/global_index/lucene/CMakeLists.txt | 9 ++++-- .../global_index/lucene/lucene_utils.cpp | 31 +++++++++++++++++++ src/paimon/global_index/lucene/lucene_utils.h | 12 +------ 3 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 src/paimon/global_index/lucene/lucene_utils.cpp diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 728564d6b..ae0fca337 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -16,6 +16,7 @@ if(PAIMON_ENABLE_LUCENE) set(PAIMON_LUCENE lucene_global_index.cpp lucene_directory.cpp + lucene_utils.cpp jieba_analyzer.cpp lucene_global_index_writer.cpp lucene_global_index_reader.cpp @@ -41,6 +42,12 @@ if(PAIMON_ENABLE_LUCENE) paimon_shared SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) + if(PAIMON_BUILD_TESTS) + target_compile_definitions(paimon_lucene_index_objlib + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + endif() + if(PAIMON_BUILD_TESTS) add_paimon_test(lucene_index_test @@ -61,8 +68,6 @@ if(PAIMON_ENABLE_LUCENE) paimon_lucene_index_static "-Wl,--no-whole-archive" ${GTEST_LINK_TOOLCHAIN}) - target_compile_definitions(paimon-lucene-index-test - PUBLIC JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() endif() diff --git a/src/paimon/global_index/lucene/lucene_utils.cpp b/src/paimon/global_index/lucene/lucene_utils.cpp new file mode 100644 index 000000000..d9c6f08a0 --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_utils.cpp @@ -0,0 +1,31 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "paimon/global_index/lucene/lucene_utils.h" +namespace paimon::lucene { + +Result LuceneUtils::GetJiebaDictionaryDir() { + const char* env_dir = std::getenv(kJiebaDictDirEnv); + if (env_dir && *env_dir != '\0') { + return std::string(env_dir); + } +#ifdef JIEBA_TEST_DICT_DIR + return std::string(JIEBA_TEST_DICT_DIR); +#endif + return Status::Invalid( + fmt::format("cannot get dictionary dir for jieba, must set env {}", kJiebaDictDirEnv)); +} + +} // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_utils.h b/src/paimon/global_index/lucene/lucene_utils.h index 33b9996d1..473628983 100644 --- a/src/paimon/global_index/lucene/lucene_utils.h +++ b/src/paimon/global_index/lucene/lucene_utils.h @@ -34,16 +34,6 @@ class LuceneUtils { return Lucene::StringUtils::toUTF8(wstr); } - static Result GetJiebaDictionaryDir() { - const char* env_dir = std::getenv(kJiebaDictDirEnv); - if (env_dir && *env_dir != '\0') { - return std::string(env_dir); - } -#ifdef JIEBA_TEST_DICT_DIR - return std::string(JIEBA_TEST_DICT_DIR); -#endif - return Status::Invalid( - fmt::format("cannot get dictionary dir for jieba, must set env {}", kJiebaDictDirEnv)); - } + static Result GetJiebaDictionaryDir(); }; } // namespace paimon::lucene From 950de262001aa2334392ed521952f84794a3ae9f Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 14:57:06 +0800 Subject: [PATCH 09/12] fix0256 --- src/paimon/global_index/lucene/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index ae0fca337..50a4e8032 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -16,7 +16,7 @@ if(PAIMON_ENABLE_LUCENE) set(PAIMON_LUCENE lucene_global_index.cpp lucene_directory.cpp - lucene_utils.cpp + lucene_utils.cpp jieba_analyzer.cpp lucene_global_index_writer.cpp lucene_global_index_reader.cpp @@ -47,7 +47,6 @@ if(PAIMON_ENABLE_LUCENE) PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") endif() - if(PAIMON_BUILD_TESTS) add_paimon_test(lucene_index_test From c2cdf99b5215d20884df394b761bfe2e2c66a103 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 4 Feb 2026 16:36:03 +0800 Subject: [PATCH 10/12] fix 0436 --- src/paimon/global_index/lucene/jieba_analyzer.cpp | 6 ++++-- src/paimon/global_index/lucene/lucene_api_test.cpp | 8 ++++---- .../global_index/lucene/lucene_global_index_writer.h | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index 7a3606c4c..89320d038 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -98,13 +98,15 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words // to lower case bool is_alphanumeric = true; for (const auto& c : term) { - if (!std::isalnum(c)) { + if (!std::isalnum(static_cast(c))) { is_alphanumeric = false; break; } } if (is_alphanumeric && !term.empty()) { - std::transform(term.begin(), term.end(), term.begin(), ::tolower); + std::transform(term.begin(), term.end(), term.begin(), [](char ch) { + return static_cast(std::tolower(static_cast(ch))); + }); } output.emplace_back(term.data(), term.length()); } diff --git a/src/paimon/global_index/lucene/lucene_api_test.cpp b/src/paimon/global_index/lucene/lucene_api_test.cpp index fd01b3cc7..6affc7354 100644 --- a/src/paimon/global_index/lucene/lucene_api_test.cpp +++ b/src/paimon/global_index/lucene/lucene_api_test.cpp @@ -132,14 +132,14 @@ class LuceneInterfaceTest : public ::testing::Test { context->writer->addDocument(context->doc); } - struct ReadConext { + struct ReadContext { Lucene::IndexReaderPtr reader; Lucene::IndexSearcherPtr searcher; Lucene::QueryParserPtr parser; }; - ReadConext CreateReadContext(const Lucene::DirectoryPtr& lucene_dir, - const Lucene::AnalyzerPtr& analyzer) const { + ReadContext CreateReadContext(const Lucene::DirectoryPtr& lucene_dir, + const Lucene::AnalyzerPtr& analyzer) const { auto lucene_analyzer = analyzer ? analyzer : Lucene::newLucene( Lucene::LuceneVersion::LUCENE_CURRENT); @@ -155,7 +155,7 @@ class LuceneInterfaceTest : public ::testing::Test { const std::optional> selected_id, const std::vector& expected_doc_id_vec, const std::vector& expected_doc_id_content_vec, - ReadConext* context) const { + ReadContext* context) const { Lucene::QueryPtr query = context->parser->parse(query_str); Lucene::TopDocsPtr results; if (selected_id) { diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index b7bfd7776..d945327eb 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -18,8 +18,8 @@ #include "arrow/type.h" #include "cppjieba/Jieba.hpp" #include "lucene++/LuceneHeaders.h" -#include "paimon//global_index/io/global_index_file_writer.h" #include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/io/global_index_file_writer.h" #include "paimon/global_index/lucene/lucene_defs.h" namespace paimon::lucene { From 80c8c79253563c5c3bc595e381b77c1e2a316fc0 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 5 Feb 2026 09:32:07 +0800 Subject: [PATCH 11/12] fix --- .../lucene/lucene_global_index_reader.cpp | 179 +++++++++++------- .../lucene/lucene_global_index_reader.h | 20 ++ 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 06d9a69fc..fa89fa19f 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -117,6 +117,111 @@ std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::stri return wterms; } +Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery( + const std::shared_ptr& full_text_search) const { + assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL || + full_text_search->search_type == FullTextSearch::SearchType::MATCH_ANY); + Lucene::BooleanClause::Occur occur = + full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL + ? Lucene::BooleanClause::Occur::MUST + : Lucene::BooleanClause::Occur::SHOULD; + std::vector query_terms = TokenizeQuery(full_text_search->query); + if (query_terms.size() == 1) { + return Lucene::newLucene( + Lucene::newLucene(wfield_name_, query_terms[0])); + } else { + auto typed_query = Lucene::newLucene(); + for (const auto& term : query_terms) { + typed_query->add(Lucene::newLucene( + Lucene::newLucene(wfield_name_, term)), + occur); + } + return typed_query; + } +} + +Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery( + const std::shared_ptr& full_text_search) const { + assert(full_text_search->search_type == FullTextSearch::SearchType::PHRASE); + std::vector query_terms = TokenizeQuery(full_text_search->query); + auto typed_query = Lucene::newLucene(); + for (const auto& term : query_terms) { + typed_query->add(Lucene::newLucene(wfield_name_, term)); + } + return typed_query; +} + +Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery( + const std::shared_ptr& full_text_search) const { + assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX); + return Lucene::newLucene(Lucene::newLucene( + wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); +} + +Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery( + const std::shared_ptr& full_text_search) const { + assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD); + return Lucene::newLucene(Lucene::newLucene( + wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); +} + +Result> LuceneGlobalIndexReader::SearchWithLimit( + const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const { + assert(full_text_search->limit); + try { + Lucene::FilterPtr filter = + full_text_search->pre_filter + ? Lucene::newLucene(&(full_text_search->pre_filter.value())) + : Lucene::FilterPtr(); + + Lucene::TopDocsPtr results = + searcher_->search(query, filter, full_text_search->limit.value()); + + // prepare BitmapVectorSearchGlobalIndexResult + std::map id_to_score; + for (auto score_doc : results->scoreDocs) { + Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); + std::string row_id_str = + LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); + std::optional row_id = StringUtils::StringToValue(row_id_str); + if (!row_id) { + return Status::Invalid( + fmt::format("parse row id str {} to int failed", row_id_str)); + } + id_to_score[static_cast(row_id.value())] = + static_cast(score_doc->score); + } + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(id_to_score.size()); + for (const auto& [id, score] : id_to_score) { + bitmap.Add(id); + scores.push_back(score); + } + return std::make_shared(std::move(bitmap), + std::move(scores)); + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("visit term query failed, with {} error.", e.what())); + } catch (...) { + return Status::UnknownError("visit term query failed, with unknown error."); + } +} + +Result> LuceneGlobalIndexReader::SearchWithNoLimit( + const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const { + assert(!full_text_search->limit); + Lucene::FilterPtr filter = + full_text_search->pre_filter + ? Lucene::newLucene(&(full_text_search->pre_filter.value())) + : Lucene::FilterPtr(); + + // with no limit & no score + auto collector = Lucene::newLucene(); + searcher_->search(query, filter, collector); + return std::make_shared( + [collector]() -> Result { return collector->GetBitmap(); }); +} + Result> LuceneGlobalIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { try { @@ -124,42 +229,19 @@ Result> LuceneGlobalIndexReader::VisitFullTex switch (full_text_search->search_type) { case FullTextSearch::SearchType::MATCH_ALL: case FullTextSearch::SearchType::MATCH_ANY: { - Lucene::BooleanClause::Occur occur = - full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL - ? Lucene::BooleanClause::Occur::MUST - : Lucene::BooleanClause::Occur::SHOULD; - std::vector query_terms = TokenizeQuery(full_text_search->query); - if (query_terms.size() == 1) { - query = Lucene::newLucene( - Lucene::newLucene(wfield_name_, query_terms[0])); - } else { - auto typed_query = Lucene::newLucene(); - for (const auto& term : query_terms) { - typed_query->add(Lucene::newLucene( - Lucene::newLucene(wfield_name_, term)), - occur); - } - query = typed_query; - } + query = ConstructMatchQuery(full_text_search); break; } case FullTextSearch::SearchType::PHRASE: { - std::vector query_terms = TokenizeQuery(full_text_search->query); - auto typed_query = Lucene::newLucene(); - for (const auto& term : query_terms) { - typed_query->add(Lucene::newLucene(wfield_name_, term)); - } - query = typed_query; + query = ConstructPhraseQuery(full_text_search); break; } case FullTextSearch::SearchType::PREFIX: { - query = Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + query = ConstructPrefixQuery(full_text_search); break; } case FullTextSearch::SearchType::WILDCARD: { - query = Lucene::newLucene(Lucene::newLucene( - wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); + query = ConstructWildCardQuery(full_text_search); break; } default: @@ -167,49 +249,16 @@ Result> LuceneGlobalIndexReader::VisitFullTex fmt::format("Not support for FullTextSearch SearchType {}", static_cast(full_text_search->search_type))); } - Lucene::FilterPtr filter = - full_text_search->pre_filter - ? Lucene::newLucene(&(full_text_search->pre_filter.value())) - : Lucene::FilterPtr(); - if (full_text_search->limit) { - Lucene::TopDocsPtr results = - searcher_->search(query, filter, full_text_search->limit.value()); - - // prepare BitmapVectorSearchGlobalIndexResult - std::map id_to_score; - for (auto score_doc : results->scoreDocs) { - Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); - std::string row_id_str = - LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); - std::optional row_id = StringUtils::StringToValue(row_id_str); - if (!row_id) { - return Status::Invalid( - fmt::format("parse row id str {} to int failed", row_id_str)); - } - id_to_score[static_cast(row_id.value())] = - static_cast(score_doc->score); - } - RoaringBitmap64 bitmap; - std::vector scores; - scores.reserve(id_to_score.size()); - for (const auto& [id, score] : id_to_score) { - bitmap.Add(id); - scores.push_back(score); - } - return std::make_shared(std::move(bitmap), - std::move(scores)); + return SearchWithLimit(query, full_text_search); } else { - // with no limit & no score - auto collector = Lucene::newLucene(); - searcher_->search(query, filter, collector); - return std::make_shared( - [collector]() -> Result { return collector->GetBitmap(); }); + return SearchWithNoLimit(query, full_text_search); } } catch (const std::exception& e) { - return Status::Invalid(fmt::format("visit term query failed, with {} error.", e.what())); + return Status::Invalid( + fmt::format("visit full text search failed, with {} error.", e.what())); } catch (...) { - return Status::UnknownError("visit term query failed, with unknown error."); + return Status::UnknownError("visit full text search failed, with unknown error."); } } diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index 11cafe113..de5898420 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -131,6 +131,26 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { return BitmapGlobalIndexResult::FromRanges({Range(0, range_end_)}); } + Lucene::QueryPtr ConstructMatchQuery( + const std::shared_ptr& full_text_search) const; + + Lucene::QueryPtr ConstructPhraseQuery( + const std::shared_ptr& full_text_search) const; + + Lucene::QueryPtr ConstructPrefixQuery( + const std::shared_ptr& full_text_search) const; + + Lucene::QueryPtr ConstructWildCardQuery( + const std::shared_ptr& full_text_search) const; + + Result> SearchWithLimit( + const Lucene::QueryPtr& query, + const std::shared_ptr& full_text_search) const; + + Result> SearchWithNoLimit( + const Lucene::QueryPtr& query, + const std::shared_ptr& full_text_search) const; + private: int64_t range_end_; std::wstring wfield_name_; From 1fed25142172f5e707a075faeb84985056b184cd Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 5 Feb 2026 10:46:55 +0800 Subject: [PATCH 12/12] fix noexcept --- .../lucene/lucene_global_index_reader.cpp | 72 +++++++++---------- .../lucene/lucene_global_index_reader.h | 14 ++-- 2 files changed, 39 insertions(+), 47 deletions(-) diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index fa89fa19f..6c39ca686 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -118,7 +118,7 @@ std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::stri } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery( - const std::shared_ptr& full_text_search) const { + const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL || full_text_search->search_type == FullTextSearch::SearchType::MATCH_ANY); Lucene::BooleanClause::Occur occur = @@ -141,7 +141,7 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery( } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery( - const std::shared_ptr& full_text_search) const { + const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::PHRASE); std::vector query_terms = TokenizeQuery(full_text_search->query); auto typed_query = Lucene::newLucene(); @@ -152,63 +152,55 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery( } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery( - const std::shared_ptr& full_text_search) const { + const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX); return Lucene::newLucene(Lucene::newLucene( wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); } Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery( - const std::shared_ptr& full_text_search) const { + const std::shared_ptr& full_text_search) const noexcept(false) { assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD); return Lucene::newLucene(Lucene::newLucene( wfield_name_, LuceneUtils::StringToWstring(full_text_search->query))); } Result> LuceneGlobalIndexReader::SearchWithLimit( - const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const { + const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const + noexcept(false) { assert(full_text_search->limit); - try { - Lucene::FilterPtr filter = - full_text_search->pre_filter - ? Lucene::newLucene(&(full_text_search->pre_filter.value())) - : Lucene::FilterPtr(); + Lucene::FilterPtr filter = + full_text_search->pre_filter + ? Lucene::newLucene(&(full_text_search->pre_filter.value())) + : Lucene::FilterPtr(); - Lucene::TopDocsPtr results = - searcher_->search(query, filter, full_text_search->limit.value()); + Lucene::TopDocsPtr results = searcher_->search(query, filter, full_text_search->limit.value()); - // prepare BitmapVectorSearchGlobalIndexResult - std::map id_to_score; - for (auto score_doc : results->scoreDocs) { - Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); - std::string row_id_str = - LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); - std::optional row_id = StringUtils::StringToValue(row_id_str); - if (!row_id) { - return Status::Invalid( - fmt::format("parse row id str {} to int failed", row_id_str)); - } - id_to_score[static_cast(row_id.value())] = - static_cast(score_doc->score); - } - RoaringBitmap64 bitmap; - std::vector scores; - scores.reserve(id_to_score.size()); - for (const auto& [id, score] : id_to_score) { - bitmap.Add(id); - scores.push_back(score); + // prepare BitmapVectorSearchGlobalIndexResult + std::map id_to_score; + for (auto score_doc : results->scoreDocs) { + Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); + std::string row_id_str = LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); + std::optional row_id = StringUtils::StringToValue(row_id_str); + if (!row_id) { + return Status::Invalid(fmt::format("parse row id str {} to int failed", row_id_str)); } - return std::make_shared(std::move(bitmap), - std::move(scores)); - } catch (const std::exception& e) { - return Status::Invalid(fmt::format("visit term query failed, with {} error.", e.what())); - } catch (...) { - return Status::UnknownError("visit term query failed, with unknown error."); + id_to_score[static_cast(row_id.value())] = static_cast(score_doc->score); + } + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(id_to_score.size()); + for (const auto& [id, score] : id_to_score) { + bitmap.Add(id); + scores.push_back(score); } + return std::make_shared(std::move(bitmap), + std::move(scores)); } -Result> LuceneGlobalIndexReader::SearchWithNoLimit( - const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const { +std::shared_ptr LuceneGlobalIndexReader::SearchWithNoLimit( + const Lucene::QueryPtr& query, const std::shared_ptr& full_text_search) const + noexcept(false) { assert(!full_text_search->limit); Lucene::FilterPtr filter = full_text_search->pre_filter diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index de5898420..66418dca2 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -132,24 +132,24 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { } Lucene::QueryPtr ConstructMatchQuery( - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); Lucene::QueryPtr ConstructPhraseQuery( - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); Lucene::QueryPtr ConstructPrefixQuery( - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); Lucene::QueryPtr ConstructWildCardQuery( - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); Result> SearchWithLimit( const Lucene::QueryPtr& query, - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); - Result> SearchWithNoLimit( + std::shared_ptr SearchWithNoLimit( const Lucene::QueryPtr& query, - const std::shared_ptr& full_text_search) const; + const std::shared_ptr& full_text_search) const noexcept(false); private: int64_t range_end_;