From 3775afd5ba5aef86f0aa926cd829bfed60e5246c Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 11 Feb 2026 16:21:13 +0800 Subject: [PATCH 1/3] feat(fts): strip whitespace in tokenization & reduce I/O by removing row_id --- .../global_index/lucene/jieba_analyzer.cpp | 9 +++++++- .../global_index/lucene/jieba_analyzer.h | 2 ++ src/paimon/global_index/lucene/lucene_defs.h | 1 - .../lucene/lucene_global_index_reader.cpp | 8 +------ .../lucene/lucene_global_index_writer.cpp | 23 ++++++++----------- .../lucene/lucene_global_index_writer.h | 4 +--- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index 3457f06ed..7ebe671ce 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -82,6 +82,11 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb } } +bool JiebaTokenizer::IsWhitespaceOnly(const std::string& term) { + return !term.empty() && + std::all_of(term.begin(), term.end(), [](unsigned char c) { return std::isspace(c); }); +} + void JiebaTokenizer::Normalize(const std::unordered_set& stop_words, std::vector* input_ptr, std::vector* output_ptr) { @@ -90,11 +95,13 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words output.clear(); output.reserve(input.size()); for (auto& term : input) { + if (IsWhitespaceOnly(term)) { + continue; + } // remove stop words if (stop_words.find(term) != stop_words.end()) { continue; } - // to lower case bool is_alphanumeric = true; for (const auto& c : term) { diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index 5f6da9a4e..31a174bba 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -60,6 +60,8 @@ class JiebaTokenizer : public Lucene::Tokenizer { private: void InnerReset(); + static bool IsWhitespaceOnly(const std::string& term); + private: JiebaTokenizerContext context_; size_t term_index_ = 0; diff --git a/src/paimon/global_index/lucene/lucene_defs.h b/src/paimon/global_index/lucene/lucene_defs.h index ed576a8de..23346fc28 100644 --- a/src/paimon/global_index/lucene/lucene_defs.h +++ b/src/paimon/global_index/lucene/lucene_defs.h @@ -23,7 +23,6 @@ namespace paimon::lucene { static inline const int32_t kVersion = 1; static inline const char kIdentifier[] = "lucene-fts"; static inline const wchar_t kEmptyWstring[] = L""; -static inline const wchar_t kRowIdFieldWstring[] = L"_ROW_ID"; static inline const char kOptionKeyPrefix[] = "lucene-fts."; diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index a3cb56125..3f30b89f7 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -179,13 +179,7 @@ Result> LuceneGlobalIndexReader::SearchWithLi // prepare BitmapScoredGlobalIndexResult std::map id_to_score; for (auto score_doc : results->scoreDocs) { - Lucene::DocumentPtr result_doc = searcher_->doc(score_doc->doc); - std::string row_id_str = LuceneUtils::WstringToString(result_doc->get(kRowIdFieldWstring)); - std::optional row_id = StringUtils::StringToValue(row_id_str); - if (!row_id) { - return Status::Invalid(fmt::format("parse row id str {} to int failed", row_id_str)); - } - id_to_score[static_cast(row_id.value())] = static_cast(score_doc->score); + id_to_score[static_cast(score_doc->doc)] = static_cast(score_doc->score); } RoaringBitmap64 bitmap; std::vector scores; diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 0752b784e..ec8d4a275 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -40,13 +40,12 @@ namespace paimon::lucene { LuceneGlobalIndexWriter::LuceneWriteContext::LuceneWriteContext( const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, const Lucene::IndexWriterPtr& _index_writer, const Lucene::DocumentPtr& _doc, - const Lucene::FieldPtr& _field, const Lucene::FieldPtr& _row_id_field) + const Lucene::FieldPtr& _field) : tmp_index_path(_tmp_index_path), lucene_dir(_lucene_dir), index_writer(_index_writer), doc(_doc), - field(_field), - row_id_field(_row_id_field) {} + field(_field) {} Result> LuceneGlobalIndexWriter::Create( const std::string& field_name, const std::shared_ptr& arrow_type, @@ -89,17 +88,11 @@ Result> LuceneGlobalIndexWriter::Create auto field = Lucene::newLucene(LuceneUtils::StringToWstring(field_name), kEmptyWstring, Lucene::Field::STORE_NO, Lucene::Field::INDEX_ANALYZED_NO_NORMS); - auto row_id_field = Lucene::newLucene( - kRowIdFieldWstring, kEmptyWstring, Lucene::Field::STORE_YES, - Lucene::Field::INDEX_NOT_ANALYZED_NO_NORMS); field->setOmitTermFreqAndPositions(omit_term_freq_and_positions); - row_id_field->setOmitTermFreqAndPositions(true); doc->add(field); - doc->add(row_id_field); return std::shared_ptr(new LuceneGlobalIndexWriter( - field_name, arrow_type, - LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field, row_id_field), file_writer, - options, pool)); + field_name, arrow_type, LuceneWriteContext(tmp_path, lucene_dir, writer, doc, field), + file_writer, options, pool)); } catch (const std::exception& e) { return Status::Invalid( fmt::format("create lucene global index writer failed, with {} error.", e.what())); @@ -153,8 +146,7 @@ Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array) { auto view = string_array->Value(i); write_context_.field->setValue(LuceneUtils::StringToWstring(view)); } - write_context_.row_id_field->setValue( - LuceneUtils::StringToWstring(std::to_string(row_id_++))); + row_id_++; write_context_.index_writer->addDocument(write_context_.doc); } } catch (const std::exception& e) { @@ -170,6 +162,11 @@ Status LuceneGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array) { Result LuceneGlobalIndexWriter::FlushIndexToFinal() { try { // flush index to tmp dir + if (write_context_.index_writer->numDocs() != row_id_) { + return Status::Invalid( + fmt::format("lucene writer row count {} mismatch paimon inner row count {}", + write_context_.index_writer->numDocs(), row_id_)); + } write_context_.index_writer->optimize(); write_context_.index_writer->close(); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index a929efeb7..21df08f19 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -29,8 +29,7 @@ class LuceneGlobalIndexWriter : public GlobalIndexWriter { LuceneWriteContext(const std::string& _tmp_index_path, const Lucene::FSDirectoryPtr& _lucene_dir, const Lucene::IndexWriterPtr& _index_writer, - const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field, - const Lucene::FieldPtr& _row_id_field); + const Lucene::DocumentPtr& _doc, const Lucene::FieldPtr& _field); LuceneWriteContext(LuceneWriteContext&&) = default; LuceneWriteContext& operator=(LuceneWriteContext&&) = default; @@ -40,7 +39,6 @@ class LuceneGlobalIndexWriter : public GlobalIndexWriter { Lucene::IndexWriterPtr index_writer; Lucene::DocumentPtr doc; Lucene::FieldPtr field; - Lucene::FieldPtr row_id_field; }; static Result> Create( From 95b620232728f29826c65155c5591923510783eb Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 11 Feb 2026 16:27:41 +0800 Subject: [PATCH 2/3] fix --- src/paimon/global_index/lucene/jieba_analyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index 7ebe671ce..3c6485f84 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -83,7 +83,7 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb } bool JiebaTokenizer::IsWhitespaceOnly(const std::string& term) { - return !term.empty() && + return term.empty() || std::all_of(term.begin(), term.end(), [](unsigned char c) { return std::isspace(c); }); } From b60eed16a477b91a2f74a8429d745a4321dd7e93 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 11 Feb 2026 16:36:19 +0800 Subject: [PATCH 3/3] add test --- src/paimon/global_index/lucene/jieba_analyzer_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp index 8c3f74626..e26b72943 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp @@ -90,7 +90,7 @@ TEST_P(JiebaAnalyzerTest, TestWithPosition) { TEST_P(JiebaAnalyzerTest, TestNormalize) { auto tokenizer = CreateJiebaTokenizer( /*with_position=*/false, - L"由于购买了Iphone14，我越来越热爱网上学习了！Happy work, happy day!"); + L"由于购买了Iphone14，我越来越热爱网上学习了！Happy work, happy day! \n\t"); auto term_att = tokenizer->addAttribute();