Skip to content

Commit 0135204

Browse files
authored
fix: fix IsNullOrWhitespaceOnly utility in JiebaTokenizer (#140)
1 parent 251e9f5 commit 0135204

2 files changed

Lines changed: 2 additions & 8 deletions

File tree

src/paimon/global_index/lucene/jieba_analyzer.cpp

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
#include "paimon/global_index/lucene/jieba_analyzer.h"
1717

18+
#include "paimon/common/utils/string_utils.h"
1819
#include "paimon/global_index/lucene/lucene_utils.h"
1920

2021
namespace paimon::lucene {
@@ -82,11 +83,6 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb
8283
}
8384
}
8485

85-
bool JiebaTokenizer::IsWhitespaceOnly(const std::string& term) {
86-
return term.empty() ||
87-
std::all_of(term.begin(), term.end(), [](unsigned char c) { return std::isspace(c); });
88-
}
89-
9086
void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words,
9187
std::vector<std::string>* input_ptr,
9288
std::vector<std::string_view>* output_ptr) {
@@ -95,7 +91,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set<std::string>& stop_words
9591
output.clear();
9692
output.reserve(input.size());
9793
for (auto& term : input) {
98-
if (IsWhitespaceOnly(term)) {
94+
if (StringUtils::IsNullOrWhitespaceOnly(term)) {
9995
continue;
10096
}
10197
// remove stop words

src/paimon/global_index/lucene/jieba_analyzer.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,6 @@ class JiebaTokenizer : public Lucene::Tokenizer {
6060
private:
6161
void InnerReset();
6262

63-
static bool IsWhitespaceOnly(const std::string& term);
64-
6563
private:
6664
JiebaTokenizerContext context_;
6765
size_t term_index_ = 0;

0 commit comments

Comments
 (0)