Merge branch 'main' into clean_metric

Author: lucasfang

CMakeLists.txt

@@ -290,6 +290,22 @@ if(PAIMON_ENABLE_LUMINA)
             DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endif()
 
+if(PAIMON_ENABLE_LUCENE)
+    set(PAIMON_DICT_DEST "share/paimon/dict")
+
+    install(DIRECTORY ${JIEBA_DICT_DIR}/
+            DESTINATION ${PAIMON_DICT_DEST}
+            FILES_MATCHING
+            PATTERN "jieba.dict.utf8"
+            PATTERN "hmm_model.utf8"
+            PATTERN "idf.utf8"
+            PATTERN "stop_words.utf8"
+            PATTERN "user.dict.utf8"
+            PATTERN "pos_dict"
+            PATTERN ".git*" EXCLUDE
+            PATTERN "*.md" EXCLUDE)
+endif()
+
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/
         DESTINATION "include"
         FILES_MATCHING
@@ -389,7 +405,6 @@ if(PAIMON_BUILD_TESTS)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
     endif()
-
 endif()
 
 include(CMakePackageConfigHelpers)

LICENSE

@@ -266,6 +266,8 @@ This product includes code from Apache Iceberg C++.
 * Avro direct decoder/encoder:
   * src/paimon/format/avro/avro_direct_decoder.cpp
   * src/paimon/format/avro/avro_direct_decoder.h
+  * src/paimon/format/avro/avro_direct_encoder.cpp
+  * src/paimon/format/avro/avro_direct_encoder.h
 
 Copyright: 2024-2025 The Apache Software Foundation.
 Home page: https://iceberg.apache.org/

cmake_modules/DefineOptions.cmake

@@ -92,7 +92,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     define_option_string(PAIMON_CXXFLAGS "Compiler flags to append when compiling Paimon"
                          "")
 
-    define_option(PAIMON_BUILD_STATIC "Build static libraries" OFF)
+    define_option(PAIMON_BUILD_STATIC "Build static libraries" ON)
 
     define_option(PAIMON_BUILD_SHARED "Build shared libraries" ON)
     #----------------------------------------------------------------------

cmake_modules/ThirdpartyToolchain.cmake

@@ -322,8 +322,7 @@ macro(build_lucene)
     set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include")
     # The include directory must exist before it is referenced by a target.
     file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}")
-    include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR}
-                        ${BOOST_EXTRA_INCLUDE_DIR})
+    include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR})
     add_library(lucene INTERFACE IMPORTED)
     target_include_directories(lucene SYSTEM INTERFACE "${LUCENE_INCLUDE_DIR}")
     target_compile_options(lucene INTERFACE -pthread)
@@ -343,6 +342,43 @@ macro(build_lucene)
     add_dependencies(lucene lucene_ep)
 endmacro()
 
+macro(build_jieba)
+    message(STATUS "Building jieba from source")
+    set(JIEBA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-prefix")
+    set(JIEBA_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-install")
+    set(JIEBA_INCLUDE_DIR "${JIEBA_INSTALL}/include")
+    set(JIEBA_DICT_DIR "${JIEBA_INSTALL}/dict")
+    file(MAKE_DIRECTORY ${JIEBA_INCLUDE_DIR})
+    file(MAKE_DIRECTORY ${JIEBA_DICT_DIR})
+
+    set(JIEBA_CMAKE_ARGS
+        ${EP_COMMON_CMAKE_ARGS} "-DENABLE_TEST=OFF" "-DCPPJIEBA_TOP_LEVEL_PROJECT=OFF"
+        "-DCMAKE_INSTALL_PREFIX=${JIEBA_INSTALL}")
+
+    set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff")
+    externalproject_add(jieba_ep
+                        ${EP_COMMON_OPTIONS}
+                        GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git
+                        GIT_TAG ${PAIMON_JIEBA_BUILD_VERSION}
+                        GIT_SHALLOW FALSE
+                        GIT_PROGRESS TRUE
+                        GIT_SUBMODULES_RECURSE TRUE
+                        CMAKE_ARGS ${JIEBA_CMAKE_ARGS}
+                        LOG_PATCH ON
+                        PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> bash -c
+                                      "[ -f .patched ] && echo '<SOURCE_DIR> patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched"
+                        INSTALL_COMMAND bash -c
+                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/"
+    )
+
+    # The include directory must exist before it is referenced by a target.
+    include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR})
+    add_library(jieba INTERFACE IMPORTED)
+    target_include_directories(jieba SYSTEM
+                               INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}")
+    add_dependencies(jieba jieba_ep)
+endmacro()
+
 macro(build_rapidjson)
     message(STATUS "Building RapidJSON from source")
     set(RAPIDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rapidjson_ep-install")
@@ -1272,4 +1308,5 @@ endif()
 if(PAIMON_ENABLE_LUCENE)
     build_boost()
     build_lucene()
+    build_jieba()
 endif()

cmake_modules/jieba.diff

@@ -0,0 +1,16 @@
+diff --git a/include/cppjieba/KeywordExtractor.hpp b/include/cppjieba/KeywordExtractor.hpp
+index 24b2c40..c7c6a94 100644
+--- a/include/cppjieba/KeywordExtractor.hpp
++++ b/include/cppjieba/KeywordExtractor.hpp
+@@ -89,6 +89,11 @@ class KeywordExtractor {
+     std::partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
+     keywords.resize(topN);
+   }
++
++  const std::unordered_set<std::string>& GetStopWords() const {
++    return stopWords_;
++  }
++
+  private:
+   void LoadIdfDict(const std::string& idfPath) {
+     std::ifstream ifs(idfPath.c_str());

include/paimon/defs.h

@@ -130,7 +130,7 @@ struct PAIMON_EXPORT Options {
     static const char MANIFEST_TARGET_FILE_SIZE[];
 
     /// "manifest.format" - Specify the message format of manifest files.
-    /// Default value is orc.
+    /// Default value is avro.
     static const char MANIFEST_FORMAT[];
 
     /// "manifest.compression" - File compression for manifest, default value is zstd.
@@ -287,6 +287,8 @@ struct PAIMON_EXPORT Options {
     /// "global-index.external-path" - Global index root directory, if not set, the global index
     /// files will be stored under the index directory.
     static const char GLOBAL_INDEX_EXTERNAL_PATH[];
+    /// "scan.tag-name" - Optional tag name used in case of "from-snapshot" scan mode.
+    static const char SCAN_TAG_NAME[];
 };
 
 static constexpr int64_t BATCH_WRITE_COMMIT_IDENTIFIER = std::numeric_limits<int64_t>::max();

include/paimon/file_index/file_index_reader.h

@@ -60,6 +60,8 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<Fil
     Result<std::shared_ptr<FileIndexResult>> VisitEndsWith(const Literal& suffix) override;
 
     Result<std::shared_ptr<FileIndexResult>> VisitContains(const Literal& literal) override;
+
+    Result<std::shared_ptr<FileIndexResult>> VisitLike(const Literal& literal) override;
 };
 
 }  // namespace paimon

include/paimon/predicate/function_visitor.h

@@ -71,5 +71,8 @@ class PAIMON_EXPORT FunctionVisitor {
 
     /// Evaluates whether string values contain the given substring.
     virtual Result<T> VisitContains(const Literal& literal) = 0;
+
+    /// Evaluates whether string values like the given string.
+    virtual Result<T> VisitLike(const Literal& literal) = 0;
 };
 }  // namespace paimon

include/paimon/read_context.h

@@ -27,7 +27,6 @@
 #include "paimon/result.h"
 #include "paimon/type_fwd.h"
 #include "paimon/utils/read_ahead_cache.h"
-#include "paimon/utils/special_field_ids.h"
 #include "paimon/visibility.h"
 
 namespace paimon {
@@ -54,8 +53,8 @@ class PAIMON_EXPORT ReadContext {
                 const std::shared_ptr<Executor>& executor,
                 const std::shared_ptr<FileSystem>& specific_file_system,
                 const std::map<std::string, std::string>& fs_scheme_to_identifier_map,
-                const std::map<std::string, std::string>& options, bool enable_prefetch_cache,
-                const CacheConfig& cache_config);
+                const std::map<std::string, std::string>& options,
+                PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config);
     ~ReadContext();
 
     const std::string& GetPath() const {
@@ -117,8 +116,8 @@ class PAIMON_EXPORT ReadContext {
         return specific_file_system_;
     }
 
-    bool EnablePrefetchCache() const {
-        return enable_prefetch_cache_;
+    PrefetchCacheMode GetPrefetchCacheMode() const {
+        return prefetch_cache_mode_;
     }
 
     const CacheConfig& GetCacheConfig() const {
@@ -143,7 +142,7 @@ class PAIMON_EXPORT ReadContext {
     std::shared_ptr<FileSystem> specific_file_system_;
     std::map<std::string, std::string> fs_scheme_to_identifier_map_;
     std::map<std::string, std::string> options_;
-    bool enable_prefetch_cache_;
+    PrefetchCacheMode prefetch_cache_mode_;
     CacheConfig cache_config_;
 };
 
@@ -179,9 +178,9 @@ class PAIMON_EXPORT ReadContextBuilder {
     /// @param read_field_ids Vector of field ids to read from the table.
     /// @return Reference to this builder for method chaining.
     /// @note Currently supports top-level field selection. Future versions may support
-    ///       nested field selection using ArrowSchema for more granular projection,
-    ///       If SetReadFieldIds() call and SetReadSchema() are natually are mutually
-    ///       exclusive. Calling both will ignore the read schema set by SetReadSchema().
+    ///       nested field selection using ArrowSchema for more granular projection.
+    /// @note SetReadFieldIds() and SetReadSchema() are mutually exclusive.
+    ///       Calling both will ignore the read schema set by SetReadSchema().
     ReadContextBuilder& SetReadFieldIds(const std::vector<int32_t>& read_field_ids);
 
     /// Set a configuration options map to set some option entries which are not defined in the
@@ -227,13 +226,13 @@ class PAIMON_EXPORT ReadContextBuilder {
     /// @return Reference to this builder for method chaining.
     ReadContextBuilder& EnablePrefetch(bool enabled);
 
-    /// Enable or disable prefetch cache for read operations.
+    /// Set prefetch cache mode for read operations.
     ///
-    /// When enabled, a prefetch cache is used to prebuffer data ranges before they are needed,
+    /// A prefetch cache is used to prebuffer data ranges before they are needed,
     /// which can improve read performance by reducing redundant I/O operations.
-    /// @param enabled Whether to enable prefetch cache (default: true)
+    /// @param mode (default: PrefetchCacheMode::ALWAYS)
     /// @return Reference to this builder for method chaining.
-    ReadContextBuilder& EnablePrefetchCache(bool enabled);
+    ReadContextBuilder& SetPrefetchCacheMode(PrefetchCacheMode mode);
 
     /// Set the cache configuration for prefetch read operations.
     ///

include/paimon/reader/file_batch_reader.h

@@ -50,7 +50,7 @@ class PAIMON_EXPORT FileBatchReader : public BatchReader {
     virtual uint64_t GetPreviousBatchFirstRowNumber() const = 0;
 
     /// Get the number of rows in the file.
-    virtual uint64_t GetNumberOfRows() const = 0;
+    virtual Result<uint64_t> GetNumberOfRows() const = 0;
 
     /// Get whether or not support read precisely while bitmap pushed down.
     virtual bool SupportPreciseBitmapSelection() const = 0;