Merge branch 'main' into feat/zstd-level-config

Author: dalingmeng

LICENSE

@@ -373,3 +373,78 @@ Home page: https://roaringbitmap.org/
 License: https://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
+
+This product includes code from boost.
+
+* Boost source code in third_party/boost/ directory
+
+Copyright: 2003-2023 The Boost Authors
+Home page: https://www.boost.org/
+License: https://www.boost.org/LICENSE_1_0.txt
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product includes code from LucenePlusPlus.
+
+* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory
+
+Copyright: 2009-2014 Alan Wright.
+Home page: https://github.com/luceneplusplus/LucenePlusPlus
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from cppjieba.
+
+* cppjieba utility in src/paimon/global_index/lucene/ directory
+
+Copyright: 2013
+Home page: https://github.com/yanyiwu/cppjieba
+License: https://opensource.org/licenses/MIT
+
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------

NOTICE

@@ -28,3 +28,12 @@ Copyright (C) 2012-2023 Yann Collet
 
 This product includes software from CRoaring project (Apache 2.0)
 Copyright 2016-2022 The CRoaring authors
+
+This product includes software from boost project (BSL 1.0)
+Copyright 2003-2023 The Boost authors
+
+This product includes software from LucenePlusPlus project (Apache 2.0)
+Copyright 2009-2014 Alan Wright.
+
+This product includes software from cppjieba project (MIT)
+Copyright 2013

cmake_modules/ThirdpartyToolchain.cmake

@@ -124,6 +124,30 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_LIMONP_URL})
+    set(LIMONP_SOURCE_URL "$ENV{PAIMON_LIMONP_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}")
+        set_urls(LIMONP_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}")
+    else()
+        set_urls(LIMONP_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}"
+        )
+    endif()
+endif()
+
+if(DEFINED ENV{PAIMON_JIEBA_URL})
+    set(JIEBA_SOURCE_URL "$ENV{PAIMON_JIEBA_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}")
+        set_urls(JIEBA_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}")
+    else()
+        set_urls(JIEBA_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_GLOG_URL})
     set(GLOG_SOURCE_URL "$ENV{PAIMON_GLOG_URL}")
 else()
@@ -289,6 +313,10 @@ set(EP_COMMON_CMAKE_ARGS
 
 macro(build_lucene)
     message(STATUS "Building lucene from source")
+
+    get_target_property(LUCENE_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(LUCENE_ZLIB_ROOT "${LUCENE_ZLIB_INCLUDE_DIR}" DIRECTORY)
+
     set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install")
     set(LUCENE_CMAKE_ARGS
         ${EP_COMMON_CMAKE_ARGS}
@@ -302,6 +330,9 @@ macro(build_lucene)
         "-DBOOST_ROOT=${BOOST_INSTALL}"
         "-DBoost_CHRONO_FOUND=TRUE"
         "-DBoost_THREAD_FOUND=TRUE"
+        "-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIR}"
+        "-DZLIB_LIBRARY_RELEASE=${ZLIB_LIBRARIES}"
+        "-DZLIB_ROOT=${LUCENE_ZLIB_ROOT}"
         "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")
 
     set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a")
@@ -311,7 +342,8 @@ macro(build_lucene)
                         URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}"
                         CMAKE_ARGS ${LUCENE_CMAKE_ARGS}
                         BUILD_BYPRODUCTS ${LUCENE_LIB}
-                        DEPENDS boost_date_time
+                        DEPENDS zlib
+                                boost_date_time
                                 boost_filesystem
                                 boost_regex
                                 boost_thread
@@ -331,7 +363,8 @@ macro(build_lucene)
                                      "${LUCENE_INCLUDE_DIR}")
 
     target_link_libraries(lucene
-                          INTERFACE boost_date_time
+                          INTERFACE zlib
+                                    boost_date_time
                                     boost_filesystem
                                     boost_regex
                                     boost_thread
@@ -345,6 +378,13 @@ macro(build_lucene)
 endmacro()
 
 macro(build_jieba)
+    message(STATUS "Building limonp from source")
+    set(LIMONP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/limonp_ep-prefix")
+    externalproject_add(limonp_ep
+                        URL ${LIMONP_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}"
+                        INSTALL_COMMAND "")
+
     message(STATUS "Building jieba from source")
     set(JIEBA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-prefix")
     set(JIEBA_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-install")
@@ -360,24 +400,22 @@ macro(build_jieba)
     set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff")
     externalproject_add(jieba_ep
                         ${EP_COMMON_OPTIONS}
-                        GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git
-                        GIT_TAG ${PAIMON_JIEBA_BUILD_VERSION}
-                        GIT_SHALLOW FALSE
-                        GIT_PROGRESS TRUE
-                        GIT_SUBMODULES_RECURSE TRUE
+                        URL ${JIEBA_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_JIEBA_BUILD_SHA256_CHECKSUM}"
                         CMAKE_ARGS ${JIEBA_CMAKE_ARGS}
                         LOG_PATCH ON
                         PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> bash -c
                                       "[ -f .patched ] && echo '<SOURCE_DIR> patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched"
                         INSTALL_COMMAND bash -c
-                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/"
+                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${LIMONP_PREFIX}/src/limonp_ep/include/* ${JIEBA_INSTALL}/include/"
     )
 
     # The include directory must exist before it is referenced by a target.
     include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR})
     add_library(jieba INTERFACE IMPORTED)
     target_include_directories(jieba SYSTEM
                                INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}")
+    add_dependencies(jieba_ep limonp_ep)
     add_dependencies(jieba jieba_ep)
 endmacro()
 
@@ -468,11 +506,8 @@ macro(build_boost)
         ${BOOST_LIBRARY_DIR}/libboost_iostreams.a)
 
     externalproject_add(boost_ep
-                        GIT_REPOSITORY https://github.com/boostorg/boost.git
-                        GIT_TAG boost-${PAIMON_BOOST_BUILD_VERSION}
-                        GIT_SHALLOW FALSE
-                        GIT_PROGRESS TRUE
-                        GIT_SUBMODULES_RECURSE TRUE
+                        URL "${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}"
+                        URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}"
                         CONFIGURE_COMMAND ${BOOST_PREFIX}/src/boost_ep/bootstrap.sh
                                           --with-libraries=date_time,filesystem,iostreams,regex,system,thread,chrono,atomic
                         BUILD_IN_SOURCE TRUE

docs/source/user_guide/data_types.rst

@@ -63,7 +63,7 @@ and `Arrow DataTypes <https://arrow.apache.org/docs/format/Columnar.html#data-ty
    * - ``BINARY``
 
        ``BINARY(n)``
-     - Binary
+     - Not Supported
      - Data type of a fixed-length binary string (=a sequence of bytes).
 
        The type can be declared using ``BINARY(n)`` where n is the number of

src/paimon/core/core_options.cpp

@@ -150,7 +150,7 @@ class ConfigParser {
     Status ParseSortOrder(SortOrder* sort_order) const {
         auto iter = config_map_.find(Options::SEQUENCE_FIELD_SORT_ORDER);
         if (iter != config_map_.end()) {
-            const auto& str = iter->second;
+            std::string str = StringUtils::ToLowerCase(iter->second);
             if (str == "ascending") {
                 *sort_order = SortOrder::ASCENDING;
             } else if (str == "descending") {
@@ -166,7 +166,7 @@ class ConfigParser {
     Status ParseSortEngine(SortEngine* sort_engine) const {
         auto iter = config_map_.find(Options::SORT_ENGINE);
         if (iter != config_map_.end()) {
-            const auto& str = iter->second;
+            std::string str = StringUtils::ToLowerCase(iter->second);
             if (str == "min-heap") {
                 *sort_engine = SortEngine::MIN_HEAP;
             } else if (str == "loser-tree") {
@@ -182,7 +182,7 @@ class ConfigParser {
     Status ParseMergeEngine(MergeEngine* merge_engine) const {
         auto iter = config_map_.find(Options::MERGE_ENGINE);
         if (iter != config_map_.end()) {
-            const auto& str = iter->second;
+            std::string str = StringUtils::ToLowerCase(iter->second);
             if (str == "deduplicate") {
                 *merge_engine = MergeEngine::DEDUPLICATE;
             } else if (str == "partial-update") {
@@ -202,7 +202,7 @@ class ConfigParser {
     Status ParseChangelogProducer(ChangelogProducer* changelog_producer) const {
         auto iter = config_map_.find(Options::CHANGELOG_PRODUCER);
         if (iter != config_map_.end()) {
-            const auto& str = iter->second;
+            std::string str = StringUtils::ToLowerCase(iter->second);
             if (str == "none") {
                 *changelog_producer = ChangelogProducer::NONE;
             } else if (str == "input") {
@@ -222,7 +222,7 @@ class ConfigParser {
     Status ParseExternalPathStrategy(ExternalPathStrategy* external_path_strategy) const {
         auto iter = config_map_.find(Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY);
         if (iter != config_map_.end()) {
-            const auto& str = iter->second;
+            std::string str = StringUtils::ToLowerCase(iter->second);
             if (str == "none") {
                 *external_path_strategy = ExternalPathStrategy::NONE;
             } else if (str == "specific-fs") {
@@ -236,6 +236,16 @@ class ConfigParser {
         return Status::OK();
     }
 
+    // Parse StartupMode
+    Status ParseStartupMode(StartupMode* startup_mode) const {
+        auto iter = config_map_.find(Options::SCAN_MODE);
+        if (iter != config_map_.end()) {
+            std::string str = StringUtils::ToLowerCase(iter->second);
+            PAIMON_ASSIGN_OR_RAISE(*startup_mode, StartupMode::FromString(str));
+        }
+        return Status::OK();
+    }
+
  private:
     const std::map<std::string, std::string> config_map_;
 };
@@ -361,12 +371,7 @@ Result<CoreOptions> CoreOptions::FromMap(
                                                 &impl->file_system));
 
     // Parse startup mode
-    std::string startup_mode_str;
-    if (options_map.find(Options::SCAN_MODE) != options_map.end()) {
-        PAIMON_RETURN_NOT_OK(parser.ParseString(Options::SCAN_MODE, &startup_mode_str));
-        PAIMON_ASSIGN_OR_RAISE(auto startup_mode, StartupMode::FromString(startup_mode_str));
-        impl->startup_mode = startup_mode;
-    }
+    PAIMON_RETURN_NOT_OK(parser.ParseStartupMode(&impl->startup_mode));
 
     // Special handling for ExpireConfig
     int32_t snapshot_num_retain_min = 10;

src/paimon/core/core_options_test.cpp

@@ -336,4 +336,23 @@ TEST(CoreOptionsTest, TestFileSystem) {
             typed_fs->GetRealFileSystem("oss:///tmp/").value_or(nullptr)));
     }
 }
+
+TEST(CoreOptionsTest, TestNormalizeValueInCoreOption) {
+    std::map<std::string, std::string> options = {
+        {Options::SEQUENCE_FIELD_SORT_ORDER, "ASCENDING"},
+        {Options::SORT_ENGINE, "MIN-heap"},
+        {Options::MERGE_ENGINE, "first-ROW"},
+        {Options::CHANGELOG_PRODUCER, "LOOKUP"},
+        {Options::DATA_FILE_EXTERNAL_PATHS_STRATEGY, "ROUND-ROBIN"},
+        {Options::SCAN_MODE, "DEFAULT"},
+    };
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options));
+
+    ASSERT_EQ(StartupMode::LatestFull(), core_options.GetStartupMode());
+    ASSERT_EQ(ExternalPathStrategy::ROUND_ROBIN, core_options.GetExternalPathStrategy());
+    ASSERT_EQ(ChangelogProducer::LOOKUP, core_options.GetChangelogProducer());
+    ASSERT_EQ(MergeEngine::FIRST_ROW, core_options.GetMergeEngine());
+    ASSERT_EQ(SortEngine::MIN_HEAP, core_options.GetSortEngine());
+    ASSERT_TRUE(core_options.SequenceFieldSortOrderIsAscending());
+}
 }  // namespace paimon::test

third_party/boost/boost_1_66_0.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e9200637800fbfd1292b2c6876189dba7e8e1c5282c71fac6515e96f7af2b0
+size 88165095

third_party/versions.txt

@@ -71,13 +71,21 @@ PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2
 PAIMON_JINDOSDK_C_BUILD_SHA256_CHECKSUM=23e61c9815fab1cd88c369445bdbe1eab02cc09bafed3bb5118ecaf5b2fbc518
 PAIMON_JINDOSDK_C_PKG_NAME=jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}.tar.gz
 
-PAIMON_BOOST_BUILD_VERSION=1.66.0
-
 PAIMON_LUCENE_BUILD_VERSION=3.0.9
 PAIMON_LUCENE_BUILD_SHA256_CHECKSUM=4e69e29d5d79a976498ef71eab70c9c88c7014708be4450a9fda7780fe93584e
 PAIMON_LUCENE_PKG_NAME=rel_${PAIMON_LUCENE_BUILD_VERSION}.tar.gz
 
+PAIMON_LIMONP_BUILD_VERSION=1.0.1
+PAIMON_LIMONP_BUILD_SHA256_CHECKSUM=c7b18794f020dbaa1006229b49a39217a463da0cb3586aee83eb7471f4ae71df
+PAIMON_LIMONP_PKG_NAME=v${PAIMON_LIMONP_BUILD_VERSION}.tar.gz
+
 PAIMON_JIEBA_BUILD_VERSION=v5.6.0
+PAIMON_JIEBA_BUILD_SHA256_CHECKSUM=e6e517b778e0f4a99cbed1ee3eaa041616b74bc685e03a6ca08887ad9cedfe49
+PAIMON_JIEBA_PKG_NAME=${PAIMON_JIEBA_BUILD_VERSION}.tar.gz
+
+PAIMON_BOOST_BUILD_VERSION=1_66_0
+PAIMON_BOOST_BUILD_SHA256_CHECKSUM=28e9200637800fbfd1292b2c6876189dba7e8e1c5282c71fac6515e96f7af2b0
+PAIMON_BOOST_PKG_NAME=boost_${PAIMON_BOOST_BUILD_VERSION}.tar.gz
 
 # The first field is the name of the environment variable expected by cmake.
 # This _must_ match what is defined. The second field is the name of the
@@ -99,4 +107,7 @@ DEPENDENCIES=(
   "PAIMON_RAPIDJSON_URL ${PAIMON_RAPIDJSON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/miloyip/rapidjson/archive/${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz"
   "PAIMON_JINDOSDK_C_URL ${PAIMON_JINDOSDK_C_PKG_NAME} https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux.tar.gz"
   "PAIMON_LUCENE_URL ${PAIMON_LUCENE_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/luceneplusplus/LucenePlusPlus/archive/refs/tags/${PAIMON_LUCENE_PKG_NAME}"
+  "PAIMON_LIMONP_URL ${PAIMON_LIMONP_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}"
+  "PAIMON_JIEBA_URL ${PAIMON_JIEBA_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}"
+  "PAIMON_BOOST_URL ${PAIMON_BOOST_PKG_NAME} ${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}"
 )