diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_and_test.yaml similarity index 54% rename from .github/workflows/build_release.yaml rename to .github/workflows/build_and_test.yaml index 152048cc9..3094cf72c 100644 --- a/.github/workflows/build_release.yaml +++ b/.github/workflows/build_and_test.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Build Release +name: Build and Test on: push: @@ -30,55 +30,56 @@ permissions: contents: read jobs: - clang-release: + build-and-test: + name: ${{ matrix.name }} runs-on: ubuntu-24.04 timeout-minutes: 120 strategy: fail-fast: false + matrix: + include: + - name: gcc-release + cc: gcc-14 + cxx: g++-14 + build_args: --build_type Release + - name: clang-release + build_args: --build_type Release + - name: gcc-debug + cc: gcc-14 + cxx: g++-14 + - name: clang-debug + fetch_depth: '0' # fetch the PR target branch history for clang-tidy + build_args: >- + --check_clang_tidy + --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}" + - name: asan-ubsan + build_args: --enable_asan --enable_ubsan + - name: tsan + skip_rust: true + build_args: --enable_tsan steps: - name: Checkout paimon-cpp uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: lfs: true + fetch-depth: ${{ matrix.fetch_depth || '1' }} - name: Setup ccache uses: ./.github/actions/setup-ccache with: - cache-key-prefix: ccache-clang-release + cache-key-prefix: ccache-${{ matrix.name }} - name: Install Rust toolchain (tantivy-fts) + if: ${{ !matrix.skip_rust }} shell: bash run: ci/scripts/setup_rust.sh - name: Build Paimon shell: bash env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false false Release - - name: Show ccache statistics - if: always() - run: ccache -s - gcc-release: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - lfs: true - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-gcc-release - - name: Install Rust toolchain (tantivy-fts) - shell: bash - run: ci/scripts/setup_rust.sh - - name: Build Paimon - shell: bash - env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) false false Release + CC: ${{ matrix.cc || 'clang' }} + CXX: ${{ matrix.cxx || 'clang++' }} + run: >- + ci/scripts/build_paimon.sh + --source_dir "$(pwd)" + ${{ matrix.build_args }} - name: Show ccache statistics if: always() run: ccache -s diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml deleted file mode 100644 index 824a5d45d..000000000 --- a/.github/workflows/clang_test.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2025-present Alibaba Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Clang Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - clang-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - lfs: true - fetch-depth: 0 # fetch all history for git diff in clang-tidy - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-clang-test - - name: Install Rust toolchain (tantivy-fts) - shell: bash - run: ci/scripts/setup_rust.sh - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index e88411e82..c4316b032 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -68,7 +68,7 @@ jobs: env: CC: gcc-8 CXX: g++-8 - run: ci/scripts/build_paimon.sh $(pwd) + run: ci/scripts/build_paimon.sh --source_dir "$(pwd)" - name: Show ccache statistics if: always() run: ccache -s diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml deleted file mode 100644 index af6e0ddbd..000000000 --- a/.github/workflows/gcc_test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2024-present Alibaba Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Gcc Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - gcc-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - lfs: true - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-gcc-test - - name: Install Rust toolchain (tantivy-fts) - shell: bash - run: ci/scripts/setup_rust.sh - - name: Build Paimon - shell: bash - env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml deleted file mode 100644 index a083a773b..000000000 --- a/.github/workflows/test_with_sanitizer.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2025-present Alibaba Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Test with sanitizer - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - test-with-sanitizer: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - lfs: true - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-sanitizer - - name: Install Rust toolchain (tantivy-fts) - shell: bash - run: ci/scripts/setup_rust.sh - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index 13a83393a..4c7aef1bf 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -14,3 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library. +race:liblance_lib_rc.so +thread:liblance_lib_rc.so +race:liblumina.so +race:libjindosdk_c.so.6 diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 5584d84f5..ba022423c 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -16,10 +16,87 @@ set -eux -source_dir=${1} -enable_sanitizer=${2:-false} -check_clang_tidy=${3:-false} -build_type=${4:-Debug} +usage() { + echo "Usage: $0 --source_dir [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type ] [--lint_git_target_commit ]" +} + +source_dir="" +enable_asan="false" +enable_ubsan="false" +enable_tsan="false" +check_clang_tidy="false" +build_type="Debug" +lint_git_target_commit="origin/main" + +while [[ $# -gt 0 ]]; do + case "$1" in + --source_dir) + if [[ $# -lt 2 ]]; then + echo "Missing value for --source_dir" >&2 + usage >&2 + exit 1 + fi + source_dir=$2 + shift 2 + ;; + --enable_asan) + enable_asan="true" + shift + ;; + --enable_ubsan) + enable_ubsan="true" + shift + ;; + --enable_tsan) + enable_tsan="true" + shift + ;; + --check_clang_tidy) + check_clang_tidy="true" + shift + ;; + --build_type) + if [[ $# -lt 2 ]]; then + echo "Missing value for --build_type" >&2 + usage >&2 + exit 1 + fi + build_type=$2 + shift 2 + ;; + --lint_git_target_commit) + if [[ $# -lt 2 ]]; then + echo "Missing value for --lint_git_target_commit" >&2 + usage >&2 + exit 1 + fi + lint_git_target_commit=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${source_dir}" ]]; then + echo "--source_dir is required" >&2 + usage >&2 + exit 1 +fi + +if [[ "${enable_asan}" == "true" && "${enable_tsan}" == "true" ]]; then + echo "ASAN and TSAN cannot be enabled together" >&2 + usage >&2 + exit 1 +fi + build_dir="${source_dir}/build" # Display ccache status if available @@ -46,6 +123,9 @@ if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then # Consider supporting Lance from source compilation in the future ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image. fi +if [[ "${enable_tsan}" == "true" ]]; then + ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented. +fi CMAKE_ARGS=( "-G Ninja" @@ -56,13 +136,17 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) -if [[ "${enable_sanitizer}" == "true" ]]; then - CMAKE_ARGS+=( - "-DPAIMON_USE_ASAN=ON" - "-DPAIMON_USE_UBSAN=ON" - ) +if [[ "${enable_asan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_ASAN=ON") +fi +if [[ "${enable_ubsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_UBSAN=ON") +fi +if [[ "${enable_tsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_TSAN=ON") fi cmake "${CMAKE_ARGS[@]}" "${source_dir}" diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index cc7dd1b82..7b70ed7f9 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -183,11 +183,13 @@ function(add_paimon_lib LIB_NAME) if(NOT APPLE) set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic -Wl,--gc-sections) - # -z defs (--no-undefined) rejects the __asan_*/__ubsan_* symbols that + # -z defs (--no-undefined) rejects the __asan_*/__tsan_*/__ubsan_* symbols that # sanitizer-instrumented shared libraries legitimately leave undefined # (they are resolved at load time from the executable's sanitizer # runtime). Only enforce it for non-sanitizer builds. - if(NOT PAIMON_USE_ASAN AND NOT PAIMON_USE_UBSAN) + if(NOT PAIMON_USE_ASAN + AND NOT PAIMON_USE_TSAN + AND NOT PAIMON_USE_UBSAN) list(APPEND SHARED_LINK_OPTIONS -Wl,-z,defs) endif() target_link_options(${LIB_NAME}_shared PRIVATE ${SHARED_LINK_OPTIONS}) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 9d5bf29fc..54dc3b3c0 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -23,6 +23,8 @@ set(THIRDPARTY_LOG_OPTIONS LOG_INSTALL 1 LOG_DOWNLOAD + 1 + LOG_OUTPUT_ON_FAILURE 1) set(THIRDPARTY_CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}") if(CMAKE_GENERATOR_TOOLSET) @@ -356,7 +358,7 @@ if(NOT MSVC_TOOLCHAIN) # Set -fPIC on all external projects string(APPEND EP_CXX_FLAGS " -fPIC -Wno-error -Wno-sign-compare -Wno-ignored-attributes") - string(APPEND EP_C_FLAGS " -fPIC") + string(APPEND EP_C_FLAGS " -fPIC -Wno-error") endif() if(PAIMON_USE_CXX11_ABI) @@ -724,19 +726,12 @@ macro(build_lucene) set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install") - set(LUCENE_CMAKE_CXX_FLAGS "-pthread") - if(PAIMON_USE_CXX11_ABI) - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") - else() - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") - endif() - set(LUCENE_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DLUCENE_BUILD_SHARED=OFF" "-DENABLE_TEST=OFF" - "-DCMAKE_C_FLAGS=-pthread" - "-DCMAKE_CXX_FLAGS=${LUCENE_CMAKE_CXX_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS} -pthread" + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -pthread" "-DCMAKE_EXE_LINKER_FLAGS=-pthread" "-DBoost_NO_BOOST_CMAKE=ON" "-DBoost_NO_SYSTEM_PATHS=ON" @@ -804,6 +799,8 @@ macro(build_jieba) externalproject_add(limonp_ep URL ${LIMONP_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "") message(STATUS "Building jieba from source") @@ -880,17 +877,9 @@ macro(build_fmt) "${FMT_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${FMT_STATIC_LIB_NAME}${FMT_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(FMT_LIBRARIES ${FMT_STATIC_LIB}) - set(FMT_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(FMT_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" FMT_CMAKE_CXX_FLAGS ${FMT_CMAKE_CXX_FLAGS}) - set(FMT_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} - "-DCMAKE_CXX_FLAGS=${FMT_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${FMT_CMAKE_C_FLAGS}" - -DFMT_TEST=OFF - -DFMT_DOC=OFF) + set(FMT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} + -DFMT_TEST=OFF -DFMT_DOC=OFF) set(FMT_CONFIGURE CMAKE_ARGS ${FMT_CMAKE_ARGS}) externalproject_add(fmt_ep URL ${FMT_SOURCE_URL} @@ -926,12 +915,16 @@ macro(build_boost) ${BOOST_LIBRARY_DIR}/libboost_chrono.a ${BOOST_LIBRARY_DIR}/libboost_iostreams.a) - set(BOOST_CXX_FLAGS "-fPIC") - if(PAIMON_USE_CXX11_ABI) - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(BOOST_TOOLSET clang) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(BOOST_TOOLSET gcc) else() - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") + message(FATAL_ERROR "Unsupported compiler for Boost: ${CMAKE_CXX_COMPILER_ID}") endif() + set(BOOST_USER_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/boost-user-config.jam") + file(WRITE ${BOOST_USER_CONFIG} + "using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n") externalproject_add(boost_ep URL "${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}" @@ -945,14 +938,17 @@ macro(build_boost) -sZLIB_INCLUDE=${ZLIB_INCLUDE_DIR} -sZLIB_LIBRARY_PATH=${ZLIB_PREFIX}/lib runtime-link=shared threading=multi variant=release - cxxflags=${BOOST_CXX_FLAGS} install + --user-config=${BOOST_USER_CONFIG} + toolset=${BOOST_TOOLSET} cxxflags=${EP_CXX_FLAGS} + linkflags=${EP_CXX_FLAGS} install INSTALL_COMMAND bash -c "mkdir -p ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/include/boost/* ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/*/include/boost/* ${BOOST_INSTALL}/include/boost" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} LOG_DOWNLOAD ON LOG_CONFIGURE ON - LOG_BUILD ON) + LOG_BUILD ON + LOG_OUTPUT_ON_FAILURE ON) include_directories(SYSTEM ${BOOST_INCLUDE_DIR}) @@ -1096,17 +1092,9 @@ macro(build_zstd) "${ZSTD_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(ZSTD_LIBRARIES ${ZSTD_STATIC_LIB}) - set(ZSTD_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ZSTD_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ZSTD_CMAKE_CXX_FLAGS ${ZSTD_CMAKE_CXX_FLAGS}) - set(ZSTD_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} - "-DCMAKE_CXX_FLAGS=${ZSTD_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ZSTD_CMAKE_C_FLAGS}" - -DZSTD_BUILD_SHARED=OFF - -DZSTD_BUILD_PROGRAMS=OFF) + set(ZSTD_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} + -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_PROGRAMS=OFF) set(ZSTD_CONFIGURE SOURCE_SUBDIR "build/cmake" CMAKE_ARGS ${ZSTD_CMAKE_ARGS}) externalproject_add(zstd_ep @@ -1210,14 +1198,8 @@ macro(build_jindosdk_nextarch) get_target_property(JINDOSDK_C_LIBRARY_LOCATION jindosdk::c_sdk IMPORTED_LOCATION) get_filename_component(JINDOSDK_C_DIR_ROOT "${JINDOSDK_C_INCLUDE_DIR}" DIRECTORY) - # Compile flags for jindosdk-nextarch - set(JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - set(JINDOSDK_NEXTARCH_CMAKE_C_FLAGS "${EP_C_FLAGS}") set(JINDOSDK_NEXTARCH_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" - "-DCMAKE_CXX_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_C_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" -DJINDOSDK_ROOT=${JINDOSDK_C_DIR_ROOT} -DJINDOSDK_LIBRARY_NAME=${JINDOSDK_C_DYNAMIC_LIB_NAME}) @@ -1348,14 +1330,9 @@ macro(build_avro) get_target_property(AVRO_FMT_INCLUDE_DIR fmt INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(AVRO_FMT_ROOT "${AVRO_FMT_INCLUDE_DIR}" DIRECTORY) - set(AVRO_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(AVRO_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - set(AVRO_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${AVRO_PREFIX}" - "-DCMAKE_CXX_FLAGS=${AVRO_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${AVRO_CMAKE_C_FLAGS}" "-DAVRO_BUILD_TESTS=OFF" "-DAVRO_BUILD_EXECUTABLES=OFF" "-DZLIB_ROOT=${AVRO_ZLIB_ROOT}" @@ -1412,13 +1389,6 @@ macro(build_orc) "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}") endif() - string(REPLACE "-Werror" "" EP_CXX_FLAGS ${EP_CXX_FLAGS}) - - set(ORC_CMAKE_CXX_FLAGS - "${EP_CXX_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") - set(ORC_CMAKE_C_FLAGS - "${EP_C_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") - set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-prefix") set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") set(ORC_SOURCE_DIR "${ORC_PREFIX}/cpp") @@ -1426,16 +1396,9 @@ macro(build_orc) set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/liborc.a") - message("ORC_STATIC_LIB IS ${ORC_STATIC_LIB}") - message("ORC_CMAKE_CXX_FLAGS ${ORC_CMAKE_CXX_FLAGS}") - message("ORC_CMAKE_C_FLAGS ${ORC_CMAKE_C_FLAGS}") - set(ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}" - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" ${ORC_LINKER_FLAGS} "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" "-DLZ4_HOME=${ORC_LZ4_ROOT}" @@ -1501,9 +1464,7 @@ macro(build_arrow) get_target_property(ARROW_RE2_INCLUDE_DIR re2::re2 INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(ARROW_RE2_ROOT "${ARROW_RE2_INCLUDE_DIR}" DIRECTORY) - set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ARROW_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ARROW_CMAKE_CXX_FLAGS ${ARROW_CMAKE_CXX_FLAGS}) + set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") # Fix for thrift Mutex.h missing #include (GCC 15 strictness) # Use -include to force include cstdint for all C++ files string(APPEND ARROW_CMAKE_CXX_FLAGS " -include cstdint") @@ -1538,8 +1499,6 @@ macro(build_arrow) ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}" "-DCMAKE_CXX_FLAGS=${ARROW_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_BUILD_SHARED=OFF @@ -1660,9 +1619,6 @@ endmacro(build_arrow) macro(build_gtest) message(STATUS "Building gtest from source") - set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS}) - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-install") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -1683,10 +1639,7 @@ macro(build_gtest) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main.a") endif() set(GTEST_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" - "-DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GTEST_CMAKE_CXX_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" "-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${_GTEST_RUNTIME_DIR}" "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}=${_GTEST_RUNTIME_DIR}") @@ -1725,9 +1678,8 @@ endmacro() macro(build_tbb) message(STATUS "Building Tbb from source") - set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) + set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") + set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS}") string(REPLACE "-Wdocumentation" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_C_FLAGS ${TBB_CMAKE_C_FLAGS}) @@ -1750,7 +1702,6 @@ macro(build_tbb) "-DCMAKE_INSTALL_PREFIX=${TBB_PREFIX}" "-DCMAKE_CXX_FLAGS=${TBB_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${TBB_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${TBB_CMAKE_CXX_FLAGS}" -DTBB_TEST=OFF) externalproject_add(tbb_ep @@ -1822,20 +1773,9 @@ macro(build_glog) set(GLOG_LIB_SUFFIX "") endif() set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog${GLOG_LIB_SUFFIX}.a") - set(GLOG_CMAKE_CXX_FLAGS " -Wno-error ${EP_CXX_FLAGS}") - set(GLOG_CMAKE_C_FLAGS " -Wno-error ${EP_C_FLAGS}") - if(CMAKE_THREAD_LIBS_INIT) - string(APPEND GLOG_CMAKE_CXX_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - string(APPEND GLOG_CMAKE_C_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - endif() - set(GLOG_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} - -DWITH_GFLAGS=OFF - -DWITH_GTEST=OFF - -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS=${GLOG_CMAKE_C_FLAGS}) + set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} + -DWITH_GFLAGS=OFF -DWITH_GTEST=OFF) if(NOT LIBUNWIND_LIBRARY) list(APPEND GLOG_CMAKE_ARGS -DWITH_UNWIND=none) endif() diff --git a/cmake_modules/san-config.cmake b/cmake_modules/san-config.cmake index 9d67f0fb0..9e51d505c 100644 --- a/cmake_modules/san-config.cmake +++ b/cmake_modules/san-config.cmake @@ -12,6 +12,11 @@ add_library(paimon_sanitizer_flags INTERFACE) +if(PAIMON_USE_ASAN AND PAIMON_USE_TSAN) + message(FATAL_ERROR "Address Sanitizer and Thread Sanitizer cannot be enabled together" + ) +endif() + if(PAIMON_USE_ASAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=address @@ -23,6 +28,19 @@ if(PAIMON_USE_ASAN) endif() endif() +if(PAIMON_USE_TSAN) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # Bundled dependencies are linked statically into Paimon. Instrument them too so TSAN can + # observe their synchronization primitives and does not report false races at the boundary. + string(APPEND CMAKE_C_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + string(APPEND CMAKE_CXX_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread) + message(STATUS "Thread Sanitizer enabled") + else() + message(WARNING "Thread Sanitizer is only supported for GCC and Clang compilers") + endif() +endif() + if(PAIMON_USE_UBSAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(paimon_sanitizer_flags diff --git a/src/paimon/core/operation/metrics/compaction_metrics.h b/src/paimon/core/operation/metrics/compaction_metrics.h index 868f7371e..08fd0cc80 100644 --- a/src/paimon/core/operation/metrics/compaction_metrics.h +++ b/src/paimon/core/operation/metrics/compaction_metrics.h @@ -50,16 +50,16 @@ class CompactionMetrics { : metrics_(metrics), partition_(partition), bucket_(bucket) {} void ReportLevel0FileCount(int64_t count) { - level0_file_count_ = count; + level0_file_count_.store(count, std::memory_order_relaxed); } void ReportCompactionInputSize(int64_t bytes) { - compaction_input_size_ = bytes; + compaction_input_size_.store(bytes, std::memory_order_relaxed); } void ReportCompactionOutputSize(int64_t bytes) { - compaction_output_size_ = bytes; + compaction_output_size_.store(bytes, std::memory_order_relaxed); } void ReportTotalFileSize(int64_t bytes) { - total_file_size_ = bytes; + total_file_size_.store(bytes, std::memory_order_relaxed); } void ReportCompactionTime(int64_t time) { metrics_->ReportCompactionTime(time); @@ -82,19 +82,19 @@ class CompactionMetrics { } int64_t Level0FileCount() const { - return level0_file_count_; + return level0_file_count_.load(std::memory_order_relaxed); } int64_t CompactionInputSize() const { - return compaction_input_size_; + return compaction_input_size_.load(std::memory_order_relaxed); } int64_t CompactionOutputSize() const { - return compaction_output_size_; + return compaction_output_size_.load(std::memory_order_relaxed); } int64_t TotalFileSize() const { - return total_file_size_; + return total_file_size_.load(std::memory_order_relaxed); } private: @@ -103,10 +103,10 @@ class CompactionMetrics { int32_t bucket_; // Data fields for metrics. - int64_t level0_file_count_ = 0; - int64_t compaction_input_size_ = 0; - int64_t compaction_output_size_ = 0; - int64_t total_file_size_ = 0; + std::atomic level0_file_count_ = {0}; + std::atomic compaction_input_size_ = {0}; + std::atomic compaction_output_size_ = {0}; + std::atomic total_file_size_ = {0}; }; std::shared_ptr CreateReporter(const BinaryRow& partition, int32_t bucket) { diff --git a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp index a2be4f01f..079920e98 100644 --- a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp +++ b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp @@ -17,6 +17,7 @@ #include "paimon/core/operation/metrics/compaction_metrics.h" #include +#include #include "gtest/gtest.h" #include "paimon/testing/utils/testharness.h" @@ -147,4 +148,38 @@ TEST(CompactionMetricsTest, TestCompactionTimeWindow) { EXPECT_DOUBLE_EQ(60.5, avg_time); } +TEST(CompactionMetricsTest, TestConcurrentReporterUpdateAndMetricsSnapshot) { + CompactionMetrics metrics; + std::shared_ptr reporter = + metrics.CreateReporter(BinaryRow::EmptyRow(), 0); + + constexpr int64_t kLastValue = 9999; + std::thread writer([&reporter]() { + for (int64_t value = 0; value <= kLastValue; ++value) { + reporter->ReportLevel0FileCount(value); + reporter->ReportCompactionInputSize(value); + reporter->ReportCompactionOutputSize(value); + reporter->ReportTotalFileSize(value); + } + }); + for (int64_t i = 0; i <= kLastValue; ++i) { + metrics.GetMetrics(); + } + writer.join(); + + std::shared_ptr snapshot = metrics.GetMetrics(); + ASSERT_OK_AND_ASSIGN(double level0_file_count, + snapshot->GetGauge(CompactionMetrics::MAX_LEVEL0_FILE_COUNT)); + ASSERT_OK_AND_ASSIGN(double compaction_input_size, + snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_INPUT_SIZE)); + ASSERT_OK_AND_ASSIGN(double compaction_output_size, + snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_OUTPUT_SIZE)); + ASSERT_OK_AND_ASSIGN(double total_file_size, + snapshot->GetGauge(CompactionMetrics::MAX_TOTAL_FILE_SIZE)); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), level0_file_count); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_input_size); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_output_size); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), total_file_size); +} + } // namespace paimon::test diff --git a/src/paimon/testing/utils/counting_cache_test_utils.h b/src/paimon/testing/utils/counting_cache_test_utils.h index ae683e732..ebb90912c 100644 --- a/src/paimon/testing/utils/counting_cache_test_utils.h +++ b/src/paimon/testing/utils/counting_cache_test_utils.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "paimon/cache/cache.h" @@ -45,16 +46,23 @@ class CountingRoutingCache : public Cache { const std::shared_ptr& key, std::function>(const std::shared_ptr&)> supplier) override { - ++get_count_; - last_kind_ = key->GetKind(); - ++get_count_by_kind_[key->GetKind()]; + CacheKind kind = key->GetKind(); + { + std::lock_guard lock(count_mutex_); + ++get_count_; + last_kind_ = kind; + ++get_count_by_kind_[kind]; + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); return cache->Get( key, [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) -> Result> { - ++supplier_call_count_; - ++supplier_call_count_by_kind_[supplier_key->GetKind()]; + { + std::lock_guard lock(count_mutex_); + ++supplier_call_count_; + ++supplier_call_count_by_kind_[supplier_key->GetKind()]; + } return supplier(supplier_key); }); } @@ -87,22 +95,27 @@ class CountingRoutingCache : public Cache { } int64_t GetCount() const { + std::lock_guard lock(count_mutex_); return get_count_; } int64_t GetCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); return GetCount(get_count_by_kind_, kind); } int64_t SupplierCallCount() const { + std::lock_guard lock(count_mutex_); return supplier_call_count_; } int64_t SupplierCallCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); return GetCount(supplier_call_count_by_kind_, kind); } CacheKind LastKind() const { + std::lock_guard lock(count_mutex_); return last_kind_; } @@ -129,6 +142,7 @@ class CountingRoutingCache : public Cache { int64_t get_count_ = 0; int64_t supplier_call_count_ = 0; CacheKind last_kind_ = CacheKind::DEFAULT; + mutable std::mutex count_mutex_; }; } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 5bbede302..c39edef80 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include #include @@ -3580,7 +3581,8 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { TEST_P(ReadInteTest, TestSpecificFs) { class CountableInputStream : public InputStream { public: - CountableInputStream(const std::shared_ptr& input, size_t* io_count) + CountableInputStream(const std::shared_ptr& input, + std::atomic* io_count) : input_(input), io_count_(io_count) {} ~CountableInputStream() override = default; @@ -3591,16 +3593,16 @@ TEST_P(ReadInteTest, TestSpecificFs) { return input_->GetPos(); } Result Read(char* buffer, int64_t size) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size); } Result Read(char* buffer, int64_t size, int64_t offset) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size, offset); } void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->ReadAsync(buffer, size, offset, std::move(callback)); } @@ -3615,12 +3617,12 @@ TEST_P(ReadInteTest, TestSpecificFs) { } std::shared_ptr input_; - size_t* io_count_; + std::atomic* io_count_; }; class CountableFileSystem : public FileSystem { public: - CountableFileSystem(const std::shared_ptr& fs, size_t* io_count) + CountableFileSystem(const std::shared_ptr& fs, std::atomic* io_count) : fs_(fs), io_count_(io_count) {} ~CountableFileSystem() override = default; @@ -3658,10 +3660,10 @@ TEST_P(ReadInteTest, TestSpecificFs) { } std::shared_ptr fs_; - size_t* io_count_; + std::atomic* io_count_; }; - size_t io_count = 0; + std::atomic io_count = {0}; auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; @@ -3714,7 +3716,7 @@ TEST_P(ReadInteTest, TestSpecificFs) { &expected_array); ASSERT_TRUE(array_status.ok()); ASSERT_TRUE(result_array->Equals(expected_array)); - ASSERT_GT(io_count, 0); + ASSERT_GT(io_count.load(std::memory_order_relaxed), 0); } } // namespace paimon::test