Merge branch 'main' into codex/build-lumina-from-release-package

Author: zjw1111

.github/workflows/build_release.yaml .github/workflows/build_and_test.yaml.github/workflows/build_release.yaml renamed to .github/workflows/build_and_test.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Build Release
+name: Build and Test
 
 on:
   push:
@@ -30,55 +30,56 @@ permissions:
   contents: read
 
 jobs:
-  clang-release:
+  build-and-test:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-24.04
     timeout-minutes: 120
     strategy:
       fail-fast: false
+      matrix:
+        include:
+          - name: gcc-release
+            cc: gcc-14
+            cxx: g++-14
+            build_args: --build_type Release
+          - name: clang-release
+            build_args: --build_type Release
+          - name: gcc-debug
+            cc: gcc-14
+            cxx: g++-14
+          - name: clang-debug
+            fetch_depth: '0' # fetch the PR target branch history for clang-tidy
+            build_args: >-
+              --check_clang_tidy
+              --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
+          - name: asan-ubsan
+            build_args: --enable_asan --enable_ubsan
+          - name: tsan
+            skip_rust: true
+            build_args: --enable_tsan
     steps:
       - name: Checkout paimon-cpp
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           lfs: true
+          fetch-depth: ${{ matrix.fetch_depth || '1' }}
       - name: Setup ccache
         uses: ./.github/actions/setup-ccache
         with:
-          cache-key-prefix: ccache-clang-release
+          cache-key-prefix: ccache-${{ matrix.name }}
       - name: Install Rust toolchain (tantivy-fts)
+        if: ${{ !matrix.skip_rust }}
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Build Paimon
         shell: bash
         env:
-          CC: clang
-          CXX: clang++
-        run: ci/scripts/build_paimon.sh $(pwd) false false Release
-      - name: Show ccache statistics
-        if: always()
-        run: ccache -s
-  gcc-release:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 120
-    strategy:
-      fail-fast: false
-    steps:
-      - name: Checkout paimon-cpp
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-        with:
-          lfs: true
-      - name: Setup ccache
-        uses: ./.github/actions/setup-ccache
-        with:
-          cache-key-prefix: ccache-gcc-release
-      - name: Install Rust toolchain (tantivy-fts)
-        shell: bash
-        run: ci/scripts/setup_rust.sh
-      - name: Build Paimon
-        shell: bash
-        env:
-          CC: gcc-14
-          CXX: g++-14
-        run: ci/scripts/build_paimon.sh $(pwd) false false Release
+          CC: ${{ matrix.cc || 'clang' }}
+          CXX: ${{ matrix.cxx || 'clang++' }}
+        run: >-
+          ci/scripts/build_paimon.sh
+          --source_dir "$(pwd)"
+          ${{ matrix.build_args }}
       - name: Show ccache statistics
         if: always()
         run: ccache -s

.github/workflows/clang_test.yaml

@@ -68,7 +68,7 @@ jobs:
         env:
           CC: gcc-8
           CXX: g++-8
-        run: ci/scripts/build_paimon.sh $(pwd)
+        run: ci/scripts/build_paimon.sh --source_dir "$(pwd)"
       - name: Show ccache statistics
         if: always()
         run: ccache -s

.github/workflows/gcc8_test.yaml

@@ -57,6 +57,8 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_NETWORK_TESTS
+       "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
 option(PAIMON_ENABLE_TANTIVY
@@ -70,6 +72,11 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_NETWORK_TESTS)
+    if(NOT PAIMON_BUILD_TESTS)
+        message(FATAL_ERROR "PAIMON_ENABLE_NETWORK_TESTS requires PAIMON_BUILD_TESTS=ON")
+    endif()
+endif()
 if(PAIMON_USE_CXX11_ABI)
     add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 else()

.github/workflows/gcc_test.yaml

@@ -14,3 +14,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+# Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library.
+race:liblance_lib_rc.so
+thread:liblance_lib_rc.so
+race:liblumina.so
+race:libjindosdk_c.so.6