ci: split S3/MinIO into a Linux and macOS-only workflow

Closes apache#640

Author: zhjwpku

.github/workflows/s3_test.yml

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# S3-backed tests against MinIO (Linux and macOS only).
+name: S3 Tests
+
+on:
+  push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  ICEBERG_HOME: /tmp/iceberg
+
+jobs:
+  s3-minio:
+    name: S3 (${{ matrix.title }})
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 35
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - title: AMD64 Ubuntu 24.04
+            runs-on: ubuntu-24.04
+            CC: gcc-14
+            CXX: g++-14
+          - title: AArch64 macOS 26
+            runs-on: macos-26
+    env:
+      ICEBERG_TEST_S3_URI: s3://iceberg-test
+      AWS_ACCESS_KEY_ID: minio
+      AWS_SECRET_ACCESS_KEY: minio123
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ENDPOINT_URL: http://127.0.0.1:9000
+      AWS_EC2_METADATA_DISABLED: "TRUE"
+    steps:
+      - name: Checkout iceberg-cpp
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+      - name: Install dependencies on Ubuntu
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set Ubuntu Compilers
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
+        run: |
+          echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
+          echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
+      - name: Start MinIO
+        shell: bash
+        run: bash ci/scripts/start_minio.sh
+      - name: Build and test Iceberg with S3
+        shell: bash
+        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ON

.github/workflows/test.yml

@@ -43,13 +43,6 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
-    env:
-      ICEBERG_TEST_S3_URI: s3://iceberg-test
-      AWS_ACCESS_KEY_ID: minio
-      AWS_SECRET_ACCESS_KEY: minio123
-      AWS_DEFAULT_REGION: us-east-1
-      AWS_ENDPOINT_URL: http://127.0.0.1:9000
-      AWS_EC2_METADATA_DISABLED: "TRUE"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -58,9 +51,6 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
-      - name: Start MinIO
-        shell: bash
-        run: bash ci/scripts/start_minio.sh
       - name: Build Iceberg
         shell: bash
         env:
@@ -79,21 +69,11 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
-    env:
-      ICEBERG_TEST_S3_URI: s3://iceberg-test
-      AWS_ACCESS_KEY_ID: minio
-      AWS_SECRET_ACCESS_KEY: minio123
-      AWS_DEFAULT_REGION: us-east-1
-      AWS_ENDPOINT_URL: http://127.0.0.1:9000
-      AWS_EC2_METADATA_DISABLED: "TRUE"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
           persist-credentials: false
-      - name: Start MinIO
-        shell: bash
-        run: bash ci/scripts/start_minio.sh
       - name: Build Iceberg
         shell: bash
         run: ci/scripts/build_iceberg.sh $(pwd)
@@ -106,13 +86,6 @@ jobs:
     timeout-minutes: 60
     strategy:
       fail-fast: false
-    env:
-      ICEBERG_TEST_S3_URI: s3://iceberg-test
-      AWS_ACCESS_KEY_ID: minio
-      AWS_SECRET_ACCESS_KEY: minio123
-      AWS_DEFAULT_REGION: us-east-1
-      AWS_ENDPOINT_URL: http://127.0.0.1:9000
-      AWS_EC2_METADATA_DISABLED: "TRUE"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -124,9 +97,6 @@ jobs:
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
-      - name: Start MinIO
-        shell: bash
-        run: bash ci/scripts/start_minio.sh
       - name: Build Iceberg
         shell: pwsh
         env:

ci/scripts/build_iceberg.sh

@@ -16,13 +16,16 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+#
+# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF]
 
 set -eux
 
 source_dir=${1}
 build_dir=${1}/build
 build_rest_integration_test=${2:-OFF}
 build_enable_sccache=${3:-OFF}
+build_enable_s3=${4:-OFF}
 
 mkdir ${build_dir}
 pushd ${build_dir}
@@ -36,10 +39,15 @@ CMAKE_ARGS=(
     "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
     "-DICEBERG_BUILD_STATIC=ON"
     "-DICEBERG_BUILD_SHARED=ON"
-    "-DICEBERG_S3=ON"
     "-DICEBERG_BUILD_REST_INTEGRATION_TESTS=${build_rest_integration_test}"
 )
 
+if [[ "${build_enable_s3}" == "ON" ]]; then
+    CMAKE_ARGS+=("-DICEBERG_S3=ON")
+else
+    CMAKE_ARGS+=("-DICEBERG_S3=OFF")
+fi
+
 if is_windows; then
     CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
     CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")