Skip to content

Commit db7bb30

Browse files
zhjwpkuzhjwpku
authored
ci: split S3/MinIO into a Linux and macOS-only workflow (#642)
1 parent ff42229 commit db7bb30

3 files changed

Lines changed: 90 additions & 31 deletions

File tree

‎.github/workflows/s3_test.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# S3-backed tests against MinIO (Linux and macOS only).
19+
name: S3 Tests
20+
21+
on:
22+
push:
23+
branches:
24+
- '**'
25+
- '!dependabot/**'
26+
tags:
27+
- '**'
28+
pull_request:
29+
30+
concurrency:
31+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
32+
cancel-in-progress: true
33+
34+
permissions:
35+
contents: read
36+
37+
env:
38+
ICEBERG_HOME: /tmp/iceberg
39+
40+
jobs:
41+
s3-minio:
42+
name: S3 (${{ matrix.title }})
43+
runs-on: ${{ matrix.runs-on }}
44+
timeout-minutes: 35
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
include:
49+
- title: AMD64 Ubuntu 24.04
50+
runs-on: ubuntu-24.04
51+
CC: gcc-14
52+
CXX: g++-14
53+
- title: AArch64 macOS 26
54+
runs-on: macos-26
55+
env:
56+
ICEBERG_TEST_S3_URI: s3://iceberg-test
57+
AWS_ACCESS_KEY_ID: minio
58+
AWS_SECRET_ACCESS_KEY: minio123
59+
AWS_DEFAULT_REGION: us-east-1
60+
AWS_ENDPOINT_URL: http://127.0.0.1:9000
61+
AWS_EC2_METADATA_DISABLED: "TRUE"
62+
steps:
63+
- name: Checkout iceberg-cpp
64+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
65+
with:
66+
persist-credentials: false
67+
- name: Install dependencies on Ubuntu
68+
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
69+
shell: bash
70+
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
71+
- name: Set Ubuntu Compilers
72+
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
73+
run: |
74+
echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
75+
echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
76+
- name: Start MinIO
77+
shell: bash
78+
run: bash ci/scripts/start_minio.sh
79+
- name: Build and test Iceberg with S3
80+
shell: bash
81+
run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ON

‎.github/workflows/test.yml‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,6 @@ jobs:
4343
timeout-minutes: 30
4444
strategy:
4545
fail-fast: false
46-
env:
47-
ICEBERG_TEST_S3_URI: s3://iceberg-test
48-
AWS_ACCESS_KEY_ID: minio
49-
AWS_SECRET_ACCESS_KEY: minio123
50-
AWS_DEFAULT_REGION: us-east-1
51-
AWS_ENDPOINT_URL: http://127.0.0.1:9000
52-
AWS_EC2_METADATA_DISABLED: "TRUE"
5346
steps:
5447
- name: Checkout iceberg-cpp
5548
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -58,9 +51,6 @@ jobs:
5851
- name: Install dependencies
5952
shell: bash
6053
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
61-
- name: Start MinIO
62-
shell: bash
63-
run: bash ci/scripts/start_minio.sh
6454
- name: Build Iceberg
6555
shell: bash
6656
env:
@@ -79,21 +69,11 @@ jobs:
7969
timeout-minutes: 30
8070
strategy:
8171
fail-fast: false
82-
env:
83-
ICEBERG_TEST_S3_URI: s3://iceberg-test
84-
AWS_ACCESS_KEY_ID: minio
85-
AWS_SECRET_ACCESS_KEY: minio123
86-
AWS_DEFAULT_REGION: us-east-1
87-
AWS_ENDPOINT_URL: http://127.0.0.1:9000
88-
AWS_EC2_METADATA_DISABLED: "TRUE"
8972
steps:
9073
- name: Checkout iceberg-cpp
9174
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
9275
with:
9376
persist-credentials: false
94-
- name: Start MinIO
95-
shell: bash
96-
run: bash ci/scripts/start_minio.sh
9777
- name: Build Iceberg
9878
shell: bash
9979
run: ci/scripts/build_iceberg.sh $(pwd)
@@ -106,13 +86,6 @@ jobs:
10686
timeout-minutes: 60
10787
strategy:
10888
fail-fast: false
109-
env:
110-
ICEBERG_TEST_S3_URI: s3://iceberg-test
111-
AWS_ACCESS_KEY_ID: minio
112-
AWS_SECRET_ACCESS_KEY: minio123
113-
AWS_DEFAULT_REGION: us-east-1
114-
AWS_ENDPOINT_URL: http://127.0.0.1:9000
115-
AWS_EC2_METADATA_DISABLED: "TRUE"
11689
steps:
11790
- name: Checkout iceberg-cpp
11891
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -128,9 +101,6 @@ jobs:
128101
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows
129102
- name: Setup sccache
130103
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
131-
- name: Start MinIO
132-
shell: bash
133-
run: bash ci/scripts/start_minio.sh
134104
- name: Build Iceberg
135105
shell: pwsh
136106
env:

‎ci/scripts/build_iceberg.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
# KIND, either express or implied. See the License for the
1717
# specific language governing permissions and limitations
1818
# under the License.
19+
#
20+
# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF]
1921

2022
set -eux
2123

2224
source_dir=${1}
2325
build_dir=${1}/build
2426
build_rest_integration_test=${2:-OFF}
2527
build_enable_sccache=${3:-OFF}
28+
build_enable_s3=${4:-OFF}
2629

2730
mkdir ${build_dir}
2831
pushd ${build_dir}
@@ -36,10 +39,15 @@ CMAKE_ARGS=(
3639
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
3740
"-DICEBERG_BUILD_STATIC=ON"
3841
"-DICEBERG_BUILD_SHARED=ON"
39-
"-DICEBERG_S3=ON"
4042
"-DICEBERG_BUILD_REST_INTEGRATION_TESTS=${build_rest_integration_test}"
4143
)
4244

45+
if [[ "${build_enable_s3}" == "ON" ]]; then
46+
CMAKE_ARGS+=("-DICEBERG_S3=ON")
47+
else
48+
CMAKE_ARGS+=("-DICEBERG_S3=OFF")
49+
fi
50+
4351
if is_windows; then
4452
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
4553
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")

0 commit comments

Comments
 (0)