Skip to content

Commit 3597a00

Browse files
author
Rafał Hibner
committed
Merge branch 'SerialSequencingBackpressure' into combined2
2 parents 8e9af06 + cd7d82f commit 3597a00

37 files changed

Lines changed: 649 additions & 65 deletions

‎.env‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ LLVM=18
6767
MAVEN=3.8.7
6868
NODE=20
6969
NUMBA=latest
70+
NUMBA_CUDA=latest
7071
NUMPY=latest
7172
PANDAS=latest
7273
PYTHON=3.9

‎.github/workflows/cpp_extra.yml‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ jobs:
8181
outputs:
8282
ci-extra: ${{ steps.check.outputs.ci-extra }}
8383
steps:
84+
- name: Checkout Arrow
85+
if: github.event_name == 'pull_request'
86+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
8487
- name: Check
8588
id: check
8689
env:
@@ -99,7 +102,15 @@ jobs:
99102
if [ "${n_ci_extra_labels}" -eq 1 ]; then
100103
ci_extra=true
101104
else
102-
ci_extra=false
105+
git fetch origin ${GITHUB_BASE_REF}
106+
if git diff --stat origin/${GITHUB_BASE_REF}.. | \
107+
grep \
108+
--fixed-strings ".github/workflows/cpp_extra.yml" \
109+
--quiet; then
110+
ci_extra=true
111+
else
112+
ci_extra=false
113+
fi
103114
fi
104115
;;
105116
esac
@@ -126,6 +137,8 @@ jobs:
126137
title: AMD64 Ubuntu Meson
127138
# TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved
128139
- continue-on-error: true
140+
envs:
141+
- DEBIAN=13
129142
image: debian-cpp
130143
run-options: >-
131144
-e CMAKE_CXX_STANDARD=23
@@ -158,10 +171,16 @@ jobs:
158171
env:
159172
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
160173
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
174+
ENVS: ${{ toJSON(matrix.envs) }}
161175
run: |
162176
# GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
163177
sudo sysctl -w vm.mmap_rnd_bits=28
164178
source ci/scripts/util_enable_core_dumps.sh
179+
if [ "${ENVS}" != "null" ]; then
180+
echo "${ENVS}" | jq -r '.[]' | while read env; do
181+
echo "${env}" >> .env
182+
done
183+
fi
165184
archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }}
166185
- name: Docker Push
167186
if: >-
@@ -174,3 +193,88 @@ jobs:
174193
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
175194
continue-on-error: true
176195
run: archery docker push ${{ matrix.image }}
196+
197+
jni-macos:
198+
needs: check-labels
199+
name: JNI macOS
200+
runs-on: macos-14
201+
if: needs.check-labels.outputs.ci-extra == 'true'
202+
timeout-minutes: 45
203+
env:
204+
MACOSX_DEPLOYMENT_TARGET: "14.0"
205+
steps:
206+
- name: Checkout Arrow
207+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
208+
with:
209+
fetch-depth: 0
210+
submodules: recursive
211+
- name: Install dependencies
212+
run: |
213+
brew bundle --file=cpp/Brewfile
214+
# We want to link aws-sdk-cpp statically but Homebrew's
215+
# aws-sdk-cpp provides only shared library. If we have
216+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
217+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
218+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
219+
brew uninstall aws-sdk-cpp
220+
# We want to use bundled RE2 for static linking. If
221+
# Homebrew's RE2 is installed, its header file may be used.
222+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
223+
brew uninstall grpc || : # gRPC depends on RE2
224+
brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
225+
brew uninstall re2
226+
# We want to use bundled Protobuf for static linking. If
227+
# Homebrew's Protobuf is installed, its library file may be
228+
# used on test We uninstall Homebrew's Protobuf to ensure using
229+
# bundled Protobuf.
230+
brew uninstall protobuf
231+
- name: Prepare ccache
232+
run: |
233+
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
234+
- name: Cache ccache
235+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
236+
with:
237+
path: ccache
238+
key: jni-macos-${{ hashFiles('cpp/**') }}
239+
restore-keys: jni-macos-
240+
- name: CMake
241+
run: |
242+
cmake \
243+
-S cpp \
244+
-B cpp.build \
245+
--preset=ninja-release-jni-macos \
246+
-DARROW_BUILD_TESTS=ON \
247+
-DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
248+
- name: Build
249+
run: |
250+
cmake --build cpp.build
251+
- name: Install
252+
run: |
253+
cmake --install cpp.build
254+
- name: Test
255+
env:
256+
ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
257+
PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
258+
run: |
259+
# MinIO is required
260+
exclude_tests="arrow-s3fs-test"
261+
# unstable
262+
exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
263+
exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
264+
ctest \
265+
--exclude-regex "${exclude_tests}" \
266+
--label-regex unittest \
267+
--output-on-failure \
268+
--parallel "$(sysctl -n hw.ncpu)" \
269+
--test-dir "cpp.build" \
270+
--timeout 300
271+
- name: Build example
272+
run: |
273+
cmake \
274+
-S cpp/examples/minimal_build/ \
275+
-B cpp/examples/minimal_build.build \
276+
-GNinja \
277+
-DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
278+
cmake --build cpp/examples/minimal_build.build
279+
cd cpp/examples/minimal_build
280+
../minimal_build.build/arrow-example

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616
# under the License.
1717

1818
apache-rat-*.jar
19-
arrow-src.tar
20-
arrow-src.tar.gz
19+
apache-arrow.tar.gz
2120

2221
# Compiled source
2322
*.a

‎ci/conda_env_sphinx.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,6 @@ sphinx-copybutton
3333
sphinx-lint
3434
sphinxcontrib-jquery
3535
sphinxcontrib-mermaid
36-
sphinx==6.2
36+
sphinx
3737
pytest-cython
3838
pandas

‎ci/docker/debian-13-cpp.dockerfile‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG arch=amd64
19+
FROM ${arch}/debian:13
20+
ARG arch
21+
22+
ENV DEBIAN_FRONTEND noninteractive
23+
24+
ARG llvm
25+
RUN apt-get update -y -q && \
26+
apt-get install -y -q --no-install-recommends \
27+
apt-transport-https \
28+
ca-certificates \
29+
gnupg \
30+
lsb-release \
31+
wget && \
32+
if [ ${llvm} -ge 20 ]; then \
33+
wget -O /usr/share/keyrings/llvm-snapshot.asc \
34+
https://apt.llvm.org/llvm-snapshot.gpg.key && \
35+
(echo "Types: deb"; \
36+
echo "URIs: https://apt.llvm.org/$(lsb_release --codename --short)/"; \
37+
echo "Suites: llvm-toolchain-$(lsb_release --codename --short)-${llvm}"; \
38+
echo "Components: main"; \
39+
echo "Signed-By: /usr/share/keyrings/llvm-snapshot.asc") | \
40+
tee /etc/apt/sources.list.d/llvm.sources; \
41+
fi && \
42+
apt-get update -y -q && \
43+
apt-get install -y -q --no-install-recommends \
44+
autoconf \
45+
ccache \
46+
clang-${llvm} \
47+
cmake \
48+
curl \
49+
g++ \
50+
gcc \
51+
gdb \
52+
git \
53+
libbenchmark-dev \
54+
libboost-filesystem-dev \
55+
libboost-system-dev \
56+
libbrotli-dev \
57+
libbz2-dev \
58+
libc-ares-dev \
59+
libcurl4-openssl-dev \
60+
libgflags-dev \
61+
libgmock-dev \
62+
libgoogle-glog-dev \
63+
libgrpc++-dev \
64+
libidn2-dev \
65+
libkrb5-dev \
66+
libldap-dev \
67+
liblz4-dev \
68+
libnghttp2-dev \
69+
libprotobuf-dev \
70+
libprotoc-dev \
71+
libpsl-dev \
72+
libre2-dev \
73+
librtmp-dev \
74+
libsnappy-dev \
75+
libsqlite3-dev \
76+
libssh-dev \
77+
libssh2-1-dev \
78+
libssl-dev \
79+
libthrift-dev \
80+
libutf8proc-dev \
81+
libxml2-dev \
82+
libxsimd-dev \
83+
libzstd-dev \
84+
llvm-${llvm}-dev \
85+
make \
86+
ninja-build \
87+
nlohmann-json3-dev \
88+
npm \
89+
opentelemetry-cpp-dev \
90+
pkg-config \
91+
protobuf-compiler-grpc \
92+
python3-dev \
93+
python3-pip \
94+
python3-venv \
95+
rapidjson-dev \
96+
rsync \
97+
tzdata \
98+
zlib1g-dev && \
99+
apt-get clean && \
100+
rm -rf /var/lib/apt/lists/*
101+
102+
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
103+
RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
104+
105+
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
106+
RUN /arrow/ci/scripts/install_gcs_testbench.sh default
107+
108+
COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
109+
RUN /arrow/ci/scripts/install_azurite.sh
110+
111+
COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
112+
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
113+
114+
# Prioritize system packages and local installation.
115+
ENV ARROW_ACERO=ON \
116+
ARROW_AZURE=ON \
117+
ARROW_BUILD_TESTS=ON \
118+
ARROW_DATASET=ON \
119+
ARROW_DEPENDENCY_SOURCE=SYSTEM \
120+
ARROW_DATASET=ON \
121+
ARROW_FLIGHT=ON \
122+
ARROW_FLIGHT_SQL=ON \
123+
ARROW_GANDIVA=ON \
124+
ARROW_GCS=ON \
125+
ARROW_HOME=/usr/local \
126+
ARROW_JEMALLOC=ON \
127+
ARROW_ORC=ON \
128+
ARROW_PARQUET=ON \
129+
ARROW_S3=ON \
130+
ARROW_SUBSTRAIT=ON \
131+
ARROW_USE_CCACHE=ON \
132+
ARROW_WITH_BROTLI=ON \
133+
ARROW_WITH_BZ2=ON \
134+
ARROW_WITH_LZ4=ON \
135+
ARROW_WITH_OPENTELEMETRY=ON \
136+
ARROW_WITH_SNAPPY=ON \
137+
ARROW_WITH_ZLIB=ON \
138+
ARROW_WITH_ZSTD=ON \
139+
AWSSDK_SOURCE=BUNDLED \
140+
Azure_SOURCE=BUNDLED \
141+
google_cloud_cpp_storage_SOURCE=BUNDLED \
142+
ORC_SOURCE=BUNDLED \
143+
PATH=/usr/lib/ccache/:$PATH \
144+
PYTHON=python3

‎ci/docker/linux-apt-python-3.dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,10 @@ RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
3232
-r arrow/python/requirements-test.txt
3333

3434
ARG numba
35+
ARG numba_cuda
3536
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/
3637
RUN if [ "${numba}" != "" ]; then \
37-
/arrow/ci/scripts/install_numba.sh ${numba} \
38+
/arrow/ci/scripts/install_numba.sh ${numba} ${numba_cuda} \
3839
; fi
3940

4041
ENV ARROW_ACERO=ON \

‎ci/scripts/install_numba.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919

2020
set -e
2121

22-
if [ "$#" -ne 1 ]; then
23-
echo "Usage: $0 <numba version>"
22+
if [ "$#" -ne 1 ] && [ "$#" -ne 2 ]; then
23+
echo "Usage: $0 <numba version> [numba-cuda version]"
2424
exit 1
2525
fi
2626

@@ -41,3 +41,17 @@ elif [ "${numba}" = "latest" ]; then
4141
else
4242
pip install "numba==${numba}"
4343
fi
44+
45+
if [ "$#" -eq 1 ]; then
46+
exit 0
47+
fi
48+
49+
numba_cuda=$2
50+
51+
if [ "${numba_cuda}" = "master" ]; then
52+
pip install https://github.com/NVIDIA/numba-cuda/archive/main.tar.gz#egg=numba-cuda
53+
elif [ "${numba_cuda}" = "latest" ]; then
54+
pip install numba-cuda
55+
else
56+
pip install "numba-cuda==${numba_cuda}"
57+
fi

‎cpp/CMakePresets.json‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,34 @@
582582
],
583583
"displayName": "Benchmarking build with everything enabled",
584584
"cacheVariables": {}
585+
},
586+
{
587+
"name": "ninja-release-jni-macos",
588+
"inherits": [
589+
"base-release"
590+
],
591+
"displayName": "Build for JNI on macOS",
592+
"cacheVariables": {
593+
"ARROW_ACERO": "ON",
594+
"ARROW_BUILD_SHARED": "OFF",
595+
"ARROW_BUILD_STATIC": "ON",
596+
"ARROW_CSV": "ON",
597+
"ARROW_DATASET": "ON",
598+
"ARROW_DEPENDENCY_USE_SHARED": "OFF",
599+
"ARROW_GANDIVA": "ON",
600+
"ARROW_GANDIVA_STATIC_LIBSTDCPP": "ON",
601+
"ARROW_JSON": "ON",
602+
"ARROW_ORC": "ON",
603+
"ARROW_PARQUET": "ON",
604+
"ARROW_S3": "ON",
605+
"ARROW_SUBSTRAIT": "ON",
606+
"AWSSDK_SOURCE": "BUNDLED",
607+
"GTest_SOURCE": "BUNDLED",
608+
"PARQUET_BUILD_EXAMPLES": "OFF",
609+
"PARQUET_BUILD_EXECUTABLES": "OFF",
610+
"PARQUET_REQUIRE_ENCRYPTION": "OFF",
611+
"re2_SOURCE": "BUNDLED"
612+
}
585613
}
586614
]
587615
}

0 commit comments

Comments
 (0)