Skip to content

Commit 9c6f764

Browse files
2sin182sin18
authored
[CI] Build packages on platform manylinux_2_24.
1 parent 9c58cbc commit 9c6f764

16 files changed

Lines changed: 118 additions & 315 deletions

‎.github/workflows/tensorflow1.14-py3.6-cibuild.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Tests with TensorFlow 1.14 on CPU
33
on: workflow_dispatch
44

55
env:
6-
IMAGE: registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.14-py3.6
6+
IMAGE: registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.14-manylinux_2_24-py3.6
77
JOBNAME: hbci-tf114-cpu-${{ github.run_id }}
88
PODNAME: hbci-tf114-cpu-${{ github.run_id }}-chief-0
99
ARROW_CACHE_URL_PREFIX: ${{ secrets.ARROW_CACHE_URL_PREFIX }}
@@ -31,11 +31,13 @@ jobs:
3131
- name: Nightly Build
3232
if: github.event_name != 'push' || ! startsWith(github.ref, 'refs/tags')
3333
run: |-
34-
kubectl exec -it ${PODNAME} -- cibuild/patch_build -cpu-legacy a${{ github.run_id }}
34+
kubectl exec -it ${PODNAME} -- sh -c 'make -j32 HYBRIDBACKEND_WHEEL_ALIAS=-cpu-legacy-nightly HYBRIDBACKEND_WHEEL_BUILD=rc${{ github.run_id }}' && \
35+
kubectl exec -it ${PODNAME} -- cibuild/repair_dist
3536
- name: Release Build
3637
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
3738
run: |-
38-
kubectl exec -it ${PODNAME} -- cibuild/patch_build -cpu-legacy
39+
kubectl exec -it ${PODNAME} -- sh -c 'make -j32 HYBRIDBACKEND_WHEEL_ALIAS=-cpu-legacy HYBRIDBACKEND_WHEEL_REQUIRES="tensorflow>=1.14,<1.15"' && \
40+
kubectl exec -it ${PODNAME} -- cibuild/repair_dist
3941
- name: Check
4042
run: |-
4143
kubectl exec -it ${PODNAME} -- make cpu_test && \

‎.github/workflows/tensorflow1.15-py3.6-cibuild.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Tests with TensorFlow 1.15 on CPU
33
on: workflow_dispatch
44

55
env:
6-
IMAGE: registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.15-py3.6
6+
IMAGE: registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.15-manylinux_2_24-py3.6
77
JOBNAME: hbci-tf115-cpu-${{ github.run_id }}
88
PODNAME: hbci-tf115-cpu-${{ github.run_id }}-chief-0
99
ARROW_CACHE_URL_PREFIX: ${{ secrets.ARROW_CACHE_URL_PREFIX }}
@@ -31,14 +31,16 @@ jobs:
3131
- name: Nightly Build
3232
if: github.event_name != 'push' || ! startsWith(github.ref, 'refs/tags')
3333
run: |-
34-
kubectl exec -it ${PODNAME} -- cibuild/patch_build -cpu a${{ github.run_id }} "\"tensorflow>=1.15,<2.0\""
34+
kubectl exec -it ${PODNAME} -- sh -c 'make -j32 HYBRIDBACKEND_WHEEL_ALIAS=-cpu-nightly HYBRIDBACKEND_WHEEL_BUILD=rc${{ github.run_id }}' && \
35+
kubectl exec -it ${PODNAME} -- cibuild/repair_dist
3536
- name: Release Build
3637
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
3738
run: |-
38-
kubectl exec -it ${PODNAME} -- cibuild/patch_build -cpu "" "\"tensorflow>=1.15,<2.0\""
39+
kubectl exec -it ${PODNAME} -- sh -c 'make -j32 HYBRIDBACKEND_WHEEL_ALIAS=-cpu HYBRIDBACKEND_WHEEL_REQUIRES="tensorflow>=1.15,<2.0"' && \
40+
kubectl exec -it ${PODNAME} -- cibuild/repair_dist
3941
- name: Check
4042
run: |-
41-
kubectl exec -it ${PODNAME} -- make cpu_test && \
43+
kubectl exec -it ${PODNAME} -- make lint cpu_test && \
4244
kubectl exec -it ${PODNAME} -- tar -cf dist.tar dist/ && \
4345
kubectl cp ${PODNAME}:dist.tar ./dist.tar && \
4446
tar -xf ./dist.tar

‎Makefile‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ HYBRIDBACKEND_WITH_ARROW_S3 ?= ON
1313
HYBRIDBACKEND_WITH_ARROW_SIMD_LEVEL ?= AVX512
1414
HYBRIDBACKEND_WITH_TENSORFLOW ?= ON
1515
HYBRIDBACKEND_USE_CXX11_ABI ?= 1
16+
HYBRIDBACKEND_WHEEL_ALIAS ?= ""
17+
HYBRIDBACKEND_WHEEL_BUILD ?= ""
18+
HYBRIDBACKEND_WHEEL_REQUIRES ?= ""
1619

1720
CXX ?= gcc
1821
LOCAL_HOME ?= /usr/local
@@ -23,7 +26,7 @@ CFLAGS := -O3 -g \
2326
-D_GLIBCXX_USE_CXX11_ABI=$(HYBRIDBACKEND_USE_CXX11_ABI) \
2427
-isystem $(LOCAL_HOME) \
2528
-isystem $(PAI_HOME)/include \
26-
-Ihybridbackend/include \
29+
-I$(LIBNAME)/include \
2730
-I.
2831

2932
CXX_CFLAGS := -std=c++11 \
@@ -115,19 +118,23 @@ COMMON_LDFLAGS := $(COMMON_LDFLAGS) \
115118
-larrow_bundled_dependencies
116119
endif
117120

118-
-include hybridbackend/cpp/common/Makefile
121+
COMMON_LIB := $(LIBNAME)/lib$(LIBNAME).so
122+
-include $(LIBNAME)/cpp/common/Makefile
119123
CORE_DEPS := $(COMMON_LIB)
120124

121125
ifeq ($(HYBRIDBACKEND_WITH_TENSORFLOW),ON)
122-
TENSORFLOW_LIB := hybridbackend/tensorflow/lib$(LIBNAME)_tensorflow.so
123-
-include hybridbackend/cpp/tensorflow/Makefile
126+
TENSORFLOW_LIB := $(LIBNAME)/tensorflow/lib$(LIBNAME)_tensorflow.so
127+
-include $(LIBNAME)/cpp/tensorflow/Makefile
124128
CORE_DEPS := $(CORE_DEPS) $(TENSORFLOW_LIB)
125129
CFLAGS := $(CFLAGS) \
126130
-DHYBRIDBACKEND_TENSORFLOW=1
127131
endif
128132

129133
.PHONY: build
130134
build: $(CORE_DEPS)
135+
WHEEL_ALIAS=$(HYBRIDBACKEND_WHEEL_ALIAS) \
136+
WHEEL_BUILD=$(HYBRIDBACKEND_WHEEL_BUILD) \
137+
WHEEL_REQUIRES=$(HYBRIDBACKEND_WHEEL_REQUIRES) \
131138
python setup.py bdist_wheel
132139
@ls dist/*.whl
133140

@@ -157,12 +164,11 @@ cpu_test:
157164

158165
.PHONY: doc
159166
doc:
160-
PYTHONPATH=$(shell pwd) \
161167
sphinx-build -M html docs/ dist/doc
162168

163169
.PHONY: lint
164170
lint:
165-
tools/lint
171+
cibuild/lint
166172

167173
.PHONY: cibuild
168174
cibuild: lint build doc test

‎README.md‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,24 @@ information.
1414

1515
## Installation
1616

17-
Install latest CPU version:
17+
Install latest CPU version for TensorFlow 1.15:
1818

1919
```bash
20-
pip install hybridbackend-cpu # tensorflow/tensorflow:1.15.5-py3
20+
pip install hybridbackend-cpu
21+
```
22+
23+
Install latest CPU version for TensorFlow 1.14:
24+
25+
```bash
26+
pip install hybridbackend-cpu-legacy
27+
```
28+
29+
Note:
30+
31+
You might need to upgrade pip before above installations:
32+
33+
```bash
34+
pip install -U pip
2135
```
2236

2337
## Contributing

‎cibuild/auditwheel‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
POLICY_JSON=$(find / -type f -name manylinux-policy.json)
2+
3+
sed -i "s/libresolv.so.2\"/libresolv.so.2\", \"libtensorflow_framework.so.1\", \"libhybridbackend.so\", \"libcrypto-3cc778a5.so.1.1\", \"libssl-6db5df62.so.1.1\"/g" $POLICY_JSON
4+
5+
auditwheel $@
6+

cibuild/dockerfiles/Dockerfile.tensorflow-1.14.0-py3 renamed to cibuild/dockerfiles/Dockerfile.tensorflow-1.14.0-manylinux_2_24-py3

Lines changed: 24 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,50 @@
1-
FROM tensorflow/tensorflow:1.14.0-py3
1+
FROM quay.io/pypa/manylinux_2_24_x86_64
22

3-
COPY sources.list /etc/apt/sources.list
4-
5-
RUN rm -rf /etc/apt/sources.list.d/* && \
6-
ldconfig
3+
ENV PATH=/opt/python/cp36-cp36m/bin:$PATH
74

85
RUN apt-get update && \
9-
apt-get remove -y gcc gcc-7 gcc-7-base libgcc-7-dev gcc-8 && \
106
DEBIAN_FRONTEND="noninteractive" \
11-
apt-get install -y gcc-4.9 g++-4.9 gdb && \
12-
apt-get clean
13-
14-
RUN apt-get update && \
15-
DEBIAN_FRONTEND="noninteractive" \
16-
apt-get install -y \
7+
apt-get install -y --allow-unauthenticated \
178
cmake \
189
clang-format-3.9 \
19-
unzip \
2010
libssl-dev \
2111
libcurl4-openssl-dev \
22-
wget \
23-
curl && \
12+
wget && \
2413
apt-get clean && \
2514
ln -sf clang-format-3.9 /usr/bin/clang-format
2615

27-
COPY pip.conf /etc/pip.conf
16+
COPY sources.list /etc/apt/sources.list
17+
RUN rm -rf /etc/apt/sources.list.d/* && \
18+
ldconfig && apt-get update
19+
RUN apt-get update && \
20+
DEBIAN_FRONTEND="noninteractive" \
21+
apt-get install -y --allow-unauthenticated \
22+
gcc-4.9 g++-4.9
2823

29-
RUN pip install -U \
30-
setuptools wheel pip
3124
RUN pip install \
25+
pybind11 \
26+
sphinx \
27+
sphinx-rtd-theme \
28+
myst-parser \
29+
pandas \
30+
pyarrow \
31+
portpicker \
3232
six==1.16.0 \
3333
cffi==1.14.5 \
3434
urllib3==1.26.4 \
3535
requests==2.25.1 \
36-
pandas \
37-
pybind11 \
38-
pyarrow==5.0 \
39-
numpy==1.16.6 \
4036
isort==4.3.18 \
41-
sphinx \
42-
sphinx-rtd-theme \
43-
myst-parser \
44-
pylint==1.9.4 \
4537
toposort==1.5 \
46-
portpicker \
47-
docutils==0.16
38+
docutils==0.16 \
39+
numpy==1.16.6 \
40+
pylint==1.9.4
41+
42+
RUN pip install \
43+
tensorflow==1.14.0
4844

4945
ENV CXX=/usr/bin/g++-4.9 \
5046
HYBRIDBACKEND_WITH_CUDA=OFF \
5147
HYBRIDBACKEND_WITH_ARROW_ZEROCOPY=OFF \
5248
HYBRIDBACKEND_WITH_ARROW_SIMD_LEVEL=AVX2 \
5349
HYBRIDBACKEND_WITH_GRENDEZ=OFF \
5450
HYBRIDBACKEND_USE_CXX11_ABI=0
55-

cibuild/dockerfiles/Dockerfile.tensorflow-1.15.5-py3 renamed to cibuild/dockerfiles/Dockerfile.tensorflow-1.15.5-manylinux_2_24-py3

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,38 @@
1-
FROM tensorflow/tensorflow:1.15.5-py3
1+
FROM quay.io/pypa/manylinux_2_24_x86_64
22

3-
COPY sources.list /etc/apt/sources.list
4-
5-
RUN rm -rf /etc/apt/sources.list.d/* && \
6-
ldconfig
3+
ENV PATH=/opt/python/cp36-cp36m/bin:$PATH
74

85
RUN apt-get update && \
96
DEBIAN_FRONTEND="noninteractive" \
10-
apt-get install -y \
7+
apt-get install -y --allow-unauthenticated \
118
cmake \
129
clang-format-3.9 \
13-
unzip \
1410
libssl-dev \
1511
libcurl4-openssl-dev \
16-
wget \
17-
curl && \
12+
wget && \
1813
apt-get clean && \
1914
ln -sf clang-format-3.9 /usr/bin/clang-format
2015

21-
COPY pip.conf /etc/pip.conf
22-
23-
RUN pip install -U \
24-
setuptools wheel pip
2516
RUN pip install \
17+
pybind11 \
18+
sphinx \
19+
sphinx-rtd-theme \
20+
myst-parser \
21+
pandas \
22+
pyarrow \
23+
portpicker \
2624
six==1.16.0 \
2725
cffi==1.14.5 \
2826
urllib3==1.26.4 \
2927
requests==2.25.1 \
30-
pandas \
31-
pybind11 \
32-
pyarrow==5.0 \
33-
numpy==1.16.6 \
3428
isort==4.3.18 \
35-
sphinx \
36-
sphinx-rtd-theme \
37-
myst-parser \
38-
pylint==1.9.4 \
3929
toposort==1.5 \
40-
portpicker \
41-
docutils==0.16
30+
docutils==0.16 \
31+
numpy==1.16.6 \
32+
pylint==1.9.4
33+
34+
RUN pip install \
35+
tensorflow==1.15.5
4236

4337
ENV HYBRIDBACKEND_WITH_CUDA=OFF \
4438
HYBRIDBACKEND_WITH_ARROW_ZEROCOPY=ON \

cibuild/patch_build renamed to cibuild/repair_dist

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,8 @@
1616

1717
set -eo pipefail
1818

19-
PATCH=$1
20-
shift
21-
REVISION=$1
22-
shift
23-
24-
sed -i "s/__version__ = '\(.*\)'/__version__ = '\1${REVISION}'/g" hybridbackend/__init__.py
25-
sed -i "s/name='\(.*\)'/name='\1${PATCH}'/g" setup.py
26-
sed -i "s/install_requires=\[\]/install_requires=[$@]/g" setup.py
27-
make -j$(nproc)
19+
mkdir -p build/
20+
mv dist build/dist
21+
mkdir -p dist/
22+
cibuild/auditwheel repair build/dist/*.whl -w dist/
2823

‎cibuild/run‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ if [[ -z "$DOCKER_RUN_ARGS" ]]; then
3636
fi
3737

3838
if [[ -z "$DOCKER_RUN_IMAGE" ]]; then
39-
DOCKER_RUN_IMAGE=registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.15-py3.6
39+
DOCKER_RUN_IMAGE=registry.cn-shanghai.aliyuncs.com/pai-dlc/hybridbackend:developer-tensorflow1.15-manylinux_2_24-py3.6
4040
fi
4141

4242
DIR=$(pwd)

‎docs/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ sphinx_rtd_theme
33
myst-parser
44
docutils==0.16
55
tensorflow==1.15.5
6-
hybridbackend-cpu
6+
hybridbackend-cpu-nightly

0 commit comments

Comments
 (0)