Skip to content

Commit 90a640a

Browse files
coryancoryan
authored
Simplify the TEST_INSTALL builds. (#1751)
Instead of installing all the dependencies during the build script, install them in the build docker image. This change has almost no effect on the CI builds, but it makes running those builds locally, in a developer workstation, far easier (and faster).
1 parent 65c16e4 commit 90a640a

4 files changed

Lines changed: 189 additions & 131 deletions

File tree

‎.travis.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ matrix:
5252
compiler: clang
5353
env:
5454
TEST_INSTALL=yes
55-
DISTRO=ubuntu
55+
DISTRO=ubuntu-install
5656
DISTRO_VERSION=18.04
5757
if: repo =~ ^GoogleCloudPlatform/ or head_repo =~ ^GoogleCloudPlatform/
5858
- # Verify that we can compile shared libraries, and verify that the ABI is
@@ -65,7 +65,7 @@ matrix:
6565
TEST_INSTALL=yes
6666
CMAKE_FLAGS=-DBUILD_SHARED_LIBS=yes
6767
BUILD_TYPE=Debug
68-
DISTRO=ubuntu
68+
DISTRO=ubuntu-install
6969
DISTRO_VERSION=18.04
7070
if: repo =~ ^GoogleCloudPlatform/ or head_repo =~ ^GoogleCloudPlatform/
7171
- # Run clang-tidy, clang-format, and generate the documentation.
@@ -141,7 +141,7 @@ matrix:
141141
TEST_INSTALL=yes
142142
CMAKE_FLAGS=-DBUILD_SHARED_LIBS=yes
143143
BUILD_TYPE=Debug
144-
DISTRO=ubuntu
144+
DISTRO=ubuntu-install
145145
DISTRO_VERSION=18.04
146146

147147
script:

‎ci/travis/Dockerfile.ubuntu-install‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ARG DISTRO_VERSION=18.04
16+
FROM ubuntu:${DISTRO_VERSION}
17+
MAINTAINER "Carlos O'Ryan <coryan@google.com>"
18+
19+
RUN apt update && \
20+
apt install -y \
21+
abi-compliance-checker \
22+
abi-dumper \
23+
automake \
24+
build-essential \
25+
ccache \
26+
clang \
27+
clang-format \
28+
cmake \
29+
curl \
30+
doxygen \
31+
gawk \
32+
git \
33+
gcc \
34+
g++ \
35+
cmake \
36+
libcurl4-openssl-dev \
37+
libssl-dev \
38+
libtool \
39+
lsb-release \
40+
make \
41+
pkg-config \
42+
python-pip \
43+
tar \
44+
wget \
45+
zlib1g-dev
46+
47+
# By default, Ubuntu 18.04 does not install the alternatives for clang-format
48+
# and clang-tidy, so we need to manually install those.
49+
RUN if grep -q 18.04 /etc/lsb-release; then \
50+
apt update && apt install -y clang-tidy clang-format clang-tools; \
51+
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-6.0 100; \
52+
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-6.0 100; \
53+
update-alternatives --install /usr/bin/scan-build scan-build /usr/bin/scan-build-6.0 100; \
54+
fi
55+
56+
# Install the the buildifier tool, which does not compile with the default
57+
# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
58+
RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
59+
RUN chmod 755 /usr/bin/buildifier
60+
61+
# Install cmake_format to automatically format the CMake list files.
62+
# https://github.com/cheshirekow/cmake_format
63+
# Pin this to an specific version because the formatting changes when the
64+
# "latest" version is updated, and we do not want the builds to break just
65+
# because some third party changed something.
66+
RUN pip install --upgrade pip
67+
RUN pip install numpy cmake_format==0.4.0
68+
69+
# Install Python packages used in the integration tests.
70+
RUN pip install flask httpbin gevent gunicorn crc32c
71+
72+
# Install the Cloud Bigtable emulator and the Cloud Bigtable command-line
73+
# client. They are used in the integration tests.
74+
WORKDIR /var/tmp/install/cbt-components
75+
RUN wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-201.0.0-linux-x86_64.tar.gz
76+
RUN tar x -C /usr/local -f google-cloud-sdk-201.0.0-linux-x86_64.tar.gz
77+
RUN /usr/local/google-cloud-sdk/bin/gcloud --quiet components install cbt bigtable
78+
RUN /usr/local/google-cloud-sdk/bin/gcloud --quiet components update || \
79+
echo "Ignoring update failure for Google Cloud SDK"
80+
81+
# Parallelize the builds if possible. The default is chosen to work well on
82+
# Travis, but developers may want to override this value when building on their
83+
# workstations.
84+
ARG NCPU=2
85+
86+
# Install Crc32c library.
87+
WORKDIR /var/tmp/build
88+
RUN wget -q https://github.com/google/crc32c/archive/1.0.6.tar.gz
89+
RUN tar -xf 1.0.6.tar.gz
90+
WORKDIR /var/tmp/build/crc32c-1.0.6
91+
RUN cmake \
92+
-DCMAKE_BUILD_TYPE=Release \
93+
-DBUILD_SHARED_LIBS=yes \
94+
-DCRC32C_BUILD_TESTS=OFF \
95+
-DCRC32C_BUILD_BENCHMARKS=OFF \
96+
-DCRC32C_USE_GLOG=OFF \
97+
-H. -B.build/crc32c
98+
RUN cmake --build .build/crc32c --target install -- -j ${NCPU}
99+
RUN ldconfig
100+
101+
# Install protobuf using CMake. Some distributions include protobuf, but gRPC
102+
# requires 3.4.x or newer, and many of those distribution use older versions.
103+
# We need to install both the debug and Release version because:
104+
# - When using pkg-config, only the release version works, the pkg-config
105+
# file is hard-coded to search for `libprotobuf.so` (or `.a`)
106+
# - When using CMake, only the version compiled with the same CMAKE_BUILD_TYPE
107+
# as the dependent (gRPC or google-cloud-cpp) works.
108+
WORKDIR /var/tmp/build
109+
RUN wget -q https://github.com/google/protobuf/archive/v3.6.1.tar.gz
110+
RUN tar -xf v3.6.1.tar.gz
111+
WORKDIR /var/tmp/build/protobuf-3.6.1/cmake
112+
RUN for build_type in "Debug" "Release"; do \
113+
cmake \
114+
-DCMAKE_BUILD_TYPE="${build_type}" \
115+
-DBUILD_SHARED_LIBS=yes \
116+
-Dprotobuf_BUILD_TESTS=OFF \
117+
-H. -B.build-${build_type}; \
118+
cmake --build .build-${build_type} --target install -- -j ${NCPU}; \
119+
done
120+
RUN ldconfig
121+
122+
# Many distributions include c-ares, but they do not include the CMake support
123+
# files for the library, so manually install it. c-ares requires two install
124+
# steps because (1) the CMake-based build does not install pkg-config files,
125+
# and (2) the Makefile-based build does not install CMake config files.
126+
WORKDIR /var/tmp/build
127+
RUN apt-get remove -y libc-ares-dev libc-ares2
128+
RUN wget -q https://github.com/c-ares/c-ares/archive/cares-1_14_0.tar.gz
129+
RUN tar -xf cares-1_14_0.tar.gz
130+
WORKDIR /var/tmp/build/c-ares-cares-1_14_0
131+
RUN cmake \
132+
-DCMAKE_BUILD_TYPE="Release" \
133+
-DBUILD_SHARED_LIBS=yes \
134+
-H. -B.build
135+
RUN cmake --build .build --target install -- -j ${NCPU}
136+
RUN ./buildconf
137+
RUN ./configure
138+
RUN install -m 644 -D -t /usr/local/lib/pkgconfig libcares.pc
139+
RUN ldconfig
140+
141+
# Install gRPC. Note that we use the system's zlib and ssl libraries.
142+
# For similar reasons to c-ares (see above), we need two install steps.
143+
WORKDIR /var/tmp/build
144+
RUN wget -q https://github.com/grpc/grpc/archive/v1.16.1.tar.gz
145+
RUN tar -xf v1.16.1.tar.gz
146+
RUN ls -l
147+
WORKDIR /var/tmp/build/grpc-1.16.1
148+
RUN ls -l
149+
RUN cmake \
150+
-DCMAKE_BUILD_TYPE="Release" \
151+
-DBUILD_SHARED_LIBS=yes \
152+
-DgRPC_BUILD_TESTS=OFF \
153+
-DgRPC_ZLIB_PROVIDER=package \
154+
-DgRPC_SSL_PROVIDER=package \
155+
-DgRPC_CARES_PROVIDER=package \
156+
-DgRPC_PROTOBUF_PROVIDER=package \
157+
-H. -B.build/grpc
158+
RUN cmake --build .build/grpc --target install -- -j ${NCPU}
159+
RUN make install-pkg-config_c install-pkg-config_cxx install-certs
160+
RUN ldconfig
161+
162+
# Install googletest.
163+
WORKDIR /var/tmp/build
164+
RUN wget -q https://github.com/google/googletest/archive/release-1.8.1.tar.gz
165+
RUN tar -xf release-1.8.1.tar.gz
166+
WORKDIR /var/tmp/build/googletest-release-1.8.1
167+
RUN cmake \
168+
-DCMAKE_BUILD_TYPE="Release" \
169+
-DBUILD_SHARED_LIBS=yes \
170+
-H. -B.build/googletest
171+
RUN cmake --build .build/googletest --target install -- -j ${NCPU}
172+
RUN ldconfig
173+
174+
RUN find /usr/local -type d | xargs chmod 777

‎ci/travis/build-docker.sh‎

Lines changed: 1 addition & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ source "${BINDIR}/../colors.sh"
2929
# ci/Dockerfile.* build scripts.
3030
readonly IMAGE="cached-${DISTRO}-${DISTRO_VERSION}"
3131
readonly BUILD_DIR="build-output/${IMAGE}"
32+
readonly STAGING_DIR="build-output/${IMAGE}-staging"
3233

3334
CMAKE_COMMAND="cmake"
3435
if [ "${SCAN_BUILD}" = "yes" ]; then
@@ -37,128 +38,6 @@ fi
3738

3839
ccache_command="$(which ccache)"
3940

40-
function install_crc32c {
41-
# Install googletest.
42-
echo "${COLOR_YELLOW}Installing Crc32c $(date)${COLOR_RESET}"
43-
wget -q https://github.com/google/crc32c/archive/1.0.6.tar.gz
44-
tar -xf 1.0.6.tar.gz
45-
cd crc32c-1.0.6
46-
env CXX="${cached_cxx}" CC="${cached_cc}" cmake \
47-
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
48-
-DCRC32C_BUILD_TESTS=OFF \
49-
-DCRC32C_BUILD_BENCHMARKS=OFF \
50-
-DCRC32C_USE_GLOG=OFF \
51-
${CMAKE_FLAGS} \
52-
-H. -B.build/crc32c
53-
cmake --build .build/crc32c --target install -- -j ${NCPU}
54-
ldconfig
55-
}
56-
57-
function install_protobuf {
58-
# Install protobuf using CMake. Some distributions include protobuf, but gRPC
59-
# requires 3.4.x or newer, and many of those distribution use older versions.
60-
# We need to install both the debug and Release version because:
61-
# - When using pkg-config, only the release version works, the pkg-config
62-
# file is hard-coded to search for `libprotobuf.so` (or `.a`)
63-
# - When using CMake, only the version compiled with the same CMAKE_BUILD_TYPE
64-
# as the dependent (gRPC or google-cloud-cpp) works.
65-
echo "${COLOR_YELLOW}Installing protobuf $(date)${COLOR_RESET}"
66-
wget -q https://github.com/google/protobuf/archive/v3.6.1.tar.gz
67-
tar -xf v3.6.1.tar.gz
68-
cd protobuf-3.6.1/cmake
69-
for build_type in "Debug" "Release"; do
70-
env CXX="${cached_cxx}" CC="${cached_cc}" cmake \
71-
-DCMAKE_BUILD_TYPE="${build_type}" \
72-
-Dprotobuf_BUILD_TESTS=OFF \
73-
${CMAKE_FLAGS} \
74-
-H. -B.build-${build_type}
75-
cmake --build .build-${build_type} --target install -- -j ${NCPU}
76-
done
77-
ldconfig
78-
}
79-
80-
function install_c_ares {
81-
# Many distributions include c-ares, but they do not include the CMake support
82-
# files for the library, so manually install it. c-ares requires two install
83-
# steps because (1) the CMake-based build does not install pkg-config files,
84-
# and (2) the Makefile-based build does not install CMake config files.
85-
echo "${COLOR_YELLOW}Installing c-ares $(date)${COLOR_RESET}"
86-
apt-get remove -y libc-ares-dev libc-ares2
87-
wget -q https://github.com/c-ares/c-ares/archive/cares-1_14_0.tar.gz
88-
tar -xf cares-1_14_0.tar.gz
89-
cd c-ares-cares-1_14_0
90-
env CXX="${cached_cxx}" CC="${cached_cc}" cmake \
91-
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
92-
${CMAKE_FLAGS} \
93-
-H. -B.build; \
94-
cmake --build .build --target install -- -j ${NCPU}
95-
./buildconf
96-
./configure
97-
install -m 644 -D -t /usr/local/lib/pkgconfig libcares.pc
98-
ldconfig
99-
}
100-
101-
function install_grpc {
102-
# Install gRPC. Note that we use the system's zlib and ssl libraries.
103-
# For similar reasons to c-ares (see above), we need two install steps.
104-
echo "${COLOR_YELLOW}Installing gRPC $(date)${COLOR_RESET}"
105-
wget -q https://github.com/grpc/grpc/archive/v1.16.1.tar.gz
106-
tar -xf v1.16.1.tar.gz
107-
cd grpc-1.16.1
108-
env CXX="${cached_cxx}" CC="${cached_cc}" cmake \
109-
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
110-
-DgRPC_BUILD_TESTS=OFF \
111-
-DgRPC_ZLIB_PROVIDER=package \
112-
-DgRPC_SSL_PROVIDER=package \
113-
-DgRPC_CARES_PROVIDER=package \
114-
-DgRPC_PROTOBUF_PROVIDER=package \
115-
${CMAKE_FLAGS} \
116-
-H. -B.build/grpc
117-
cmake --build .build/grpc --target install -- -j ${NCPU}
118-
make install-pkg-config_c install-pkg-config_cxx install-certs
119-
ldconfig
120-
}
121-
122-
function install_googletest {
123-
# Install googletest.
124-
echo "${COLOR_YELLOW}Installing googletest $(date)${COLOR_RESET}"
125-
wget -q https://github.com/google/googletest/archive/release-1.8.1.tar.gz
126-
tar -xf release-1.8.1.tar.gz
127-
cd googletest-release-1.8.1
128-
env CXX="${cached_cxx}" CC="${cached_cc}" cmake \
129-
-DCMAKE_BUILD_TYPE="Release" \
130-
${CMAKE_FLAGS} \
131-
-H. -B.build/googletest
132-
cmake --build .build/googletest --target install -- -j ${NCPU}
133-
ldconfig
134-
}
135-
136-
if [ "${TEST_INSTALL:-}" = "yes" -o "${SCAN_BUILD:-}" = "yes" ]; then
137-
echo
138-
echo "${COLOR_YELLOW}Started dependency install at: $(date)${COLOR_RESET}"
139-
echo
140-
echo "travis_fold:start:install-dependencies"
141-
echo
142-
mkdir -p /var/tmp/build-dependencies
143-
cached_cxx="${CXX}"
144-
cached_cc="${CC}"
145-
if [ -n "${ccache_command}" ]; then
146-
cached_cxx="${ccache_command} ${CXX}"
147-
cached_cc="${ccache_command} ${CC}"
148-
fi
149-
(cd /var/tmp/build-dependencies; install_crc32c)
150-
(cd /var/tmp/build-dependencies; install_protobuf)
151-
(cd /var/tmp/build-dependencies; install_c_ares)
152-
(cd /var/tmp/build-dependencies; install_grpc)
153-
(cd /var/tmp/build-dependencies; install_googletest)
154-
"${ccache_command}" --show-stats
155-
echo
156-
echo "${COLOR_YELLOW}Finished dependency install at: $(date)${COLOR_RESET}"
157-
echo
158-
echo "travis_fold:end:install-dependencies"
159-
echo
160-
fi
161-
16241
echo "${COLOR_YELLOW}Started CMake config at: $(date)${COLOR_RESET}"
16342
echo "travis_fold:start:configure-cmake"
16443
# Tweak configuration for TEST_INSTALL=yes and SCAN_BUILD=yes builds.

‎ci/travis/build-linux.sh‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,25 @@
1616

1717
set -eu
1818

19-
if [ "${TRAVIS_OS_NAME}" != "linux" ]; then
19+
if [[ "${TRAVIS_OS_NAME}" != "linux" ]]; then
2020
echo "Not a Linux-based build, skipping Linux-specific build steps."
2121
exit 0
2222
fi
2323

2424
readonly IMAGE="cached-${DISTRO}-${DISTRO_VERSION}"
2525

26-
# TEST_INSTALL=yes builds work better as root, but other builds should avoid
27-
# creating root-owned files in the build directory.
26+
# The default user for a Docker container has uid 0 (root). To avoid creating
27+
# root-owned files in the build directory we tell docker to use the current
28+
# user ID, if known.
2829
docker_uid="${UID:-0}"
29-
docker_home=""
30-
if [ "${TEST_INSTALL:-}" = "yes" -o "${SCAN_BUILD:-}" = "yes" ]; then
31-
docker_uid=0
30+
if [[ "${docker_uid}" == "0" ]]; then
31+
# If the UID is 0, then the HOME directory will be set to /root, and we
32+
# need to mount the ccache files is /root/.ccache.
3233
docker_home=/root
34+
else
35+
# If the UID is not zero then HOME is "/", and we need to mount the ccache
36+
# files in /.ccache.
37+
docker_home=""
3338
fi
3439

3540
# Use a volume to store the cache files. This exports the cache files from the

0 commit comments

Comments
 (0)