Merge pull request #8889 from FirebirdSQL/work/out-of-tree-build-linux

Out-of-tree builds on Linux and ARM build performance improvements

Author: asfernandes

.github/workflows/main.yml

@@ -39,7 +39,7 @@ jobs:
 
   build-linux-docker:
     name: build-linux-docker-${{ matrix.arch }}
-    runs-on: ubuntu-24.04
+    runs-on: ${{ (startsWith(matrix.arch, 'arm') && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }}
     env:
       FB_DOCKER_PATH: ${{ (startsWith(matrix.arch, 'arm') && 'arm32-arm64') || 'x86-x64' }}
 

.gitignore

@@ -28,5 +28,4 @@ extern/ttmath/release/
 /src/include/gen/parse.h
 /src/include/gen/autoconfig.auto
 /src/include/gen/autoconfig.h
-extern/libcds/lib/
 /vcpkg_installed/

autogen.sh

@@ -5,6 +5,15 @@
 
 PKG_NAME=Firebird6
 SRCDIR=`dirname $0`
+SRCDIR=`(cd "$SRCDIR" && pwd)`
+OBJDIR=`pwd`
+
+if [ "$OBJDIR" = "$SRCDIR" ]
+then
+  BUILD_MODE=in-tree
+else
+  BUILD_MODE=out-of-tree
+fi
 
 if [ -z "$AUTORECONF" ]
 then
@@ -17,8 +26,11 @@ echo "AUTORECONF="$AUTORECONF
 AUTOMAKE=true
 export AUTOMAKE
 
-# This helps some old aclocal versions find binreloc.m4 in current directory
-ACLOCAL='aclocal -I .'
+# This helps some old aclocal versions find local and source m4 macros
+ACLOCAL="aclocal -I $OBJDIR -I $OBJDIR/m4 -I $SRCDIR"
+if [ -d "$SRCDIR/m4" ]; then
+	ACLOCAL="$ACLOCAL -I $SRCDIR/m4"
+fi
 export ACLOCAL
 
 # Give a warning if no arguments to 'configure' have been supplied.
@@ -30,9 +42,9 @@ if test -z "$*" -a x$NOCONFIGURE = x; then
 fi
 
 # Some versions of autotools need it
-if [ ! -d m4 ]; then
-	rm -rf m4
-	mkdir m4
+if [ ! -d "$OBJDIR/m4" ]; then
+	rm -rf "$OBJDIR/m4"
+	mkdir -p "$OBJDIR/m4"
 fi
 
 # Ensure correct utilities are called by AUTORECONF
@@ -42,28 +54,49 @@ if [ "x$autopath" != "x" ]; then
 	export PATH
 fi
 
-echo "Running autoreconf ..."
-$AUTORECONF --install --force --verbose || exit 1
+if [ "$BUILD_MODE" = "out-of-tree" ]
+then
+	mkdir -p "$OBJDIR/builds/make.new"
+	ln -sf "$SRCDIR/configure.ac" "$OBJDIR/configure.ac"
+	ln -sf "$SRCDIR/acx_pthread.m4" "$OBJDIR/acx_pthread.m4"
+	ln -sf "$SRCDIR/binreloc.m4" "$OBJDIR/binreloc.m4"
+fi
+
+echo "Running autoreconf in $OBJDIR ($BUILD_MODE) ..."
+(cd "$OBJDIR" && $AUTORECONF --install --force --verbose) || exit 1
+
+if [ "$BUILD_MODE" = "out-of-tree" ]
+then
+	# Prefer auxiliary files generated in the build directory when running configure
+	sed -i 's#ac_aux_dir_candidates="${srcdir}/builds/make.new/config"#ac_aux_dir_candidates="$ac_pwd/builds/make.new/config${PATH_SEPARATOR}${srcdir}/builds/make.new/config"#' "$OBJDIR/configure"
+fi
 
 # Hack to bypass bug in autoreconf - --install switch not passed to libtoolize,
 # therefore missing config.sub and confg.guess files
-CONFIG_AUX_DIR=builds/make.new/config
-if [ ! -f $CONFIG_AUX_DIR/config.sub -o ! -f $CONFIG_AUX_DIR/config.guess ]; then
+CONFIG_AUX_DIR="$OBJDIR/builds/make.new/config"
+if [ ! -f "$CONFIG_AUX_DIR/config.sub" -o ! -f "$CONFIG_AUX_DIR/config.guess" ]; then
 	# re-run libtoolize with --install switch, if it does not understand that switch
 	# and there are no config.sub/guess files in CONFIG_AUX_DIR, we will anyway fail
 	echo "Re-running libtoolize ..."
 	if [ -z "$LIBTOOLIZE" ]; then
 		LIBTOOLIZE=libtoolize
 	fi
-	$LIBTOOLIZE --install --copy --force || exit 1
+	(cd "$OBJDIR" && $LIBTOOLIZE --install --copy --force) || exit 1
 fi
 
 # If NOCONFIGURE is set, skip the call to configure
 if test "x$NOCONFIGURE" = "x"; then
-  echo Running $SRCDIR/configure $conf_flags "$@" ...
+  if [ "$BUILD_MODE" = "out-of-tree" ]
+  then
+    CONFIGURE_CMD="$OBJDIR/configure --srcdir=$SRCDIR"
+  else
+    CONFIGURE_CMD="$OBJDIR/configure"
+  fi
+
+  echo Running $CONFIGURE_CMD $conf_flags "$@" ...
   rm -f config.cache config.log
-  chmod a+x $SRCDIR/configure
-  $SRCDIR/configure $conf_flags "$@" \
+  chmod a+x "$OBJDIR/configure"
+  $CONFIGURE_CMD $conf_flags "$@" \
   && echo Now type \`make\' to compile $PKG_NAME
 else
   echo Autogen skipping configure process.

builds/docker/linux/arm32-arm64/Dockerfile

@@ -7,15 +7,15 @@ ARG ARG_LIBTOOL_VERSION=2.4.6
 ARG ARG_NCURSES_VERSION=6.3
 ARG ARG_LTM_VERSION=1.2.0
 ARG ARG_ZLIB_VERSION=1.3.1
-ARG ARG_ICU_VERSION=70-1
+ARG ARG_ICU_VERSION=77-1
 
 ARG ARG_CPUCOUNT=24
 
 ARG ARG_CTNG_UID=1000
 ARG ARG_CTNG_GID=1000
 
 
-FROM debian:bookworm as builder
+FROM $ARG_BASE AS builder
 
 ARG ARG_TARGET_ARCH
 ARG ARG_CTNF_CONFIG
@@ -25,7 +25,6 @@ ARG ARG_LIBTOOL_VERSION
 ARG ARG_NCURSES_VERSION
 ARG ARG_LTM_VERSION
 ARG ARG_ZLIB_VERSION
-ARG ARG_ICU_VERSION
 
 ARG ARG_CPUCOUNT
 
@@ -71,10 +70,9 @@ RUN groupadd -g $ARG_CTNG_GID ctng && \
 
 USER ctng
 
-COPY --from=asfernandes/firebird-builder:fb6-x64-ng-v1 --chown=ctng:ctng /home/ctng/x-tools /home/ctng/x-tools
+COPY --from=firebirdsql/firebird-builder-linux:fb6-x64-ng-v2 --chown=ctng:ctng /home/ctng/x-tools /home/ctng/x-tools
 
 RUN mkdir ~/build && \
-	curl -SL --output ~/build/icu4c-${ARG_ICU_VERSION}-src.tgz https://github.com/unicode-org/icu/releases/download/release-${ARG_ICU_VERSION}/icu4c-`echo ${ARG_ICU_VERSION} | tr - _`-src.tgz && \
 	curl -SL --output ~/build/ncurses-${ARG_NCURSES_VERSION}.tar.gz https://ftp.gnu.org/pub/gnu/ncurses/ncurses-${ARG_NCURSES_VERSION}.tar.gz && \
 	curl -SL --output ~/build/libtool-${ARG_LIBTOOL_VERSION}.tar.gz https://ftpmirror.gnu.org/libtool/libtool-${ARG_LIBTOOL_VERSION}.tar.gz && \
 	curl -SL --output ~/build/ltm-${ARG_LTM_VERSION}.tar.xz https://github.com/libtom/libtommath/releases/download/v${ARG_LTM_VERSION}/ltm-${ARG_LTM_VERSION}.tar.xz && \
@@ -110,8 +108,8 @@ RUN sudo apt-get -y remove \
 		libtomcrypt-dev && \
 	sudo rm -rf /var/lib/apt/lists/*
 
-ENV PATH "/home/ctng/x-tools/x86_64-pc-linux-gnu/bin:/home/ctng/x-tools/x86_64-pc-linux-gnu/x86_64-pc-linux-gnu/sysroot/usr/local/bin:${PATH}"
-ENV PATH "/home/ctng/x-tools/${ARG_TARGET_ARCH}/bin:/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/bin:${PATH}"
+ENV PATH="/home/ctng/x-tools/x86_64-pc-linux-gnu/bin:/home/ctng/x-tools/x86_64-pc-linux-gnu/x86_64-pc-linux-gnu/sysroot/usr/local/bin:${PATH}"
+ENV PATH="/home/ctng/x-tools/${ARG_TARGET_ARCH}/bin:/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/bin:${PATH}"
 
 RUN cd ~/build && \
 	mkdir -p libtool-${ARG_LIBTOOL_VERSION}-src && \
@@ -180,38 +178,20 @@ RUN cd ~/build && \
 	make -j${ARG_CPUCOUNT} && \
 	make install
 
-RUN cd ~/build && \
-	mkdir icu4c-${ARG_ICU_VERSION}-src && \
-	tar xvf icu4c-${ARG_ICU_VERSION}-src.tgz --strip 1 -C icu4c-${ARG_ICU_VERSION}-src && \
-	mkdir icu4c-${ARG_ICU_VERSION}-build-x86_64 && \
-	cd icu4c-${ARG_ICU_VERSION}-build-x86_64 && \
-	CXXFLAGS='-std=c++20 -static-libstdc++' ../icu4c-${ARG_ICU_VERSION}-src/source/runConfigureICU \
-		Linux \
-		--host=x86_64-pc-linux-gnu && \
-	make -j${ARG_CPUCOUNT}
-
-RUN cd ~/build && \
-	mkdir icu4c-${ARG_ICU_VERSION}-build && \
-	cd icu4c-${ARG_ICU_VERSION}-build && \
-	CXXFLAGS='-std=c++20 -static-libstdc++' ../icu4c-${ARG_ICU_VERSION}-src/source/runConfigureICU \
-		Linux \
-		--host=${ARG_TARGET_ARCH} \
-		--with-cross-build=/home/ctng/build/icu4c-${ARG_ICU_VERSION}-build-x86_64 \
-		--prefix=/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local \
-		--includedir=/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/include && \
-	make -j${ARG_CPUCOUNT} && \
-	make install
-
 
 FROM $ARG_BASE
 
 ARG ARG_TARGET_ARCH
 ARG ARG_CTNG_UID
 ARG ARG_CTNG_GID
+ARG ARG_ICU_VERSION
+ARG ARG_CPUCOUNT
 
 RUN apt-get update && \
 	apt-get -y install \
 		cmake \
+		gosu \
+		curl \
 		libc6-amd64-cross \
 		libfile-copy-recursive-perl \
 		unzip && \
@@ -221,20 +201,36 @@ RUN ln -s /usr/x86_64-linux-gnu/lib64 /lib64 && \
 	groupadd -g $ARG_CTNG_GID ctng && \
 	useradd -d /home/ctng -m -g $ARG_CTNG_GID -u $ARG_CTNG_UID -s /bin/bash ctng
 
-USER ctng
-
-RUN chmod o=u /home/ctng
+RUN mkdir /home/ctng/firebird-build && \
+	chown ctng:ctng /home/ctng/firebird-build && \
+	chmod o=u /home/ctng
 
 COPY --from=builder --chown=ctng:ctng /home/ctng/x-tools /home/ctng/x-tools
 COPY --chown=ctng:ctng scripts/* /
 
 # Let non-emulated x86_64-linux-gnu make run.
 RUN rm /home/ctng/x-tools/${ARG_TARGET_ARCH}/bin/make
 
-ENV PATH "/home/ctng/x-tools/${ARG_TARGET_ARCH}/bin:/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/bin:/home/ctng/x-tools/x86_64-pc-linux-gnu/bin:${PATH}"
+ENV PATH="/home/ctng/x-tools/${ARG_TARGET_ARCH}/bin:/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/bin:/home/ctng/x-tools/x86_64-pc-linux-gnu/bin:${PATH}"
 ENV LIBRARY_PATH "/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/lib"
 ENV LD_LIBRARY_PATH "/usr/x86_64-linux-gnu/lib:/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr/local/lib"
 
+RUN mkdir ~/build && \
+	cd ~/build && \
+	curl -SL --output icu4c-${ARG_ICU_VERSION}-src.tgz https://github.com/unicode-org/icu/releases/download/release-${ARG_ICU_VERSION}/icu4c-`echo ${ARG_ICU_VERSION} | tr - _`-src.tgz && \
+	mkdir icu4c-${ARG_ICU_VERSION}-src && \
+	tar xvf icu4c-${ARG_ICU_VERSION}-src.tgz --strip 1 -C icu4c-${ARG_ICU_VERSION}-src && \
+	mkdir icu4c-${ARG_ICU_VERSION}-build && \
+	cd icu4c-${ARG_ICU_VERSION}-build && \
+	CXXFLAGS='-std=c++20 -static-libstdc++' ../icu4c-${ARG_ICU_VERSION}-src/source/runConfigureICU \
+		Linux \
+		--host=${ARG_TARGET_ARCH} \
+		--prefix=/home/ctng/x-tools/${ARG_TARGET_ARCH}/${ARG_TARGET_ARCH}/sysroot/usr && \
+	make -j${ARG_CPUCOUNT} && \
+	make install && \
+	cd / && \
+	rm -rf ~/build
+
 ENV BUILD_ARCH=$ARG_TARGET_ARCH
 
 WORKDIR /firebird

builds/docker/linux/arm32-arm64/build-arm32.sh

@@ -1,7 +1,11 @@
 #!/bin/sh
+ARG_NATIVE_ARCH=linux/arm
 docker buildx build \
+	--progress=plain \
+	--platform $ARG_NATIVE_ARCH \
 	--pull \
+	--build-arg ARG_NATIVE_ARCH=$ARG_NATIVE_ARCH \
 	--build-arg ARG_BASE=arm32v7/debian:bookworm \
 	--build-arg ARG_TARGET_ARCH=arm-pc-linux-gnueabihf \
 	--build-arg ARG_CTNF_CONFIG=crosstool-ng-config-arm32 \
-	-t asfernandes/firebird-builder:fb6-arm32-ng-v1 .
+	-t firebirdsql/firebird-builder-linux:fb6-arm32-ng-v2 .

builds/docker/linux/arm32-arm64/build-arm64.sh

@@ -1,7 +1,11 @@
 #!/bin/sh
+ARG_NATIVE_ARCH=linux/arm64
 docker buildx build \
+	--progress=plain \
+	--platform $ARG_NATIVE_ARCH \
 	--pull \
+	--build-arg ARG_NATIVE_ARCH=$ARG_NATIVE_ARCH \
 	--build-arg ARG_BASE=arm64v8/debian:bookworm \
 	--build-arg ARG_TARGET_ARCH=aarch64-pc-linux-gnu \
 	--build-arg ARG_CTNF_CONFIG=crosstool-ng-config-arm64 \
-	-t asfernandes/firebird-builder:fb6-arm64-ng-v1 .
+	-t firebirdsql/firebird-builder-linux:fb6-arm64-ng-v2 .

builds/docker/linux/arm32-arm64/push.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -e
 
-docker push asfernandes/firebird-builder:fb6-arm32-ng-v1
-docker push asfernandes/firebird-builder:fb6-arm64-ng-v1
+docker push firebirdsql/firebird-builder-linux:fb6-arm32-ng-v2
+docker push firebirdsql/firebird-builder-linux:fb6-arm64-ng-v2

builds/docker/linux/arm32-arm64/run-arm32.sh

@@ -1,2 +1,8 @@
 #!/bin/sh
-docker run --platform arm --rm --user `id -u`:`id -g` -v `pwd`/../../../..:/firebird -t asfernandes/firebird-builder:fb6-arm32-ng-v1
+mkdir -p `pwd`/../../../../gen
+docker run --platform arm --rm \
+	-e CTNG_UID=${CTNG_UID:-$(id -u)} \
+	-e CTNG_GID=${CTNG_GID:-$(id -g)} \
+	-v `pwd`/../../../..:/firebird \
+	-v `pwd`/../../../../gen:/home/ctng/firebird-build/gen \
+	-t firebirdsql/firebird-builder-linux:fb6-arm32-ng-v2

builds/docker/linux/arm32-arm64/run-arm64.sh

@@ -1,2 +1,8 @@
 #!/bin/sh
-docker run --platform arm64 --rm --user `id -u`:`id -g` -v `pwd`/../../../..:/firebird -t asfernandes/firebird-builder:fb6-arm64-ng-v1
+mkdir -p `pwd`/../../../../gen
+docker run --platform arm64 --rm \
+	-e CTNG_UID=${CTNG_UID:-$(id -u)} \
+	-e CTNG_GID=${CTNG_GID:-$(id -g)} \
+	-v `pwd`/../../../..:/firebird \
+	-v `pwd`/../../../../gen:/home/ctng/firebird-build/gen \
+	-t firebirdsql/firebird-builder-linux:fb6-arm64-ng-v2

builds/docker/linux/arm32-arm64/scripts/build.sh

@@ -5,7 +5,9 @@ gcc -v
 ld -v
 make -v
 
-./autogen.sh \
+cd /home/ctng/firebird-build
+
+/firebird/autogen.sh \
 	--host=$BUILD_ARCH \
 	--prefix=/opt/firebird \
 	--enable-binreloc \