Implement experimental support for non-containerized builds (#8)

Author: atrosinenko

Dockerfile.builder

@@ -22,6 +22,8 @@ RUN    apt-get update \
 
 ARG REPO_ROOT
 
+ENV TOOLCHAIN_BUILD_IN_DOCKER=1
+
 COPY cmake   /${REPO_ROOT}/cmake
 COPY scripts /${REPO_ROOT}/scripts
 COPY config  /${REPO_ROOT}

README.md

@@ -7,6 +7,17 @@ The resulting toolchain is written to `./output/llvm-pauth.squashfs` - it is a
 compressed read-only file system image which is intended to be `mount`ed to
 `/opt/llvm-pauth`.
 
+Another option is building the toolchain without containers, purely on the host,
+but keep in mind that Clang is able to auto-discover system-provided sysroots
+for cross-compilation (for example, the sysroot under `/usr/aarch64-linux-gnu`
+which is installed as a dependency of the `gcc-aarch64-linux-gnu` package in
+Ubuntu running on x86_64 host). This is hopefully not an issue for these scripts,
+as they explicitly specify the sysroots in Clang `*.cfg` files, and everything
+compiles successfully in x86_64 containerized build. Anyway, containerized
+builds (especially, when performed on a non-AArch64 host) look like yet another
+layer of protection against unintentionally linking to any system-provided
+libraries.
+
 The versions of LLVM and Musl as well as a few other tunables are set in `config`:
 by default, mainline `llvmorg-21.1.0-rc1` tag is used together with a patched
 version of Musl that can be obtained at https://github.com/access-softek/musl.
@@ -25,25 +36,25 @@ not checked whether the working copy is clean or not.
 Ensure `llvm-project` and `musl` repositories are cloned on the host and contain
 the commits specified in the `./config` file (by default, you need the mainline
 LLVM monorepo and patched Musl version from https://github.com/access-softek/musl).
-Alternatively, you can pass https:// or git:// URLs directly to `./docker.sh sources`.
+Alternatively, you can pass https:// or git:// URLs directly to `./build.sh sources`.
 
 Checkout the particular commits of LLVM and Musl sources under `./src` and
 download Linux kernel tarball by running:
 
 ```
-./docker.sh sources <llvm_repo_url> <musl_repo_url>
+./build.sh sources <llvm_repo_url> <musl_repo_url>
 ```
 
 if LLVM and Musl are already cloned on the host, use
 
 ```
-./docker.sh sources file:///absolute/path/to/llvm-project file:///absolute/path/to/musl
+./build.sh sources file:///absolute/path/to/llvm-project file:///absolute/path/to/musl
 ```
 
 Then build the toolchain by running
 
 ```
-./docker.sh build
+./build.sh build
 ```
 
 The build artifact is `./output/llvm-pauth.squashfs` file.
@@ -118,3 +129,44 @@ $ gdb-multiarch
 >>> b main
 >>> c
 ```
+
+# Building the toolchain without Docker
+
+The preferred way to use these scripts is building inside a container, though
+it is possible to build the toolchain purely on the host.
+
+This can be achieved by running `./build.sh host-build` instead of
+`./build.sh build` after checking out the sources to `./src/*` the same way as
+for a containerized build. After a successful build, the toolchain is installed
+to the `./inst` subdirectory inside this repository (can be customized in
+`./config`) and no archive is created under `./output`.
+
+When the toolchain is being built inside a container, a temporary volume is
+attached as a build directory, which is discarded when the container is stopped,
+thus the subsequent invocation of `./build.sh build` uses a fresh build directory
+automatically. When building on the host, on the other hand, the user is
+responsible for removing the half-built subdirectories of the main build
+directory after investigating the errors.
+
+Inside the main build directory (which is `./build` by default), each build step
+corresponds to `{build step}-{target triple}` subdirectory (it depends on the
+particular build step, whether this subdirectory is created or not) and a
+corresponding stamp file with `.stamp` suffix (which is always created **after**
+the build step finishes successfully):
+* the step is skipped if corresponding stamp file exists (whether the directory
+  exists or not)
+* an error is reported if a half-built subdirectory exists without a stamp file
+  indicating a successful build
+* otherwise, a build step is performed and the stamp file is `touch`ed on
+  success
+
+Please note that each step is performed without taking the dependencies into
+account, thus the user is responsible for removing the subdirectories
+corresponding to the steps that have to be redone after some other steps (such
+as rebuilding everything linking to `crt1.o` after the start files were
+rebuilt). It is generally possible to simply remove the entire `./build` and
+`./inst` subdirectories, since LLVM should be rebuilt rather quickly thanks to
+ccache - the main reason for not removing `./build` automatically, aside from
+simplifying the debugging in case of errors, is not to run `rm -rf` with
+computed paths, as this can be harmful to the host system in case of
+misconfiguration.

docker.sh build.shdocker.sh renamed to build.sh

@@ -4,12 +4,6 @@ ROOT="$(dirname "$0")"
 ROOT="$(realpath "$ROOT")"
 cd "$ROOT"
 
-# Path inside the container.
-REPO_ROOT=/repo
-
-. ./config
-. ./scripts/global-vars
-
 check_repo_sha() {
   local repo_path="$1"
   local expected_sha="$2"
@@ -27,6 +21,9 @@ check_repo_sha() {
 }
 
 fetch_sources() {
+  . ./config
+  . ./scripts/global-vars
+
   if [ "$#" != 2 ]; then
     echo "Usage: docker.sh sources <llvm_repo_url> <musl_repo_url>"
     echo "Note that file:///path/to/repo/ URLs can be used."
@@ -48,7 +45,12 @@ fetch_sources() {
   check_repo_sha "$ROOT/src/musl" "$MUSL_SHA"
 }
 
-build_toolchain() {
+build_in_docker() {
+  # Path inside the container.
+  REPO_ROOT=/repo
+  . ./config
+  . ./scripts/global-vars
+
   check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA"
   check_repo_sha "$ROOT/src/musl" "$MUSL_SHA"
 
@@ -66,6 +68,17 @@ build_toolchain() {
       "$DOCKER_IMAGE_NAME" "$REPO_ROOT/scripts/build-in-docker.sh"
 }
 
+build_on_host() {
+  REPO_ROOT="$ROOT"
+  . ./config
+  . ./scripts/global-vars
+
+  check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA"
+  check_repo_sha "$ROOT/src/musl" "$MUSL_SHA"
+
+  ./scripts/build-on-host.sh
+}
+
 main() {
   local subcmd="$1"
 
@@ -75,11 +88,14 @@ main() {
     fetch_sources "$@"
   ;;
   build)
-    build_toolchain
+    build_in_docker
+  ;;
+  host-build)
+    build_on_host
   ;;
   *)
     echo "Unknown subcommand: $subcmd"
-    echo "Expected: 'sources', 'build'."
+    echo "Expected one of 'sources', 'build', or 'host-build' (experimental)."
     exit 1
   ;;
   esac

config

@@ -19,9 +19,14 @@ EXTRA_FLAGS_MUSL=""
 BUILD_OPTIMIZED_RUNTIMES=yes
 
 # Toolchain installation prefix.
+
 # This path is used inside the container as a temporary location to install
 # the toolchain before packing it into a SquashFS image.
-INSTALL_DIR=/opt/llvm-pauth
+DOCKER_BUILD_INSTALL_DIR=/opt/llvm-pauth
+
+# This path is used as a final installation prefix when building the toolchain
+# on the host.
+HOST_BUILD_INSTALL_DIR="$REPO_ROOT/inst"
 
 # Command to build and run containers on the host: docker, podman, etc.
 DOCKER_CMD=docker

scripts/build-all.sh

@@ -2,8 +2,10 @@
 set -e
 
 # This script invokes all other build-*.sh scripts.
-# Inside the container, it is called by build-in-docker.sh.
-# FIXME: Add another entry point script to perform build directly on host.
+# It is called either by build-in-docker.sh or by build-on-host.sh, depending
+# on whether the build is performed inside a container.
+# The calling script is responsible for `export`ing REPO_ROOT environment
+# variable (as explained in ./global-vars).
 
 set -x
 cd "$(dirname "$0")"

scripts/build-in-docker.sh

@@ -5,13 +5,16 @@ cd "$(dirname "$0")"
 # This script is an entry point inside the Docker container.
 # Its location is expected to be $REPO_ROOT/scripts/build-in-docker.sh.
 
-# Export the REPO_ROOT variable, so it can be used by the 'config' script
-# sourced by this script and its subprocesses.
+# Export the REPO_ROOT variable, so it can be used by the 'global-vars' script
+# sourced by this script, as well as its subprocesses.
 export REPO_ROOT="$(pwd)/.."
 . ../config
 . ./global-vars
 
-./build-all.sh
+if ! ./build-all.sh; then
+  echo "Containerized build failed - $BUILD_TMP is discarded automatically."
+  exit 1
+fi
 
 # It is important to remove an old image, if any, as otherwise it would be
 # appended to instead of overwritten!

scripts/build-on-host.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+set -e
+cd "$(dirname "$0")"
+
+# This script starts the build on the host, similar to build-in-docker.sh for a
+# containeraized build.
+# Its location is expected to be $REPO_ROOT/scripts/build-on-host.sh.
+
+# Export the REPO_ROOT variable, so it can be used by the 'global-vars' script
+# sourced by this script, as well as its subprocesses.
+export REPO_ROOT="$(pwd)/.."
+. ../config
+. ./global-vars
+
+./build-all.sh
+
+echo "Build finished, the toolchain is installed to $INSTALL_DIR."

scripts/global-vars

@@ -13,22 +13,27 @@
 # NB: Make sure the path is absolute, as relative paths may be interpreted in
 # surprising ways in some contexts, such as in the argument of `--toolchain`
 # option of CMake.
-if [ "x$REPO_ROOT" = "x" ]; then
-  echo "error: REPO_ROOT variable should be set before sourcing 'config'." 1>&2
-  exit 1
-fi
+if [ "x$REPO_ROOT" != "x" ]; then
+  # Directories mounted from the host to the container in Docker-based build.
+  # When Docker is used, these path should be as seen from inside the container,
+  # otherwise they are just absolute paths on the host pointing inside this repo.
+  OUTPUT_DIR="$REPO_ROOT/output"
+  CCACHE_DIR="$REPO_ROOT/ccache"
+  BUILD_TMP="$REPO_ROOT/build"
 
-# Directories mounted from the host to the container
-# (as seen from inside the container).
-OUTPUT_DIR="$REPO_ROOT/output"
-CCACHE_DIR="$REPO_ROOT/ccache"
-BUILD_TMP="$REPO_ROOT/build"
+  SRC_DIR="$REPO_ROOT/src"
+  LLVM_SOURCE_DIR="$SRC_DIR/llvm"
+  MUSL_SOURCE_DIR="$SRC_DIR/musl"
 
-SRC_DIR="$REPO_ROOT/src"
-LLVM_SOURCE_DIR="$SRC_DIR/llvm"
-MUSL_SOURCE_DIR="$SRC_DIR/musl"
+  CMAKE_DIR="$REPO_ROOT/cmake"
+fi
 
-CMAKE_DIR="$REPO_ROOT/cmake"
+# TOOLCHAIN_BUILD_IN_DOCKER is defined in Dockerfile.builder
+if [ -n "$TOOLCHAIN_BUILD_IN_DOCKER" ]; then
+  INSTALL_DIR="$DOCKER_BUILD_INSTALL_DIR"
+else
+  INSTALL_DIR="$HOST_BUILD_INSTALL_DIR"
+fi
 
 # Linux kernel version to be used to provide user-space headers to libc.
 # Any recent version should work, so this variable is defined here instead