Simplify release scripts and update release instructions

release/README.md

@@ -1,6 +1,6 @@
 # Tools for building releases of TensorFlow Quantum
 
-This directory contains configurations and scripts that the TensorFlow Quantum
+This directory contains scripts that the TensorFlow Quantum (TFQ)
 maintainers use to create Python packages for software releases. The process of
 making a TFQ release is complex and has not been fully automated. The scripts
 in this directory help automate some steps and are a way of capturing the
@@ -46,11 +46,33 @@ libraries.
 
 ### Preliminary steps
 
-1.  Make sure you have `pyenv`, `pip`, and `jq` installed on your system.
+1.  Make sure you have `docker`, `pyenv`, `pip`, and `jq` installed on your
+    system.
 
 2.  Git clone the TensorFlow Quantum repo to a directory on your computer.
 
-3.  `cd` into your local clone directory in a Bash shell.
+3.  `cd` into this directory in a Bash shell.
+
+### Rebuild `requirements.txt`
+
+1.  Create a Python virtual environment using the lowest version of Python
+    supported by TFQ. (Currently this is Python 3.10.)
+
+2.  Run the following commands:
+
+    ```shell
+    pip install pip-tools
+    ./scripts/generate_requirements.sh
+    ```
+
+    This will update the dependency versions in the file `requirements.txt` to
+    the latest versions based on `requirements.in`. If this process fails, you
+    may have to iterate on adjusting the contraints in `requirements.in`
+    followed by running `generate_requirements.sh` again until it succeeds.
+
+3.  If any changes were made to `requirements.in`, check `release/setup.py` and
+    make corresponding changes, if the changes to `requirements.in` involved
+    packages needed to run TFQ.
 
 ### Build the release
 

release/build_distribution.sh

@@ -14,11 +14,11 @@
 # limitations under the License.
 # =============================================================================
 
-# Summary: build a wheel for TFQ using a TensorFlow SIG Build container.
+# Summary: build a wheel for TFQ using a TensorFlow build container.
 # Run this script with the option "-h" to get a usage summary.
 #
 # To ensure binary compatibility with TensorFlow, TFQ distributions are built
-# using TensorFlow's SIG Build containers and Crosstool C++ toolchain. This
+# using TensorFlow's build containers and crosstool C++ toolchain. This
 # script encapsulates the process. The basic steps this script performs are:
 #
 # 1. Write to a file a small shell script that does the following:
@@ -32,9 +32,7 @@
 # 2. Start Docker with image tensorflow/build:${tf_version}-python${py_version}
 #    and run the script written in step 1.
 #
-# 3. Do some basic tests on the wheel using standard Python utilities.
-#
-# 4. Exit.
+# 3. Exit.
 
 set -eu -o pipefail
 
@@ -49,75 +47,79 @@ repo_dir=$(git -C "${thisdir}" rev-parse --show-toplevel 2> /dev/null) || \
   quit "This script must be run from inside the TFQ git tree."
 
 # Default values for variables that can be changed via command line flags.
-tf_version="2.16"
 py_version=$(python3 --version | cut -d' ' -f2 | cut -d. -f1,2)
-cuda_version="12"
+extra_bazel_options=""
 cleanup="true"
 
 usage="Usage: ${0} [OPTIONS]
-Build a distribution wheel for TensorFlow Quantum.
-
-Configuration options:
-  -c X.Y  Use CUDA version X.Y (default: ${cuda_version})
-  -p X.Y  Use Python version X.Y (default: ${py_version})
-  -t X.Y  Use TensorFlow version X.Y (default: ${tf_version})
-
-General options:
-  -e     Don't run bazel clean at the end (default: do)
-  -n     Dry run: print commands but don't execute them
-  -h     Show this help message and exit"
-
-dry_run="false"
-while getopts "c:ehnp:t:" opt; do
+Build a Python wheel for a distribution of TensorFlow Quantum.
+Options:
+  -b "options"  Additional options to pass to Bazel build
+  -p X.Y        Use Python version X.Y (default: ${py_version})
+  -e            Don't run bazel clean at the end (default: do)
+  -h            Show this help message and exit"
+
+while getopts "b:ehp:" opt; do
   case "${opt}" in
-    c) cuda_version="${OPTARG}" ;;
+    b) extra_bazel_options="${OPTARG}" ;;
     e) cleanup="false" ;;
     h) echo "${usage}"; exit 0 ;;
-    n) dry_run="true" ;;
     p) py_version=$(echo "${OPTARG}" | cut -d. -f1,2) ;;
-    t) tf_version="${OPTARG}" ;;
     *) quit "${usage}" ;;
   esac
 done
 shift $((OPTIND -1))
 
 # See https://hub.docker.com/r/tensorflow/build/tags for available containers.
-docker_image="tensorflow/build:${tf_version}-python${py_version}"
+docker_image="tensorflow/build:2.18-python${py_version}"
 
-# This should match what TensorFlow's .bazelrc file uses.
-crosstool="@sigbuild-r${tf_version}-clang_config_cuda//crosstool:toolchain"
-
-# Note: configure.sh is run inside the container, and it creates a .bazelrc
-# file that adds other cxxopt flags. They don't need to be repeated here.
-BUILD_OPTIONS="--cxxopt=-O3 --cxxopt=-msse2 --cxxopt=-msse3 --cxxopt=-msse4"
+# The next values must match what is used by the TF release being targeted. Look
+# in the TF .bazelrc file for the Linux CPU builds, specifically rbe_linux_cpu.
+tf_compiler="/usr/lib/llvm-18/bin/clang"
+tf_sysroot="/dt9"
+tf_crosstool="@local_config_cuda//crosstool:toolchain"
 
 # Create a script to be run by the shell inside the Docker container.
 build_script=$(mktemp /tmp/tfq_build.XXXXXX)
 trap 'rm -f "${build_script}" || true' EXIT
 
 # The printf'ed section dividers are to make it easier to search the output.
-cat <<'EOF' > "${build_script}"
+cat <<EOF > "${build_script}"
 #!/bin/bash
 set -o errexit
 cd /tfq
-PREFIX='[DOCKER] '
-exec > >(sed "s/^/${PREFIX} /")
-exec 2> >(sed "s/^/${PREFIX} /" >&2)
-printf ":::::::: Build configuration inside Docker container ::::::::\n"
-printf "  Docker image:     ${docker_image}\n"
-printf "  Crosstool:        ${crosstool}\n"
-printf "  TF version:       ${tf_version}\n"
-printf "  Python version:   ${py_version}\n"
-printf "  CUDA version:     ${cuda_version}\n"
-printf "  vCPUs available:  $(nproc)\n"
-printf "\n\n:::::::: Configuring Python environment ::::::::\n\n"
+exec > >(sed "s/^/[DOCKER] /")
+exec 2> >(sed "s/^/[DOCKER] /" >&2)
+
+printf ":::::::: Configuring Python environment ::::::::\n\n"
 python3 -m pip install --upgrade pip --root-user-action ignore
-pip install -r requirements.txt --root-user-action ignore
+python3 -m pip install -r requirements.txt --root-user-action ignore
+
+printf "\n\n:::::::: Configuring TFQ build ::::::::\n\n"
 printf "Y\n" | ./configure.sh
+
+printf "\n\n:::::::: Build configuration inside Docker container ::::::::\n"
+printf "  Docker image:     ${docker_image}\n"
+printf "  Crosstool:        ${tf_crosstool}\n"
+printf "  Compiler:         ${tf_compiler}\n"
+printf "  TF_SYSROOT:       ${tf_sysroot}\n"
+printf "  Python version:   "
+python3 --version | cut -d' ' -f2
+
 printf "\n:::::::: Starting Bazel build ::::::::\n\n"
-bazel build ${build_flags} release:build_pip_package
+bazel build \
+  --cxxopt=-O3 --cxxopt=-msse2 --cxxopt=-msse3 --cxxopt=-msse4 \
+  ${extra_bazel_options} \
+  --crosstool_top="${tf_crosstool}" \
+  --host_crosstool_top="${tf_crosstool}" \
+  --extra_toolchains="${tf_crosstool}-linux-x86_64" \
+  --repo_env=CC="${tf_compiler}" \
+  --repo_env=TF_SYSROOT="${tf_sysroot}" \
+  release:build_pip_package
+
 printf "\n:::::::: Creating Python wheel ::::::::\n\n"
 bazel-bin/release/build_pip_package /build_output/
+
 if [[ "${cleanup}" == "true" ]]; then
   printf "\n:::::::: Cleaning up ::::::::\n\n"
   bazel clean --async
@@ -126,31 +128,15 @@ EOF
 
 chmod +x "${build_script}"
 
-# Use 'set --' to build the command in the positional parameters ($1, $2, ...)
-set -- docker run -it --rm --network host \
+echo "Spinning up a Docker container with ${docker_image} …"
+docker run -it --rm --network host \
   -w /tfq \
   -v "${repo_dir}":/tfq \
   -v /tmp/tensorflow_quantum:/build_output \
   -v "${build_script}:/tmp/build_script.sh" \
   -e HOST_PERMS="$(id -u):$(id -g)" \
-  -e build_flags="--crosstool_top=${crosstool} ${BUILD_OPTIONS}" \
-  -e cuda_version="${cuda_version}" \
-  -e py_version="${py_version}" \
-  -e tf_version="${tf_version}" \
-  -e docker_image="${docker_image}" \
-  -e crosstool="${crosstool}" \
-  -e cleanup="${cleanup}" \
   "${docker_image}" \
   /tmp/build_script.sh
 
-if [[ "${dry_run}" == "true" ]]; then
-  # Loop through the positional parameters and simply print them.
-  printf "(Dry run) "
-  printf '%s ' "$@"
-else
-  echo "Spinning up a Docker container with ${docker_image} …"
-  "$@"
-
-  echo "Done. Look for wheel in /tmp/tensorflow_quantum/."
-  ls -l /tmp/tensorflow_quantum/
-fi
+echo "Done. Look for wheel in /tmp/tensorflow_quantum/."
+ls -l /tmp/tensorflow_quantum/

release/clean_distribution.sh

@@ -39,7 +39,6 @@ docker_image="quay.io/pypa/manylinux_2_34_x86_64"
 platform="manylinux_2_17_x86_64"
 py_version=$(python3 --version | cut -d' ' -f2 | cut -d. -f1,2)
 action="repair"
-verbose=""
 
 usage="Usage: ${0} [OPTIONS] /path/to/wheel.whl
 Run auditwheel on the given wheel file. Available options:
@@ -51,20 +50,15 @@ Configuration options:
 
 General options:
   -h  Show this help message and exit
-  -n  Dry run: print commands but don't execute them
-  -s  Run 'auditwheel show', not repair (default: run 'auditwheel repair')
-  -v  Produce verbose output"
+  -s  Run 'auditwheel show', not repair (default: run 'auditwheel repair')"
 
-dry_run="false"
-while getopts "hm:np:st:v" opt; do
+while getopts "hm:p:st:" opt; do
   case "${opt}" in
     h) echo "${usage}"; exit 0 ;;
     m) docker_image="${OPTARG}" ;;
-    n) dry_run="true" ;;
     p) py_version="${OPTARG}" ;;
     s) action="show" ;;
     t) platform="${OPTARG}" ;;
-    v) verbose="--verbose" ;;
     *) quit "${usage}" ;;
   esac
 done
@@ -76,28 +70,21 @@ fi
 wheel_path="$(cd "$(dirname "${1}")" && pwd)/$(basename "${1}")"
 wheel_name="$(basename "${1}")"
 
-args=""
+auditwheel_args=()
 if [[ "${action}" == "repair" ]]; then
-  args="${verbose} --exclude libtensorflow_framework.so.2 --plat ${platform}"
+    auditwheel_args+=(
+        "--exclude" "libtensorflow_framework.so.2"
+        "--plat" "${platform}"
+        "-w" "/tfq/wheelhouse"
+    )
 fi
 
-# Use 'set --' to build the command in the positional parameters ($1, $2, ...)
-set -- docker run -it --rm --network host \
+echo "Running 'auditwheel ${action}' in Docker with image ${docker_image}"
+docker run -it --rm --network host \
   -w /tfq \
   -v "${repo_dir}":/tfq \
   -v "${wheel_path}":"/tmp/${wheel_name}" \
   "${docker_image}" \
-  bash -c "auditwheel ${action} ${args} -w /tfq/wheelhouse /tmp/${wheel_name}"
+  auditwheel "${action}" "${auditwheel_args[@]}" "/tmp/${wheel_name}"
 
-if [[ "${dry_run}" == "true" ]]; then
-  # Loop through the positional parameters and simply print them.
-  printf "(Dry run) "
-  printf '%s ' "$@"
-  echo
-else
-  echo "Running 'auditwheel ${action}' in Docker with image ${docker_image}"
-  "$@"
-  if [[ "${action}" == "repair" ]]; then
-    echo "Done. New wheel file written to ${repo_dir}/wheelhouse"
-  fi
-fi
+echo "Done. New wheel file written to ${repo_dir}/wheelhouse"