[Build] Use CMAKE_ARGS directly, drop legacy QUADRANTS_CMAKE_ARGS (#753)

Author: hughperkins

.github/workflows/scripts/qd_build/cmake.py

@@ -30,6 +30,8 @@ def __init__(self, environ_name):
         self.option_definitions = {
             "CMAKE_EXPORT_COMPILE_COMMANDS": ("Generate compile_commands.json", False, ""),
         }
+        # CMAKE_ARGS entries we don't parse into definitions, kept verbatim for writeback().
+        self.passthrough = ""
 
         self.finalized = False
 
@@ -42,6 +44,12 @@ def collect_options(self, *files: str) -> None:
 
     def parse_initial_args(self) -> None:
         args = os.environ.get(self.environ_name, "")
+        # DEF_RE only understands `-D<ALLCAPS>[:BOOL]=<value>` entries (the options we manage and
+        # render below). Everything else in CMAKE_ARGS -- CMake generators (`-GNinja`), typed cache
+        # entries (`-DCMAKE_BUILD_TYPE:STRING=Debug`), lowercase-named or space-containing defines --
+        # must be preserved verbatim, else writeback() would silently drop it before scikit-build-core
+        # sees CMAKE_ARGS. Stash the unparsed remainder (matched entries blanked out) and re-emit it.
+        self.passthrough = " ".join(DEF_RE.sub(" ", args).split())
         for name, value in DEF_RE.findall(args):
             self.set(name, value)
 
@@ -129,13 +137,17 @@ def print_summary(self, rendered) -> None:
     def writeback(self) -> None:
         rendered = self.render()
         self.print_summary(rendered)
-        value = " ".join([v for _, v, _ in rendered])
+        # Rendered options first, then the verbatim passthrough captured in parse_initial_args. The
+        # two sets are disjoint (parsed entries are blanked out of passthrough), so there is no
+        # duplicate-cache-var ambiguity for CMake to resolve.
+        parts = [v for _, v, _ in rendered]
+        if self.passthrough:
+            parts.append(self.passthrough)
+        value = " ".join(parts)
+        # CMAKE_ARGS is scikit-build-core's standard CMake-args passthrough, and it is also what we parse on input, so
+        # writing it back makes the environment exported by `build.py wheel`, `-w`, and `--shell` directly usable by the
+        # build with no further bridging.
         os.environ[self.environ_name] = value
-        # scikit-build-core reads CMake args from CMAKE_ARGS, not the legacy QUADRANTS_CMAKE_ARGS.
-        # Mirror the rendered args there so the environment exported by `build.py wheel`, `-w`, and
-        # `--shell` is directly usable -- no manual `export CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS"` step.
-        if self.environ_name == "QUADRANTS_CMAKE_ARGS":
-            os.environ["CMAKE_ARGS"] = value
         self.finalized = True
 
     def __setitem__(self, name: str, value: Union[str, bool]) -> None:
@@ -145,10 +157,10 @@ def __getitem__(self, name: str) -> Union[str, bool]:
         return self.definitions[name]
 
 
-cmake_args = CMakeArgsManager("QUADRANTS_CMAKE_ARGS")
+cmake_args = CMakeArgsManager("CMAKE_ARGS")
 
 
-@banner("Parsing QUADRANTS_CMAKE_ARGS")
+@banner("Parsing CMAKE_ARGS")
 def _init_cmake_args():
     cmake_args.collect_options("CMakeLists.txt", *glob.glob("cmake/*.cmake"))
     cmake_args.parse_initial_args()

.github/workflows/scripts/qd_build/entry.py

@@ -23,7 +23,7 @@
 # -- code --
 @banner("Build Quadrants Wheel")
 def build_wheel(python: Command) -> None:
-    # cmake_args.writeback() populates both QUADRANTS_CMAKE_ARGS and (for scikit-build-core) CMAKE_ARGS.
+    # cmake_args.writeback() renders the effective options into CMAKE_ARGS for scikit-build-core.
     cmake_args.writeback()
 
     plat = None

.github/workflows/scripts_new/clang_tidy/2_build.sh

@@ -2,5 +2,5 @@
 
 set -ex
 
-export QUADRANTS_CMAKE_ARGS="-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
+export CMAKE_ARGS="-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
 ./build.py wheel

.github/workflows/scripts_new/linux/2_build.sh

@@ -8,5 +8,5 @@ export PATH=${LLVM_DIR}/bin:$PATH
 which clang
 clang --version
 
-export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
+export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
 ./build.py wheel

.github/workflows/scripts_new/macosx/2_build.sh

@@ -2,5 +2,5 @@
 
 set -ex
 
-export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_METAL:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
+export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_METAL:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
 ./build.py wheel

.github/workflows/scripts_new/manylinux_wheel/2_build.sh

@@ -19,7 +19,7 @@ echo "Detected platform: $PLATFORM"
 export PATH="$PWD/taichi-llvm-15.0.7-linux-${PLATFORM}/bin:$PATH"
 
 # Taichi build options
-export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
+export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
 
 # GCC toolset include paths
 inc_base="/opt/rh/gcc-toolset-14/root/usr/include/c++/14"

.github/workflows/scripts_new/win/2_build.ps1

@@ -2,5 +2,5 @@ $ErrorActionPreference = "Stop"
 Set-PSDebug -Trace 1
 trap { Write-Error $_; exit 1 }
 
-$env:GSTAICHI_CMAKE_ARGS = "-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
+$env:CMAKE_ARGS = "-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON"
 python build.py

docs/source/user_guide/contributing.md

@@ -1,4 +1,4 @@
-# Contributing to quadrants
+# Advanced: Contributing to quadrants
 
 ## Good practice reminder
 
@@ -49,15 +49,15 @@ uv pip install --group dev --group test
 
 `build.py` is a python script to automatically set up the build environment for you before invoking the build commands:
 
-* `LLVM libraries`: downloads an archive for `LLVM` libraries, decompresses it and sets `LLVM_DIR`.
+* `LLVM libraries`: downloads an archive for [LLVM](https://llvm.org/) libraries (a library for building compilers), decompresses it and sets `LLVM_DIR`.
 * `clang`: depending on the platform, download `clang` or just check if available with the right version.
 
 `build.py` can be used at least two ways:
 
 * `build.py wheel` to build the wheel (via [scikit-build-core](https://scikit-build-core.readthedocs.io/en/latest/), i.e. `pip wheel`)
 * `build.py --shell` to enter a shell with environment variables set up as with `build.py wheel` in order to let you invoke yourself the commands.
 
-For incremental development, do an editable install (scikit-build-core "redirect" mode: the compiled core is installed and rebuilt on demand, Python edits are live):
+For incremental development, do an editable install ([scikit-build-core](https://scikit-build-core.readthedocs.io/en/latest/) "redirect" mode: the compiled core is installed and rebuilt on demand, Python edits are live):
 
 ```
 ./build.py --shell # run a new shell with environment variables
@@ -72,7 +72,7 @@ source env.sh
 pip install --no-build-isolation -e . -Ceditable.rebuild=true
 ```
 
-`build.py` exports both the legacy `QUADRANTS_CMAKE_ARGS` and the `CMAKE_ARGS` that scikit-build-core actually reads, so sourcing `env.sh` (or using `--shell`) is enough -- no manual `export CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS"` step is needed.
+`build.py` reads and exports `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough), so sourcing `env.sh` (or using `--shell`) is enough to make the configured options available to the build.
 
 ## Building the package for release purposes
 
@@ -84,13 +84,13 @@ To build the release package:
 
 We use `cmake` to build the C++ core. scikit-build-core puts the CMake build tree under `build/{wheel_tag}`, where the wheel tag encodes the Python version and host platform. For example: `build/cp310-cp310-linux_x86_64`.
 
-You can modify the cmake options to your liking in order to enable or disable some features you need or don't need. To discover them, you can use `ccmake`:
+You can modify the cmake options to your liking in order to enable or disable some features you need or don't need. To discover them, you can use [ccmake](https://cmake.org/cmake/help/latest/manual/ccmake.1.html):
 
 ```
 ccmake build/cp310-cp310-linux_x86_64
 ```
 
-You could then set the environment variable `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough) to configure the build. `build.py` also accepts the legacy `QUADRANTS_CMAKE_ARGS` and forwards it to `CMAKE_ARGS`. For instance, to disable the CUDA and AMDGPU backends:
+You could then set the environment variable `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough) to configure the build. `build.py` reads it, layers on the toolchain options it manages, and exports it back. For instance, to disable the CUDA and AMDGPU backends:
 
 ```
 export CMAKE_ARGS="-DQD_WITH_CUDA=OFF -DQD_WITH_AMDGPU=OFF"
@@ -102,7 +102,7 @@ To direct `cmake` where to look at for some dependencies, for example `LLVM`, yo
 # using an env var
 export LLVM_DIR="/path/to/llvm/"
 # or with a cmake option
-export QUADRANTS_CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS -DLLVM_ROOT=/path/to/llvm"
+export CMAKE_ARGS="$CMAKE_ARGS -DLLVM_ROOT=/path/to/llvm"
 ```
 
 ### Building with the AMD GPU backend (Linux)
@@ -114,22 +114,6 @@ The AMD GPU backend is Linux-only (it is force-disabled on macOS and Windows) an
 CMAKE_ARGS="-DQD_WITH_AMDGPU=ON -DQD_WITH_CUDA=OFF" pip install --no-build-isolation -e . -v
 ```
 
-## Advanced usage
-
-### CI Convention about compilers/LLVM
-
-Quadrants comprises at least three important parts:
-
-1. `quadrants` host runtime: Made with a mix of Python and C++. The C++ core is compiled using the OS default C/C++ compiler.
-2. `quadrants` device runtime (bitcode): C++ code compiled using `clang++` from the distribution/OS. Using `clang++` is required as it has to support the same targets as `LLVM`.
-3. `LLVM` libraries used by host runtime: statically or dynamically linked, used to lower the kernel's final IR to machine code on the host. The CI uses an LLVM version compiled from source.
-
-### Building LLVM for debugging it
-
-Sometimes, it could be useful to have a `LLVM` version that allows to print intermediate passes or with debug symbols to find out where and why LLVM fails (for example, when Instruction Selection fails). To do so you would have to build LLVM by yourself. If so, you should take some inspiration from our [CI pipeline to build LLVM](https://github.com/Genesis-Embodied-AI/quadrants-sdk-builds/blob/main/.github/workflows/llvm-ci.yml) to tweak a little bit to your liking (and not enable/disable options that would create discrepancies).
-
-You can then use `LLVM_DIR` to point to the `LLVM` build directory.
-
 ## CI checks
 
 Pull requests are validated by several CI jobs. Most run automatically; a failing check blocks merge.
@@ -227,3 +211,19 @@ quadrants/program/legacy_stream.cpp 42 -42
 The `0` in the LoC column for the two new files reflects that both files did not exist before this PR (their pre-PR code-line count is 0). The `42 -42` row for `legacy_stream.cpp` is a fully-deleted file: 42 code lines existed before this PR and all 42 were removed.
 
 This check is delayed by 30 minutes, to avoid running repeatedly if multiple commits pushed with a short delay between each.
+
+## Advanced
+
+### CI Convention about compilers/LLVM
+
+Quadrants comprises at least three important parts:
+
+1. `quadrants` host runtime: Made with a mix of Python and C++. The C++ core is compiled using the OS default C/C++ compiler.
+2. `quadrants` device runtime (bitcode): C++ code compiled using `clang++` from the distribution/OS. Using `clang++` is required as it has to support the same targets as `LLVM`.
+3. `LLVM` libraries used by host runtime: statically or dynamically linked, used to lower the kernel's final IR to machine code on the host. The CI uses an LLVM version compiled from source.
+
+### Building LLVM for debugging it
+
+Sometimes, it could be useful to have a `LLVM` version that allows to print intermediate passes or with debug symbols to find out where and why LLVM fails (for example, when Instruction Selection fails). To do so you would have to build LLVM by yourself. If so, you should take some inspiration from our [CI pipeline to build LLVM](https://github.com/Genesis-Embodied-AI/quadrants-sdk-builds/blob/main/.github/workflows/llvm-ci.yml) to tweak a little bit to your liking (and not enable/disable options that would create discrepancies).
+
+You can then use `LLVM_DIR` to point to the `LLVM` build directory.