ci: update / clean up wasm build (#6166)

* ci: update / clean up wasm build

* drop `--quiet`

* fix CC variable conflict

* switch wasm jobs to uv

* debug

* fix `test-wasm` path

Author: davidhewitt

.github/workflows/ci.yml

@@ -483,10 +483,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7.0.0
-      - uses: actions/setup-python@v6
+      - uses: astral-sh/setup-uv@v7
         with:
-          python-version: 3.14
-        id: setup-python
+          save-cache: ${{ needs.resolve.outputs.save-cache }}
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -495,42 +494,37 @@ jobs:
       - uses: actions/setup-node@v6
         with:
           node-version: 24
-      - run: python -m pip install --upgrade pip && pip install nox[uv]
       - uses: actions/cache/restore@v5
         id: cache
         with:
           path: |
             .nox/emscripten
-          key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }}
+          key: emscripten-${{ hashFiles('wasm/emscripten/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }}
       - uses: Swatinem/rust-cache@v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
       - name: Build
         if: steps.cache.outputs.cache-hit != 'true'
-        run: nox -s build-emscripten
+        run: uvx nox -s build-emscripten
       - name: Test
-        run: nox -s test-emscripten
+        run: uvx nox -s test-emscripten
       - uses: actions/cache/save@v5
         if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
         with:
           path: |
             .nox/emscripten
-          key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }}
+          key: emscripten-${{ hashFiles('wasm/emscripten/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }}
 
   wasm32-wasip1:
     name: wasm32-wasip1
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     needs: [fmt]
     runs-on: ubuntu-latest
-    env:
-      WASI_SDK_PATH: "/opt/wasi-sdk"
-      CPYTHON_PATH: "${{ github.workspace }}/wasi/cpython"
     steps:
       - uses: actions/checkout@v7.0.0
-      - uses: actions/setup-python@v6
+      - uses: astral-sh/setup-uv@v7
         with:
-          python-version: 3.14
-        id: setup-python
+          save-cache: ${{ needs.resolve.outputs.save-cache }}
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -539,59 +533,32 @@ jobs:
       - name: "Install wasmtime"
         uses: bytecodealliance/actions/wasmtime/setup@v1
       - name: "Install WASI SDK"
-        run: |
-          mkdir ${{ env.WASI_SDK_PATH }} && \
-          curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz | \
-          tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip
-          $WASI_SDK_PATH/bin/clang --version
-      - uses: actions/cache/restore@v5
-        id: cache-wasip1-python
+        uses: bytecodealliance/setup-wasi-sdk-action@main
         with:
-          path: ${{ env.CPYTHON_PATH }}/cross-build/
-          key: wasm32-wasip1-python
-      - uses: actions/checkout@v7.0.0
+          version: "24"
+          # wasi sdk sets CC variables which break Python's configure script
+          # (it also sets WASI_SDK_PATH even without `add-to-path`, which is sufficient)
+          add-to-path: false
+      - uses: actions/cache/restore@v5
+        id: cache
         with:
-          repository: python/cpython
-          ref: 3.14
-          path: ${{ env.CPYTHON_PATH }}
-          fetch-depth: 1
-      - name: Build
-        run: |
-          cd ${{ env.CPYTHON_PATH }}
-          cat >> Tools/wasm/wasi/config.site-wasm32-wasi <<'EOF'
-
-          # Force-disable POSIX dynamic loading for WASI
-          ac_cv_func_dlopen=no
-          ac_cv_lib_dl_dlopen=no
-          EOF
-          python Tools/wasm/wasi build --quiet -- --config-cache
-          cp cross-build/wasm32-wasip1/libpython3.14.a \
-             cross-build/wasm32-wasip1/Modules/_hacl/libHacl_HMAC.a \
-             cross-build/wasm32-wasip1/Modules/_decimal/libmpdec/libmpdec.a \
-             cross-build/wasm32-wasip1/Modules/expat/libexpat.a \
-             cross-build/wasm32-wasip1/build/lib.wasi-wasm32-3.14/
+          path: |
+            .nox/wasi
+          key: wasi-${{ hashFiles('wasm/wasi/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }}
       - uses: Swatinem/rust-cache@v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
+      - name: Build
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: uvx nox -s build-wasm
       - name: Test
-        env:
-          PYO3_CROSS_LIB_DIR: ${{ env.CPYTHON_PATH }}/cross-build/wasm32-wasip1/build/lib.wasi-wasm32-3.14/
-          CARGO_BUILD_TARGET: wasm32-wasip1
-          CARGO_TARGET_WASM32_WASIP1_RUNNER: wasmtime run --dir ${{ env.CPYTHON_PATH }}::/ --env PYTHONPATH=/lib
-          RUSTFLAGS: >
-            -C link-arg=-L${{ env.WASI_SDK_PATH }}/share/wasi-sysroot/lib/wasm32-wasi
-            -C link-arg=-lwasi-emulated-signal
-            -C link-arg=-lwasi-emulated-process-clocks
-            -C link-arg=-lwasi-emulated-getpid
-            -C link-arg=-lmpdec
-            -C link-arg=-lHacl_HMAC
-            -C link-arg=-lexpat
-        run: RUSTDOCFLAGS=$RUSTFLAGS cargo test
+        run: uvx nox -s test-wasm
       - uses: actions/cache/save@v5
         if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
         with:
-          path: ${{ env.CPYTHON_PATH }}/cross-build/
-          key: ${{ steps.cache-wasip1-python.outputs.cache-primary-key }}
+          path: |
+            .nox/wasi
+          key: wasi-${{ hashFiles('wasm/wasi/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }}
 
   test-debug:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}

emscripten/Makefile

@@ -426,7 +426,7 @@ def contributors(session: nox.Session) -> None:
 
 class EmscriptenInfo:
     def __init__(self):
-        self.emscripten_dir = PYO3_DIR / "emscripten"
+        self.emscripten_dir = PYO3_DIR / "wasm" / "emscripten"
         self.builddir = PYO3_DIR / ".nox/emscripten"
         self.builddir.mkdir(exist_ok=True, parents=True)
 
@@ -503,6 +503,75 @@ def test_emscripten(session: nox.Session):
     )
 
 
+class WasiInfo:
+    def __init__(self):
+        self.wasi_dir = PYO3_DIR / "wasm" / "wasi"
+        self.builddir = PYO3_DIR / ".nox/wasi"
+        self.builddir.mkdir(exist_ok=True, parents=True)
+
+        self.pyversion = sys.version.split()[0]
+        self.pymajor, self.pyminor = self.pyversion.split(".")[:2]
+        self.pymajorminor = f"{self.pymajor}.{self.pyminor}"
+
+        # In CI the WASI SDK is installed by setup-wasi-sdk-action (sets WASI_SDK_PATH);
+        # otherwise the Makefile downloads it into the build dir.
+        wasi_sdk_env = os.environ.get("WASI_SDK_PATH")
+        self.wasi_sdk = (
+            Path(wasi_sdk_env) if wasi_sdk_env else self.builddir / "wasi-sdk"
+        )
+        self.cpython_dir = self.builddir / "build" / f"Python-{self.pyversion}"
+        crossbuild_dir = self.cpython_dir / "cross-build" / "wasm32-wasip1"
+        self.libdir = crossbuild_dir / "build" / f"lib.wasi-wasm32-{self.pymajorminor}"
+
+
+@nox.session(name="build-wasm", venv_backend="none")
+def build_wasm(session: nox.Session):
+    info = WasiInfo()
+    _run(
+        session,
+        "make",
+        "-C",
+        str(info.wasi_dir),
+        f"PYTHON={sys.executable}",
+        f"BUILDROOT={info.builddir}",
+        f"PYMAJORMINORMICRO={info.pyversion}",
+        external=True,
+    )
+
+
+@nox.session(name="test-wasm", venv_backend="none")
+def test_wasm(session: nox.Session):
+    info = WasiInfo()
+
+    target = "wasm32-wasip1"
+
+    # if wasmtime was installed by build-wasm, this is where it would be;
+    # in CI it is installed by the wasmtime/setup action
+    session.env["PATH"] = (
+        f"{info.builddir / 'wasmtime'}{os.pathsep}{os.environ['PATH']}"
+    )
+    session.env["PYO3_CROSS_LIB_DIR"] = str(info.libdir)
+    session.env["CARGO_BUILD_TARGET"] = target
+    session.env["CARGO_TARGET_WASM32_WASIP1_RUNNER"] = (
+        f"wasmtime run --dir {info.cpython_dir}::/ --env PYTHONPATH=/lib"
+    )
+    session.env["RUSTFLAGS"] = " ".join(
+        [
+            f"-C link-arg=-L{info.wasi_sdk}/share/wasi-sysroot/lib/wasm32-wasi",
+            "-C link-arg=-lwasi-emulated-signal",
+            "-C link-arg=-lwasi-emulated-process-clocks",
+            "-C link-arg=-lwasi-emulated-getpid",
+            "-C link-arg=-lmpdec",
+            "-C link-arg=-lHacl_HMAC",
+            "-C link-arg=-lexpat",
+        ]
+    )
+    session.env["RUSTDOCFLAGS"] = session.env["RUSTFLAGS"]
+    _run(session, "rustup", "target", "add", target, "--toolchain", "stable")
+
+    _run(session, "cargo", "test", *session.posargs, external=True)
+
+
 @nox.session(name="test-cross-compilation-windows")
 def test_cross_compilation_windows(session: nox.Session):
     session.install("cargo-xwin")

noxfile.py

@@ -0,0 +1,44 @@
+# Shared logic to download Python source tarball for the wasm builds.
+
+CURDIR=$(abspath .)
+
+# These three are passed in from nox.
+BUILDROOT ?= $(CURDIR)/builddir
+PYTHON ?= python3
+PYMAJORMINORMICRO ?= $(shell $(PYTHON) --version 2>&1 | awk '{print $$2}')
+
+# Set version variables.
+version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
+PYMAJOR=$(word 1,$(version_tuple))
+PYMINOR=$(word 2,$(version_tuple))
+PYMICRO=$(word 3,$(version_tuple))
+PYVERSION=$(PYMAJORMINORMICRO)
+PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
+
+ifneq ($(PYMAJORMINOR),3.14)
+$(error PYMAJORMINOR must be 3.14, got '$(PYMAJORMINOR)')
+endif
+
+PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
+PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
+PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
+
+.DEFAULT_GOAL := all
+
+$(BUILDROOT)/.exists:
+	mkdir -p $(BUILDROOT)
+	touch $@
+
+$(PYTHONTARBALL): $(BUILDROOT)/.exists
+	mkdir -p $(BUILDROOT)/downloads
+	curl -sL $(PYTHONURL) -o $@
+
+$(PYTHONBUILD)/.exists: $(PYTHONTARBALL)
+	[ -d $(PYTHONBUILD) ] || ( \
+		mkdir -p $(dir $(PYTHONBUILD));\
+		tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
+	)
+	touch $@
+
+clean:
+	rm -rf $(BUILDROOT)