ci: bump emscripten build to 3.14 (#5455)

* ci: bump emscripten build to 3.13

* try 3.14

* fix build of Python 3.14

* restore env.sh usage

* copy correct .a files

* also copy mpdec, ffi

* use 3.14 head with emscripten download

* fixup using head CPython

* fix finding emsdk_env in noxfile

* cleanup

* cleanup python passing from noxfile to make

Author: davidhewitt

.github/workflows/ci.yml

@@ -457,8 +457,7 @@ jobs:
       - uses: actions/checkout@v6.0.2
       - uses: actions/setup-python@v6
         with:
-          # TODO bump emscripten builds to test on 3.13
-          python-version: 3.11
+          python-version: 3.14
         id: setup-python
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -467,7 +466,7 @@ jobs:
           components: rust-src
       - uses: actions/setup-node@v6
         with:
-          node-version: 18
+          node-version: 24
       - run: python -m pip install --upgrade pip && pip install nox[uv]
       - uses: actions/cache/restore@v5
         id: cache

emscripten/.gitignore

@@ -1 +1,2 @@
 pybuilddir.txt
+builddir

emscripten/Makefile

@@ -2,21 +2,14 @@ CURDIR=$(abspath .)
 
 # These three are passed in from nox.
 BUILDROOT ?= $(CURDIR)/builddir
-PYMAJORMINORMICRO ?= 3.11.0
+PYTHON ?= python3
+PYMAJORMINORMICRO ?= $(shell $(PYTHON) --version 2>&1 | awk '{print $$2}')
 
-EMSCRIPTEN_VERSION=3.1.68
-
-export EMSDKDIR = $(BUILDROOT)/emsdk
+export EMSDKDIR = $(PYTHONBUILD)/emsdk-cache
 
 PLATFORM=wasm32_emscripten
 SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM)
 
-# BASH_ENV tells bash to source emsdk_env.sh on startup.
-export BASH_ENV := $(CURDIR)/env.sh
-# Use bash to run each command so that env.sh will be used.
-SHELL := /bin/bash
-
-
 # Set version variables.
 version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
 PYMAJOR=$(word 1,$(version_tuple))
@@ -27,61 +20,66 @@ PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
 
 
 PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
-PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
-PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
+# TODO: resume download once 3.14.4 ships with emscripten cache
+# PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
+# PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
+PYTHONBUILD=$(BUILDROOT)/build/cpython
 
 PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib
 
+CROSS_PYTHON=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python/python.sh
+
 all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
 
-$(BUILDROOT)/.exists: 
+$(BUILDROOT)/.exists:
 	mkdir -p $(BUILDROOT)
 	touch $@
 
+# TODO: use tarball once 3.14.4 ships with emscripten cache
 
-# Install emscripten
-$(EMSDKDIR): $(CURDIR)/emscripten_patches/* $(BUILDROOT)/.exists
-	git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR)
-	$(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION)
-	cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1
-	$(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION)
+# $(PYTHONTARBALL):
+# 	[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
+# 	wget -q -O $@ $(PYTHONURL)
 
+# $(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
+# 	[ -d $(PYTHONBUILD) ] || ( \
+# 		mkdir -p $(dir $(PYTHONBUILD));\
+# 		tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
+# 	)
+# 	touch $@
 
-$(PYTHONTARBALL):
-	[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
-	wget -q -O $@ $(PYTHONURL)
+ifneq ($(PYMAJORMINOR),3.14)
+$(error PYMAJORMINOR must be 3.14, got '$(PYMAJORMINOR)')
+endif
 
-$(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
+$(PYTHONBUILD)/.patched: $(BUILDROOT)/.exists
 	[ -d $(PYTHONBUILD) ] || ( \
 		mkdir -p $(dir $(PYTHONBUILD));\
-		tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
+		git clone --depth 1 --branch 3.14 https://github.com/python/cpython $(PYTHONBUILD) \
 	)
 	touch $@
 
-$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(BUILDROOT)/emsdk
-	cd $(PYTHONBUILD) && \
-	CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
-  	emconfigure ./configure -C \
-		--host=wasm32-unknown-emscripten \
-		--build=$(shell $(PYTHONBUILD)/config.guess) \
-		--with-emscripten-target=browser \
-		--enable-wasm-dynamic-linking \
-		--with-build-python=python3.11
-
-$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile
+$(CROSS_PYTHON): $(PYTHONBUILD)/.patched
 	cd $(PYTHONBUILD) && \
-		emmake make -j3 libpython$(PYMAJORMINOR).a
+	$(PYTHON) Tools/wasm/emscripten install-emscripten --quiet --emsdk-cache=$(EMSDKDIR) && \
+	$(PYTHON) Tools/wasm/emscripten build --quiet --emsdk-cache=$(EMSDKDIR)
 
+$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a: $(CROSS_PYTHON)
 	# Generate sysconfigdata
-	_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars
+	_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python $(CROSS_PYTHON) -m sysconfig --generate-posix-vars
 	cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib
 
 	mkdir -p $(PYTHONLIBDIR)
-	# Copy libexpat.a, libmpdec.a, and libpython3.11.a
-	# In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS
-	find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
+	# Make a static library for _hacl, for some reason these are missing from the build?
+	# source emsdk_env.sh to get emar in PATH, works best when done from the emsdk directory
+	EMSDK_ENV=$$(find $(PYTHONBUILD)/emsdk-cache -name 'emsdk_env.sh' | head -n 1) && \
+	    cd $$(dirname $$EMSDK_ENV) && \
+		. $$EMSDK_ENV && \
+	    cd $(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python && \
+		emar rcs Modules/_hacl/libhacl.a Modules/_hacl/*.o
+	# Copy all .a libraries
+	find $(PYTHONBUILD)/cross-build/wasm32-emscripten/ -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
 	# Install Python stdlib
 	cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
-
 clean:
 	rm -rf $(BUILDROOT)

emscripten/emscripten_patches/0001-Add-_gxx_personality_v0-stub-to-library.js.patch

@@ -5,4 +5,7 @@
 
 p = pathlib.Path(sys.argv[1])
 
-sys.exit(subprocess.call(["node", p.name], cwd=p.parent))
+command = ["node", p.name, *sys.argv[2:]]
+print("Running:", " ".join(command))
+
+sys.exit(subprocess.call(command, cwd=p.parent))

emscripten/env.sh

@@ -12,6 +12,7 @@
 from functools import lru_cache
 from glob import glob
 from pathlib import Path
+from shlex import quote
 from typing import (
     Any,
     Callable,
@@ -438,22 +439,29 @@ def test_emscripten(session: nox.Session):
             f"-C link-arg={pythonlibdir}@/lib/python{info.pymajorminor}",
             f"-C link-arg=-lpython{info.pymajorminor}",
             "-C link-arg=-lexpat",
+            "-C link-arg=-lffi",
             "-C link-arg=-lmpdec",
-            "-C link-arg=-lsqlite3",
-            "-C link-arg=-lz",
-            "-C link-arg=-lbz2",
+            "-C link-arg=-lhacl",
+            "-C link-arg=-sUSE_SQLITE3",
+            "-C link-arg=-sUSE_ZLIB",
+            "-C link-arg=-sUSE_BZIP2",
+            "-C link-arg=-sEXPORTED_FUNCTIONS=_main,__PyRuntime",
             "-C link-arg=-sALLOW_MEMORY_GROWTH=1",
+            "-C link-arg=-sSTACK_SIZE=262144",
         ]
     )
     session.env["RUSTDOCFLAGS"] = session.env["RUSTFLAGS"]
     session.env["CARGO_BUILD_TARGET"] = target
     session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir
     _run(session, "rustup", "target", "add", target, "--toolchain", "stable")
+
+    emsdk_env = next(info.builddir.glob("**/emsdk-cache/**/emsdk_env.sh"))
+
     _run(
         session,
         "bash",
         "-c",
-        f"source {info.builddir / 'emsdk/emsdk_env.sh'} && cargo test",
+        f"source {emsdk_env} && cargo test {' '.join(quote(arg) for arg in session.posargs)}",
     )