Skip to content

Commit 54a1396

Browse files
oscarbenjaminoscarbenjamin
authored
Merge pull request #382 from oscarbenjamin/pr_pyodide_cleanup
Use cibuildwheel to build pyodide wheels
2 parents bdcad6c + 877a6a5 commit 54a1396

File tree

10 files changed

+216
-130
lines changed

10 files changed

+216
-130
lines changed

‎.github/workflows/buildwheel.yml‎

Lines changed: 123 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,57 +2,133 @@ name: Build
22

33
on: [push, pull_request]
44

5+
env:
6+
# These four values need to be kept in sync. Each pyodide version pins an
7+
# emscripten version and a CPython version.
8+
PYODIDE_VERSION: '0.29.3'
9+
PYODIDE_EMSCRIPTEN_VERSION: '4.0.9'
10+
PYODIDE_PYTHON_VERSION: '3.13'
11+
PYODIDE_CIBW_BUILD: 'cp313-*'
12+
513
jobs:
614
build_wheels:
7-
name: Build wheels for ${{ matrix.os }}
15+
name: ${{ matrix.name }}
816
runs-on: ${{ matrix.os }}
917
continue-on-error: true
1018
strategy:
1119
fail-fast: false
1220
matrix:
13-
os: [ubuntu-22.04, ubuntu-22.04-arm, windows-2022, windows-11-arm, macos-15-intel, macos-14]
21+
include:
22+
- name: Build manylinux x86-64 wheels
23+
os: ubuntu-22.04
24+
kind: native
25+
artifact_name: wheels-ubuntu-22.04
26+
cibw_platform: auto
27+
cibw_build: "*"
28+
- name: Build manylinux arm64 wheels
29+
os: ubuntu-22.04-arm
30+
kind: native
31+
artifact_name: wheels-ubuntu-22.04-arm
32+
cibw_platform: auto
33+
cibw_build: "*"
34+
- name: Build Windows x86-64 wheels
35+
os: windows-2022
36+
kind: native
37+
artifact_name: wheels-windows-2022
38+
cibw_platform: auto
39+
cibw_build: "*"
40+
- name: Build Windows arm64 wheels
41+
os: windows-11-arm
42+
kind: native
43+
artifact_name: wheels-windows-11-arm
44+
cibw_platform: auto
45+
cibw_build: "*"
46+
- name: Build macOS x86-64 wheels
47+
os: macos-15-intel
48+
kind: native
49+
artifact_name: wheels-macos-15-intel
50+
cibw_platform: auto
51+
cibw_build: "*"
52+
- name: Build macOS arm64 wheels
53+
os: macos-14
54+
kind: native
55+
artifact_name: wheels-macos-14
56+
cibw_platform: auto
57+
cibw_build: "*"
58+
- name: Build Pyodide wheels
59+
os: ubuntu-22.04
60+
kind: pyodide
61+
artifact_name: wheels-pyodide
62+
cibw_platform: pyodide
1463

1564
steps:
1665
- uses: actions/checkout@v6.0.2
1766

18-
- uses: actions/setup-python@v6
67+
- if: ${{ matrix.kind == 'native' }}
68+
uses: actions/setup-python@v6
1969
with:
2070
python-version: '3.13'
2171

22-
- uses: msys2/setup-msys2@v2.30.0
72+
# -------------- Windows stuff ---------------- #
73+
74+
- if: ${{ matrix.os == 'windows-2022' }}
75+
uses: msys2/setup-msys2@v2.30.0
2376
with:
2477
msystem: ucrt64
2578
update: true
26-
if: ${{ matrix.os == 'windows-2022' }}
2779

28-
- uses: msys2/setup-msys2@v2.30.0
80+
- if: ${{ matrix.os == 'windows-11-arm' }}
81+
uses: msys2/setup-msys2@v2.30.0
2982
with:
3083
msystem: clangarm64
3184
update: true
32-
if: ${{ matrix.os == 'windows-11-arm' }}
3385

3486
# Install pkgconfig on Windows from choco rather than from msys and
3587
# avoid using the Strawberry one.
36-
- run: choco install -y --stoponfirstfailure --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite
37-
if: ${{ startsWith( matrix.os , 'windows' ) }}
88+
- if: ${{ startsWith( matrix.os , 'windows' ) }}
89+
run: choco install -y --stoponfirstfailure --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite
3890

3991
# We have to set this here rather than in the cibuildwheel config
4092
# This is probably something to do with \ vs / in paths...
41-
- run: |
93+
- if: ${{ startsWith( matrix.os , 'windows' ) }}
94+
run: |
4295
$pkgConfigPath = "${{ github.workspace }}/.local/lib/pkgconfig"
4396
$pkgConfigPath = $pkgConfigPath.Replace('\', '/')
4497
echo "PKG_CONFIG_PATH=$pkgConfigPath" >> $env:GITHUB_ENV
45-
if: ${{ startsWith( matrix.os , 'windows' ) }}
98+
99+
# ------------- pyodide ------------- #
100+
101+
- if: ${{ matrix.kind == 'pyodide' }}
102+
name: Set up Emscripten toolchain
103+
uses: pyodide/setup-emsdk@ca2dd8aef8c2a0e11743c5c36f0b430ddb694b5c # v15
104+
with:
105+
version: ${{ env.PYODIDE_EMSCRIPTEN_VERSION }}
106+
actions-cache-folder: emsdk-cache
107+
108+
- if: ${{ matrix.kind == 'pyodide' }}
109+
name: Check out Emscripten patches for Pyodide
110+
uses: actions/checkout@v6.0.2
111+
with:
112+
repository: pyodide/pyodide
113+
ref: ${{ env.PYODIDE_VERSION }}
114+
path: pyodide-patches
115+
sparse-checkout: |
116+
emsdk/patches/
117+
118+
# ------------- actual build ------------- #
46119

47120
- name: Build wheels
48121
uses: pypa/cibuildwheel@298ed2fb2c105540f5ed055e8a6ad78d82dd3a7e # v3.3.1
49122
env:
123+
CIBW_PLATFORM: ${{ matrix.cibw_platform }}
124+
CIBW_BUILD: ${{ matrix.kind == 'pyodide' && env.PYODIDE_CIBW_BUILD || matrix.cibw_build }}
125+
CIBW_PYODIDE_VERSION: ${{ env.PYODIDE_VERSION }}
50126
# override setting in pyproject.toml to use msys2 instead of msys64 bash
51127
CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-11-arm' && 'msys2 -c bin/cibw_before_all_windows_arm64.sh' || 'msys2 -c bin/cibw_before_all_windows_amd64.sh' }}
52128

53129
- uses: actions/upload-artifact@v7
54130
with:
55-
name: wheels-${{ matrix.os }}
131+
name: ${{ matrix.artifact_name }}
56132
path: wheelhouse/*.whl
57133

58134
build_sdist:
@@ -64,7 +140,7 @@ jobs:
64140

65141
- uses: actions/setup-python@v6
66142
with:
67-
python-version: '3.13'
143+
python-version: ${{ env.PYODIDE_PYTHON_VERSION }}
68144

69145
- run: bin/install_latest_flint_ubuntu.sh
70146
- run: pip install build
@@ -120,6 +196,35 @@ jobs:
120196
121197
- run: python -m flint.test --verbose
122198

199+
test_pyodide:
200+
needs: build_wheels
201+
name: Test Pyodide wheel
202+
runs-on: ubuntu-22.04
203+
204+
steps:
205+
- uses: actions/setup-python@v6
206+
with:
207+
python-version: ${{ env.PYODIDE_PYTHON_VERSION }}
208+
209+
- uses: actions/setup-node@v6
210+
with:
211+
node-version: '22'
212+
213+
- run: pip install pyodide-build
214+
- run: pyodide xbuildenv install "${{ env.PYODIDE_VERSION }}"
215+
216+
- uses: actions/download-artifact@v8
217+
with:
218+
name: wheels-pyodide
219+
path: wheelhouse
220+
221+
- run: |
222+
pyodide venv .venv-pyodide
223+
source .venv-pyodide/bin/activate
224+
pip install wheelhouse/*.whl
225+
pip install pytest hypothesis
226+
python -m pytest -svra -p no:cacheprovider --pyargs flint
227+
123228
# On new enough Ubuntu we can build against the system deb.
124229
test_pip_flint_deb:
125230
name: Build on ${{ matrix.os }}
@@ -338,7 +443,11 @@ jobs:
338443
merge-multiple: true
339444

340445
- name: Copy the PyPI files into dist
341-
run: mkdir dist && cp wheelhouse/*.whl wheelhouse/*.tar.gz dist
446+
# pyodide wheels cannot be uploaded to PyPI
447+
run: |
448+
mkdir dist
449+
rm wheelhouse/*pyodide*.whl
450+
cp wheelhouse/*.whl wheelhouse/*.tar.gz dist
342451
343452
- name: Publish package on PyPI
344453
# It is recommended to pin a commit hash here for security but it

‎.github/workflows/ci-emscripten.yml‎

Lines changed: 0 additions & 107 deletions
This file was deleted.

‎bin/build_dependencies_unix.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ done
140140

141141
source bin/build_variables.sh
142142

143+
mkdir -p "$PREFIX"
143144
cd $PREFIX
144145
mkdir -p src
145146
cd src

‎bin/build_variables.sh‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
#!/usr/bin/env bash
22
#
3-
# Create a local directory .local to be used as --prefix when building
4-
# local installs of python-flint's dependencies. This also sets the PREFIX
5-
# shell variable and environment variables giving the versions to use for each
6-
# dependency. This script should be sourced rather than executed e.g.:
3+
# Set the PREFIX shell variable and environment variables giving the versions
4+
# to use for each dependency. This script should be sourced rather than
5+
# executed e.g.:
76
#
87
# $ source bin/build_variables.sh
98
#
109
# This is used implicitly by the other build scripts and does not need to be
1110
# executed directly.
1211

1312
PREFIX=$(pwd)/.local
14-
mkdir -p $PREFIX
1513

1614
ARBVER=2.23.0 # Not needed with flint >= 3.0.0 (Arb is included in flint)
1715

‎bin/build_wheel.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ set -o errexit
77

88
source bin/build_variables.sh
99

10+
mkdir -p "$PREFIX"
1011
python3 -m venv $PREFIX/venv
1112
source $PREFIX/venv/bin/activate
1213
pip install -U pip

‎bin/cibw_before_all_pyodide.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
6+
PROJECT_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd)
7+
WASM_LIBRARY_DIR=${WASM_LIBRARY_DIR:-"$PROJECT_DIR/wasm-library-dir"}
8+
9+
if [ -f "$WASM_LIBRARY_DIR/lib/pkgconfig/flint.pc" ]; then
10+
echo "Using cached Pyodide dependencies from $WASM_LIBRARY_DIR"
11+
exit 0
12+
fi
13+
14+
mkdir -p "$WASM_LIBRARY_DIR"
15+
16+
# The Pyodide cibuildwheel environment for building python-flint itself sets
17+
# include/library flags that reference libgmp/libmpfr/libflint. Those libraries
18+
# do not exist yet while bootstrapping the static dependencies here, so letting
19+
# them leak into configure would break link tests such as "does the C compiler
20+
# work?" for GMP. Use a minimal environment for the bootstrap step.
21+
unset PKG_CONFIG_PATH
22+
unset LDFLAGS
23+
export CFLAGS="-fPIC"
24+
25+
"$SCRIPT_DIR/pyodide_build_dependencies.sh" --wasm-library-dir "$WASM_LIBRARY_DIR"

‎bin/cibw_before_build_pyodide.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
python -m pip install wheel auditwheel_emscripten
6+
7+
EMSCRIPTEN_DIR="$HOME/.cache/cibuildwheel/emsdk-$PYODIDE_EMSCRIPTEN_VERSION/emsdk-$PYODIDE_EMSCRIPTEN_VERSION/upstream/emscripten"
8+
9+
for patch_file in pyodide-patches/emsdk/patches/*.patch; do
10+
echo "Applying Pyodide Emscripten patch $(basename "$patch_file")"
11+
patch -p1 --verbose -d "$EMSCRIPTEN_DIR" < "$patch_file"
12+
done

0 commit comments

Comments
 (0)