From a083d0241a265a97adf8356a22f955f98046bfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:08:37 -0400 Subject: [PATCH 01/20] cu: Migrate python to cibuildwheel * Add Python 3.13 to pyproject.toml supported versions. * Configure cibuildwheel for Linux and Musl Linux builds, including architecture support for x86_64 and aarch64. * Introduce new GitHub Actions workflows for building containerized and native wheels, enhancing CI/CD capabilities. * Update existing workflows to use cibuildwheel for building wheels across various Python versions and architectures. --- .../workflows/build-containerized-wheels.yaml | 79 ++++++++++++++++ .github/workflows/build-native-wheels.yaml | 80 ++++++++++++++++ .github/workflows/build-wheels-for-pr.yaml | 47 ---------- .../workflows/build-wheels-for-release.yaml | 33 ------- .github/workflows/build-wheels.yaml | 94 ------------------- .github/workflows/release-python.yaml | 2 +- python/pyproject.toml | 14 +++ 7 files changed, 174 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/build-containerized-wheels.yaml create mode 100644 .github/workflows/build-native-wheels.yaml delete mode 100644 .github/workflows/build-wheels-for-pr.yaml delete mode 100644 .github/workflows/build-wheels-for-release.yaml delete mode 100644 .github/workflows/build-wheels.yaml diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml new file mode 100644 index 0000000000..4735765a34 --- /dev/null +++ b/.github/workflows/build-containerized-wheels.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Containerized Wheels + +on: + push: + branches: [main] + tags: ["v*"] + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + pull_request: + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + +permissions: + contents: read + actions: write + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + - os: ubuntu-24.04-arm + arch: aarch64 + steps: + - uses: actions/checkout@v5 + + - name: Install Bazel + run: ./ci/run_ci.sh install_bazel + + - name: Build wheels with cibuildwheel + uses: pypa/cibuildwheel@v3 + env: + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} + with: + output-dir: dist/ + + - name: Install and verify wheel + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + + - name: Upload wheels + if: github.ref_type == 'tag' + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-linux-${{ matrix.arch }} + path: dist/*.whl diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml new file mode 100644 index 0000000000..e8f48f17ab --- /dev/null +++ b/.github/workflows/build-native-wheels.yaml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native Wheels + +on: + push: + branches: [main] + tags: ["v*"] + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + pull_request: + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + +permissions: + contents: read + actions: write + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-13, macos-14, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v5 + + - name: Install Bazel + if: ${{ !contains(matrix.os, 'windows') }} + run: ./ci/run_ci.sh install_bazel + + - name: Install Bazel for Windows + if: ${{ contains(matrix.os, 'windows') }} + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + + - name: Build wheels with cibuildwheel + uses: pypa/cibuildwheel@v3 + env: + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} + with: + output-dir: dist/ + + - name: Install and verify wheel + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + + - name: Upload wheels + if: github.ref_type == 'tag' + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }} + path: dist/*.whl diff --git a/.github/workflows/build-wheels-for-pr.yaml b/.github/workflows/build-wheels-for-pr.yaml deleted file mode 100644 index ec85455f88..0000000000 --- a/.github/workflows/build-wheels-for-pr.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: build wheels for pull request -on: - push: - branches: - - main - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' - pull_request: - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' -jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml - strategy: - matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] - python-version: ['3.8', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-wheels-for-release.yaml b/.github/workflows/build-wheels-for-release.yaml deleted file mode 100644 index e1799e6de5..0000000000 --- a/.github/workflows/build-wheels-for-release.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: build wheels for release -on: - push: - tags: ["v*"] - -jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml - strategy: - matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - bump-version: true diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml deleted file mode 100644 index 554f4a0065..0000000000 --- a/.github/workflows/build-wheels.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Build Wheels - -on: - workflow_call: - inputs: - os: - required: true - type: string - python-version: - required: true - type: string - bump-version: - description: 'Whether to bump the version in setup.py' - required: false - type: boolean - default: false - -permissions: - contents: read - actions: write - -jobs: - build_and_test: - name: Build and Test - runs-on: ${{ inputs.os }} - - steps: - - uses: actions/checkout@v5 - - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: Install bazel - if: "runner.os != 'Windows'" - run: ./ci/run_ci.sh install_bazel - - - name: Install bazel - if: "runner.os == 'Windows'" - run: ./ci/run_ci.sh install_bazel_windows - shell: bash - - - name: Update version in setup.py - if: "inputs.bump-version" - run: ./ci/deploy.sh bump_py_version - - - name: Build a binary wheel (Linux, manylinux) - if: "runner.os == 'Linux'" - env: - manylinux_x86_64_image: ${{ env.manylinux_x86_64_image }} - manylinux_aarch64_image: ${{ env.manylinux_aarch64_image }} - GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - ./ci/build_manylinux_wheel.sh --os "${{ runner.os }}" \ - --arch "${{ runner.arch }}" \ - --python "${{ inputs.python-version }}" \ - --workspace "${GITHUB_WORKSPACE}" - - - name: Build a binary wheel (native) - if: "runner.os != 'Linux'" - run: ./ci/deploy.sh build_pyfory - shell: bash - - - name: Install and verify wheel - shell: bash - run: | - python -m pip install --upgrade pip - pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - - name: Upload wheel -# if: ${{ inputs.bump-version }} - uses: actions/upload-artifact@v4 - with: - name: pyfory-wheels-${{ inputs.os }}-${{ inputs.python-version }}${{ inputs.bump-version && '-tagged' || github.sha }} - path: dist/*.whl diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 938ef2e9a4..52701a6149 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -19,7 +19,7 @@ name: Publish Python on: workflow_run: - workflows: ["build wheels for release"] + workflows: ["Build Native Wheels", "Build Containerized, Wheels"] types: [completed] permissions: diff --git a/python/pyproject.toml b/python/pyproject.toml index 58b81674e3..3bce42c8fe 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", ] keywords = ["fory", "serialization", "multi-language", "fast", "row-format", "jit", "codegen", "polymorphic", "zero-copy"] @@ -97,3 +98,16 @@ ignore = [ [tool.ruff.format] # Replicates the `inline-quotes` setting from .flake8 quote-style = "double" + +[tool.cibuildwheel] +build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" + +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64"] +build-selector = "manylinux_2_17_*" +repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" + +[tool.cibuildwheel.musllinux] +archs = ["x86_64", "aarch64"] +build-selector = "musllinux_1_2_*" +repair-wheel-command = "delvewheel repair --musllinux=1_2 -w {dest_dir} {wheel}" From 016a9fa745e0e559491cd46faac07a439b2654de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:13:43 -0400 Subject: [PATCH 02/20] pin cibuildwheel --- .github/workflows/build-containerized-wheels.yaml | 2 +- .github/workflows/build-native-wheels.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 4735765a34..d1c0083e31 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -58,7 +58,7 @@ jobs: run: ./ci/run_ci.sh install_bazel - name: Build wheels with cibuildwheel - uses: pypa/cibuildwheel@v3 + uses: pypa/cibuildwheel@v3.1.3 env: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index e8f48f17ab..3c2e1ce9f9 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -60,7 +60,7 @@ jobs: shell: bash - name: Build wheels with cibuildwheel - uses: pypa/cibuildwheel@v3 + uses: pypa/cibuildwheel@v3.1.3 env: CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} with: From 5192fba59f054e3271cee638a50447e589c15f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:23:11 -0400 Subject: [PATCH 03/20] pass working-directory: python --- .github/workflows/build-containerized-wheels.yaml | 6 ++++-- .github/workflows/build-native-wheels.yaml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index d1c0083e31..7de1bd03e5 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -59,9 +59,10 @@ jobs: - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 + working-directory: python env: CIBW_ARCHS: ${{ matrix.arch }} - CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} with: output-dir: dist/ @@ -70,10 +71,11 @@ jobs: python -m pip install --upgrade pip pip install dist/*.whl python -c "import pyfory; print(pyfory.__version__)" + working-directory: python - name: Upload wheels if: github.ref_type == 'tag' uses: actions/upload-artifact@v4 with: name: pyfory-wheels-linux-${{ matrix.arch }} - path: dist/*.whl + path: python/dist/*.whl \ No newline at end of file diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index 3c2e1ce9f9..7b5f448492 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -61,6 +61,7 @@ jobs: - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 + working-directory: python env: CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} with: From d98a4dc650adc5e083dcabb64aa3e4a431c7bdf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:24:44 -0400 Subject: [PATCH 04/20] add empty line --- .github/workflows/build-containerized-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 7de1bd03e5..efd77b425a 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -78,4 +78,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: pyfory-wheels-linux-${{ matrix.arch }} - path: python/dist/*.whl \ No newline at end of file + path: python/dist/*.whl From 39a0fb42943536f378a22842ac5c0ae54f3b89fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:30:09 -0400 Subject: [PATCH 05/20] Update cibuildwheel output directory and package dir * Change cibuildwheel output directory from `dist/` to `wheelhouse/` in `build-containerized-wheels.yaml` and `build-native-wheels.yaml`. * Set `package-dir` to `python/` in `build-native-wheels.yaml`. * Update artifact paths to reflect the new output directory. --- .github/workflows/build-containerized-wheels.yaml | 5 ++--- .github/workflows/build-native-wheels.yaml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index efd77b425a..8b0a2e5553 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -59,12 +59,11 @@ jobs: - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 - working-directory: python env: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} with: - output-dir: dist/ + output-dir: wheelhouse/ - name: Install and verify wheel run: | @@ -78,4 +77,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: pyfory-wheels-linux-${{ matrix.arch }} - path: python/dist/*.whl + path: wheelhouse/*.whl diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index 7b5f448492..b48cfed358 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -61,11 +61,11 @@ jobs: - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 - working-directory: python env: CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} with: - output-dir: dist/ + package-dir: python/ + output-dir: wheelhouse/ - name: Install and verify wheel run: | @@ -78,4 +78,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }} - path: dist/*.whl + path: wheelhouse/*.whl From d444d5c383edd4ba09441b0cf77bbce516945293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:32:22 -0400 Subject: [PATCH 06/20] Option 'musllinux' not supported in a config file --- python/pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 3bce42c8fe..9480d9dc4d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -106,8 +106,3 @@ build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" archs = ["x86_64", "aarch64"] build-selector = "manylinux_2_17_*" repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" - -[tool.cibuildwheel.musllinux] -archs = ["x86_64", "aarch64"] -build-selector = "musllinux_1_2_*" -repair-wheel-command = "delvewheel repair --musllinux=1_2 -w {dest_dir} {wheel}" From daab11b67e6bb948e4cfa453d405aa3380c020b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:33:19 -0400 Subject: [PATCH 07/20] Add package-dir: python/ --- .github/workflows/build-containerized-wheels.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 8b0a2e5553..5e8de7dfcb 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -63,6 +63,7 @@ jobs: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} with: + package-dir: python/ output-dir: wheelhouse/ - name: Install and verify wheel From 2b5e6e3ba1b10dea5f8bc2a3cabb41115004a3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:41:24 -0400 Subject: [PATCH 08/20] CIBW_SKIP: "*-win32" --- .github/workflows/build-native-wheels.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index b48cfed358..91ad10caed 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -63,6 +63,7 @@ jobs: uses: pypa/cibuildwheel@v3.1.3 env: CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} + CIBW_SKIP: "*-win32" with: package-dir: python/ output-dir: wheelhouse/ From 608cfe94f54a1943819e74d679fee62dcc89a07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 17 Aug 2025 23:49:23 -0400 Subject: [PATCH 09/20] disable [tool.cibuildwheel.linux], add "skip" --- .github/workflows/build-native-wheels.yaml | 1 - python/pyproject.toml | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index 91ad10caed..b48cfed358 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -63,7 +63,6 @@ jobs: uses: pypa/cibuildwheel@v3.1.3 env: CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} - CIBW_SKIP: "*-win32" with: package-dir: python/ output-dir: wheelhouse/ diff --git a/python/pyproject.toml b/python/pyproject.toml index 9480d9dc4d..99ad7fe945 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -101,8 +101,9 @@ quote-style = "double" [tool.cibuildwheel] build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" +skip = "*-win32" -[tool.cibuildwheel.linux] -archs = ["x86_64", "aarch64"] -build-selector = "manylinux_2_17_*" -repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" +#[tool.cibuildwheel.linux] +#archs = ["x86_64", "aarch64"] +#build-selector = "manylinux_2_17_*" +#repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" From 7d8c1784413346bbc692be78326e5ed6d055756b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:00:01 -0400 Subject: [PATCH 10/20] Install bazel in cibuildwheel w/CIBW_BEFORE_BUILD --- .github/workflows/build-containerized-wheels.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 5e8de7dfcb..264217fcb0 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -54,14 +54,15 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Install Bazel - run: ./ci/run_ci.sh install_bazel - - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 env: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} + CIBW_BEFORE_BUILD: | + ./ci/run_ci.sh install_bazel + which bazel || true + bazel --version || true with: package-dir: python/ output-dir: wheelhouse/ From 8dafb4279176a76069a9e681422e3b89b7647a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:09:53 -0400 Subject: [PATCH 11/20] Download bazel to a local folder, without sudo --- .github/workflows/build-containerized-wheels.yaml | 2 +- ci/run_ci.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 264217fcb0..187277d3a5 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -59,7 +59,7 @@ jobs: env: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} - CIBW_BEFORE_BUILD: | + CIBW_BEFORE_ALL: | ./ci/run_ci.sh install_bazel which bazel || true bazel --version || true diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 0c4fb52f64..bcb4be5e31 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -90,15 +90,15 @@ install_bazel() { esac BAZEL_VERSION=$(get_bazel_version) - BAZEL_DIR="/usr/local/bin" + BAZEL_DIR="$HOME/.local/bin" # Construct platform-specific URL BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}" echo "Downloading bazel from: $BINARY_URL" - sudo wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } + wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } - sudo chmod +x "$BAZEL_DIR/bazel" + chmod +x "$BAZEL_DIR/bazel" # Add to current shell's PATH export PATH="$BAZEL_DIR:$PATH" From 1cd8459cfc5ef6ebc7eb64b98f1588b04e3fe913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:18:19 -0400 Subject: [PATCH 12/20] Replace wget with curl for manylinux, define MACOSX_DEPLOYMENT_TARGET --- ci/run_ci.sh | 2 +- python/pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index bcb4be5e31..7a2559885f 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -96,7 +96,7 @@ install_bazel() { BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}" echo "Downloading bazel from: $BINARY_URL" - wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } + curl -fsSL -o "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } chmod +x "$BAZEL_DIR/bazel" diff --git a/python/pyproject.toml b/python/pyproject.toml index 99ad7fe945..7de4fd311c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -103,6 +103,9 @@ quote-style = "double" build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" skip = "*-win32" +[tool.cibuildwheel.macos] +environment = {MACOSX_DEPLOYMENT_TARGET="14.5"} + #[tool.cibuildwheel.linux] #archs = ["x86_64", "aarch64"] #build-selector = "manylinux_2_17_*" From d3687c4b9a2f10d81d664f6a307bf51e6977e405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:21:43 -0400 Subject: [PATCH 13/20] Create directory for curl --- .github/workflows/build-containerized-wheels.yaml | 1 + .github/workflows/build-native-wheels.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 187277d3a5..c571a01d73 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -60,6 +60,7 @@ jobs: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} CIBW_BEFORE_ALL: | + mkdir -p $HOME/.local/bin ./ci/run_ci.sh install_bazel which bazel || true bazel --version || true diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index b48cfed358..e05365bf5d 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-13, macos-14, macos-latest, windows-latest] + os: [macos-14, macos-latest, windows-latest] steps: - uses: actions/checkout@v5 From 74e20006db8eac9bfc2cb13c9bc76cd98488c8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:24:31 -0400 Subject: [PATCH 14/20] Export PATH --- .github/workflows/build-containerized-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index c571a01d73..35add18abf 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -62,7 +62,7 @@ jobs: CIBW_BEFORE_ALL: | mkdir -p $HOME/.local/bin ./ci/run_ci.sh install_bazel - which bazel || true + export PATH="$HOME.local/bin:$PATH" bazel --version || true with: package-dir: python/ From e4206c1c1827ed23496b1524e72507494f720722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:42:48 -0400 Subject: [PATCH 15/20] drop 3.13 for now --- .github/workflows/build-containerized-wheels.yaml | 1 + .github/workflows/build-native-wheels.yaml | 2 +- ci/run_ci.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 35add18abf..8f11fdd018 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -62,6 +62,7 @@ jobs: CIBW_BEFORE_ALL: | mkdir -p $HOME/.local/bin ./ci/run_ci.sh install_bazel + echo "Bazel installed" export PATH="$HOME.local/bin:$PATH" bazel --version || true with: diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index e05365bf5d..1d33b7f5d2 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -62,7 +62,7 @@ jobs: - name: Build wheels with cibuildwheel uses: pypa/cibuildwheel@v3.1.3 env: - CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp38-* cp313-*' }} + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-*' || 'cp38-* cp312-*' }} with: package-dir: python/ output-dir: wheelhouse/ diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 7a2559885f..5b2abab262 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -108,7 +108,7 @@ install_bazel() { bazel version || { echo "Bazel installation verification failed"; exit 1; } # Configure number of jobs based on memory - if [[ "$MACHINE" == linux ]]; then + if [[ "$OS" == linux ]]; then MEM=$(grep MemTotal < /proc/meminfo | awk '{print $2}') JOBS=$(( MEM / 1024 / 1024 / 3 )) echo "build --jobs=$JOBS" >> ~/.bazelrc From b1bdf3650e3985dfa36241bd557065836ac92936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:44:34 -0400 Subject: [PATCH 16/20] fix PATH --- .github/workflows/build-containerized-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 8f11fdd018..10534fec69 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -63,7 +63,7 @@ jobs: mkdir -p $HOME/.local/bin ./ci/run_ci.sh install_bazel echo "Bazel installed" - export PATH="$HOME.local/bin:$PATH" + export PATH="$HOME/.local/bin:$PATH" bazel --version || true with: package-dir: python/ From e01f30c0c9bb32c941eb1900010f7e27dfe45fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:50:52 -0400 Subject: [PATCH 17/20] MACOSX_DEPLOYMENT_TARGET=15.0 --- .github/workflows/build-containerized-wheels.yaml | 4 +--- .github/workflows/build-native-wheels.yaml | 2 +- python/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 10534fec69..5ffa9696d7 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -59,12 +59,10 @@ jobs: env: CIBW_ARCHS: ${{ matrix.arch }} CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} + CIBW_ENVIRONMENT: PATH=$HOME/.local/bin:$PATH CIBW_BEFORE_ALL: | mkdir -p $HOME/.local/bin ./ci/run_ci.sh install_bazel - echo "Bazel installed" - export PATH="$HOME/.local/bin:$PATH" - bazel --version || true with: package-dir: python/ output-dir: wheelhouse/ diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml index 1d33b7f5d2..3bc3679a0e 100644 --- a/.github/workflows/build-native-wheels.yaml +++ b/.github/workflows/build-native-wheels.yaml @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-14, macos-latest, windows-latest] + os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v5 diff --git a/python/pyproject.toml b/python/pyproject.toml index 7de4fd311c..ffe1344493 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -104,7 +104,7 @@ build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" skip = "*-win32" [tool.cibuildwheel.macos] -environment = {MACOSX_DEPLOYMENT_TARGET="14.5"} +environment = {MACOSX_DEPLOYMENT_TARGET="15"} #[tool.cibuildwheel.linux] #archs = ["x86_64", "aarch64"] From 7605699a2b3cfd39eb2aaa3859f401f514eb4557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 00:54:32 -0400 Subject: [PATCH 18/20] drop 313 in containerized wheels too --- .github/workflows/build-containerized-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml index 5ffa9696d7..a656c0b39f 100644 --- a/.github/workflows/build-containerized-wheels.yaml +++ b/.github/workflows/build-containerized-wheels.yaml @@ -58,7 +58,7 @@ jobs: uses: pypa/cibuildwheel@v3.1.3 env: CIBW_ARCHS: ${{ matrix.arch }} - CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*' || 'cp313-*' }} + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-*' || 'cp38-* cp312-*' }} CIBW_ENVIRONMENT: PATH=$HOME/.local/bin:$PATH CIBW_BEFORE_ALL: | mkdir -p $HOME/.local/bin From d1520a2fb2701994208f105b8e007394e8459d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 01:20:14 -0400 Subject: [PATCH 19/20] Use manylinux_2_24 --- python/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index ffe1344493..1a8f110bcd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -65,7 +65,7 @@ include-package-data = true zip-safe = false [tool.setuptools.package-data] -"pyfory" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] +"pyfory" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd", ] "pyfory.format" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] "pyfory.lib.mmh3" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] @@ -102,6 +102,8 @@ quote-style = "double" [tool.cibuildwheel] build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" skip = "*-win32" +manylinux_x86_64-image = "manylinux_2_24" +manylinux-aarch64-image = "manylinux_2_24" [tool.cibuildwheel.macos] environment = {MACOSX_DEPLOYMENT_TARGET="15"} From 52a43683ec6849613c8cf7f930792f70ebd092dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 18 Aug 2025 01:22:36 -0400 Subject: [PATCH 20/20] fix typo in toml file --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 1a8f110bcd..54098724bd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -102,7 +102,7 @@ quote-style = "double" [tool.cibuildwheel] build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" skip = "*-win32" -manylinux_x86_64-image = "manylinux_2_24" +manylinux-x86_64-image = "manylinux_2_24" manylinux-aarch64-image = "manylinux_2_24" [tool.cibuildwheel.macos]