diff --git a/.github/workflows/build-containerized-wheels.yaml b/.github/workflows/build-containerized-wheels.yaml new file mode 100644 index 0000000000..a656c0b39f --- /dev/null +++ b/.github/workflows/build-containerized-wheels.yaml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Containerized Wheels + +on: + push: + branches: [main] + tags: ["v*"] + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + pull_request: + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + +permissions: + contents: read + actions: write + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + - os: ubuntu-24.04-arm + arch: aarch64 + steps: + - uses: actions/checkout@v5 + + - name: Build wheels with cibuildwheel + uses: pypa/cibuildwheel@v3.1.3 + env: + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-*' || 'cp38-* cp312-*' }} + CIBW_ENVIRONMENT: PATH=$HOME/.local/bin:$PATH + CIBW_BEFORE_ALL: | + mkdir -p $HOME/.local/bin + ./ci/run_ci.sh install_bazel + with: + package-dir: python/ + output-dir: wheelhouse/ + + - name: Install and verify wheel + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + working-directory: python + + - name: Upload wheels + if: github.ref_type == 'tag' + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-linux-${{ matrix.arch }} + path: wheelhouse/*.whl diff --git a/.github/workflows/build-native-wheels.yaml b/.github/workflows/build-native-wheels.yaml new file mode 100644 index 0000000000..3bc3679a0e --- /dev/null +++ b/.github/workflows/build-native-wheels.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native Wheels + +on: + push: + branches: [main] + tags: ["v*"] + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + pull_request: + paths: + - 'python/**' + - 'cpp/**' + - 'bazel/**' + - 'BUILD' + - 'WORKSPACE' + - '.github/workflows/build-wheels*.yaml' + +permissions: + contents: read + actions: write + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest] + steps: + - uses: actions/checkout@v5 + + - name: Install Bazel + if: ${{ !contains(matrix.os, 'windows') }} + run: ./ci/run_ci.sh install_bazel + + - name: Install Bazel for Windows + if: ${{ contains(matrix.os, 'windows') }} + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + + - name: Build wheels with cibuildwheel + uses: pypa/cibuildwheel@v3.1.3 + env: + CIBW_BUILD: ${{ github.ref_type == 'tag' && 'cp38-* cp39-* cp310-* cp311-* cp312-*' || 'cp38-* cp312-*' }} + with: + package-dir: python/ + output-dir: wheelhouse/ + + - name: Install and verify wheel + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + + - name: Upload wheels + if: github.ref_type == 'tag' + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }} + path: wheelhouse/*.whl diff --git a/.github/workflows/build-wheels-for-pr.yaml b/.github/workflows/build-wheels-for-pr.yaml deleted file mode 100644 index ec85455f88..0000000000 --- a/.github/workflows/build-wheels-for-pr.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: build wheels for pull request -on: - push: - branches: - - main - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' - pull_request: - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' -jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml - strategy: - matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] - python-version: ['3.8', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-wheels-for-release.yaml b/.github/workflows/build-wheels-for-release.yaml deleted file mode 100644 index e1799e6de5..0000000000 --- a/.github/workflows/build-wheels-for-release.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: build wheels for release -on: - push: - tags: ["v*"] - -jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml - strategy: - matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - bump-version: true diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml deleted file mode 100644 index 554f4a0065..0000000000 --- a/.github/workflows/build-wheels.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Build Wheels - -on: - workflow_call: - inputs: - os: - required: true - type: string - python-version: - required: true - type: string - bump-version: - description: 'Whether to bump the version in setup.py' - required: false - type: boolean - default: false - -permissions: - contents: read - actions: write - -jobs: - build_and_test: - name: Build and Test - runs-on: ${{ inputs.os }} - - steps: - - uses: actions/checkout@v5 - - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: Install bazel - if: "runner.os != 'Windows'" - run: ./ci/run_ci.sh install_bazel - - - name: Install bazel - if: "runner.os == 'Windows'" - run: ./ci/run_ci.sh install_bazel_windows - shell: bash - - - name: Update version in setup.py - if: "inputs.bump-version" - run: ./ci/deploy.sh bump_py_version - - - name: Build a binary wheel (Linux, manylinux) - if: "runner.os == 'Linux'" - env: - manylinux_x86_64_image: ${{ env.manylinux_x86_64_image }} - manylinux_aarch64_image: ${{ env.manylinux_aarch64_image }} - GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - ./ci/build_manylinux_wheel.sh --os "${{ runner.os }}" \ - --arch "${{ runner.arch }}" \ - --python "${{ inputs.python-version }}" \ - --workspace "${GITHUB_WORKSPACE}" - - - name: Build a binary wheel (native) - if: "runner.os != 'Linux'" - run: ./ci/deploy.sh build_pyfory - shell: bash - - - name: Install and verify wheel - shell: bash - run: | - python -m pip install --upgrade pip - pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - - name: Upload wheel -# if: ${{ inputs.bump-version }} - uses: actions/upload-artifact@v4 - with: - name: pyfory-wheels-${{ inputs.os }}-${{ inputs.python-version }}${{ inputs.bump-version && '-tagged' || github.sha }} - path: dist/*.whl diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 938ef2e9a4..52701a6149 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -19,7 +19,7 @@ name: Publish Python on: workflow_run: - workflows: ["build wheels for release"] + workflows: ["Build Native Wheels", "Build Containerized, Wheels"] types: [completed] permissions: diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 0c4fb52f64..5b2abab262 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -90,15 +90,15 @@ install_bazel() { esac BAZEL_VERSION=$(get_bazel_version) - BAZEL_DIR="/usr/local/bin" + BAZEL_DIR="$HOME/.local/bin" # Construct platform-specific URL BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}" echo "Downloading bazel from: $BINARY_URL" - sudo wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } + curl -fsSL -o "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } - sudo chmod +x "$BAZEL_DIR/bazel" + chmod +x "$BAZEL_DIR/bazel" # Add to current shell's PATH export PATH="$BAZEL_DIR:$PATH" @@ -108,7 +108,7 @@ install_bazel() { bazel version || { echo "Bazel installation verification failed"; exit 1; } # Configure number of jobs based on memory - if [[ "$MACHINE" == linux ]]; then + if [[ "$OS" == linux ]]; then MEM=$(grep MemTotal < /proc/meminfo | awk '{print $2}') JOBS=$(( MEM / 1024 / 1024 / 3 )) echo "build --jobs=$JOBS" >> ~/.bazelrc diff --git a/python/pyproject.toml b/python/pyproject.toml index 58b81674e3..54098724bd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", ] keywords = ["fory", "serialization", "multi-language", "fast", "row-format", "jit", "codegen", "polymorphic", "zero-copy"] @@ -64,7 +65,7 @@ include-package-data = true zip-safe = false [tool.setuptools.package-data] -"pyfory" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] +"pyfory" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd", ] "pyfory.format" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] "pyfory.lib.mmh3" = ["**/*.pxd", "**/*.pyx", "**/*.pxd", "*.so", "*.dylib", "*.dll", "*.pyd"] @@ -97,3 +98,17 @@ ignore = [ [tool.ruff.format] # Replicates the `inline-quotes` setting from .flake8 quote-style = "double" + +[tool.cibuildwheel] +build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-*" +skip = "*-win32" +manylinux-x86_64-image = "manylinux_2_24" +manylinux-aarch64-image = "manylinux_2_24" + +[tool.cibuildwheel.macos] +environment = {MACOSX_DEPLOYMENT_TARGET="15"} + +#[tool.cibuildwheel.linux] +#archs = ["x86_64", "aarch64"] +#build-selector = "manylinux_2_17_*" +#repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"