diff --git a/.github/workflows/build-wheels-for-release.yaml b/.github/workflows/build-containerized-pr.yml similarity index 67% rename from .github/workflows/build-wheels-for-release.yaml rename to .github/workflows/build-containerized-pr.yml index e1799e6de5..5d144bc60b 100644 --- a/.github/workflows/build-wheels-for-release.yaml +++ b/.github/workflows/build-containerized-pr.yml @@ -15,19 +15,21 @@ # specific language governing permissions and limitations # under the License. -name: build wheels for release +name: Build Containerized PR Wheels on: + pull_request: + paths: [ci/**, python/**, .github/workflows/**] push: - tags: ["v*"] + branches: [main] + paths: [ci/**, python/**, .github/workflows/**] jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml + build: + runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} - bump-version: true + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v5 + - name: Build and test wheels + run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} diff --git a/.github/workflows/build-wheels-for-pr.yaml b/.github/workflows/build-containerized-release.yml similarity index 54% rename from .github/workflows/build-wheels-for-pr.yaml rename to .github/workflows/build-containerized-release.yml index ec85455f88..7551ec047b 100644 --- a/.github/workflows/build-wheels-for-pr.yaml +++ b/.github/workflows/build-containerized-release.yml @@ -15,33 +15,27 @@ # specific language governing permissions and limitations # under the License. -name: build wheels for pull request +name: Build Containerized Release Wheels on: push: - branches: - - main - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' - pull_request: - paths: - - 'python/**' - - 'cpp/**' - - 'bazel/**' - - 'BUILD' - - 'WORKSPACE' - - '.github/workflows/build-wheels*.yml' + tags: ['v*'] # NO PATH FILTER - critical for releases + jobs: - build-wheels: - uses: ./.github/workflows/build-wheels.yaml + build: + runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] - python-version: ['3.8', '3.13'] - with: - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v5 + - name: Bump version + run: ./ci/deploy.sh bump_py_version + - name: Install bazel + run: ./ci/run_ci.sh install_bazel + - name: Build and test wheels + run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }} + path: dist/*.whl diff --git a/.github/workflows/build-native-pr.yml b/.github/workflows/build-native-pr.yml new file mode 100644 index 0000000000..3c0d2c06c0 --- /dev/null +++ b/.github/workflows/build-native-pr.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native PR Wheels +on: + pull_request: + paths: [ci/**, python/**, .github/workflows/**] + push: + branches: [main] + paths: [ci/**, python/**, .github/workflows/**] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.8', '3.13'] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install bazel + if: runner.os != 'Windows' + run: ./ci/run_ci.sh install_bazel + - name: Install bazel (Windows) + if: runner.os == 'Windows' + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + - name: Build wheel + run: ./ci/deploy.sh build_pyfory + shell: bash + - name: Install and verify wheel + shell: bash + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml new file mode 100644 index 0000000000..f6c43662cf --- /dev/null +++ b/.github/workflows/build-native-release.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native Release Wheels +on: + push: + tags: ['v*'] # NO PATH FILTER - critical for releases + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + steps: + - uses: actions/checkout@v5 + - name: Bump version + run: ./ci/deploy.sh bump_py_version + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install bazel + if: runner.os != 'Windows' + run: ./ci/run_ci.sh install_bazel + - name: Install bazel (Windows) + if: runner.os == 'Windows' + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + - name: Build wheel + run: ./ci/deploy.sh build_pyfory + - name: Install and verify wheel + shell: bash + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }} + path: dist/*.whl diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml deleted file mode 100644 index 554f4a0065..0000000000 --- a/.github/workflows/build-wheels.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Build Wheels - -on: - workflow_call: - inputs: - os: - required: true - type: string - python-version: - required: true - type: string - bump-version: - description: 'Whether to bump the version in setup.py' - required: false - type: boolean - default: false - -permissions: - contents: read - actions: write - -jobs: - build_and_test: - name: Build and Test - runs-on: ${{ inputs.os }} - - steps: - - uses: actions/checkout@v5 - - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - - name: Install bazel - if: "runner.os != 'Windows'" - run: ./ci/run_ci.sh install_bazel - - - name: Install bazel - if: "runner.os == 'Windows'" - run: ./ci/run_ci.sh install_bazel_windows - shell: bash - - - name: Update version in setup.py - if: "inputs.bump-version" - run: ./ci/deploy.sh bump_py_version - - - name: Build a binary wheel (Linux, manylinux) - if: "runner.os == 'Linux'" - env: - manylinux_x86_64_image: ${{ env.manylinux_x86_64_image }} - manylinux_aarch64_image: ${{ env.manylinux_aarch64_image }} - GITHUB_WORKSPACE: ${{ github.workspace }} - run: | - ./ci/build_manylinux_wheel.sh --os "${{ runner.os }}" \ - --arch "${{ runner.arch }}" \ - --python "${{ inputs.python-version }}" \ - --workspace "${GITHUB_WORKSPACE}" - - - name: Build a binary wheel (native) - if: "runner.os != 'Linux'" - run: ./ci/deploy.sh build_pyfory - shell: bash - - - name: Install and verify wheel - shell: bash - run: | - python -m pip install --upgrade pip - pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - - name: Upload wheel -# if: ${{ inputs.bump-version }} - uses: actions/upload-artifact@v4 - with: - name: pyfory-wheels-${{ inputs.os }}-${{ inputs.python-version }}${{ inputs.bump-version && '-tagged' || github.sha }} - path: dist/*.whl diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 938ef2e9a4..c66d65e705 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -19,7 +19,7 @@ name: Publish Python on: workflow_run: - workflows: ["build wheels for release"] + workflows: ["Build Containerized Release Wheels", "Build Native Release Wheels"] types: [completed] permissions: diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py new file mode 100755 index 0000000000..02c1754e67 --- /dev/null +++ b/ci/build_linux_wheels.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Host-side wrapper: workflow provides only --arch. +Images are defined as regular Python lists (no env vars). + +Environment: + - GITHUB_WORKSPACE (optional; defaults to cwd) +""" +from __future__ import annotations +import argparse +import os +import shlex +import subprocess +import sys +from typing import List + +SCRIPT = r'''set -e +yum install -y git sudo wget || true + +git config --global --add safe.directory /work + +# Determine Python versions to test +if [ "$RELEASE" = "1" ]; then + PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" +else + PYTHON_VERSIONS="cp38-cp38 cp313-cp313" +fi + +ci/run_ci.sh install_bazel +export PATH="$HOME/.local/bin:$PATH" + +# use the python interpreters preinstalled in manylinux +OLD_PATH=$PATH +for PY in $PYTHON_VERSIONS; do + export PYTHON_PATH="/opt/python/$PY/bin/python" + export PATH="/opt/python/$PY/bin:$OLD_PATH" + echo "Using $PYTHON_PATH" + python -m pip install cython wheel pytest + ci/deploy.sh build_pyfory + + latest_wheel=$(ls -t dist/*.whl | head -n1) + echo "Attempting to install $latest_wheel" + python -m pip install "$latest_wheel" + python -c "import pyfory; print(pyfory.__version__)" + + bazel clean --expunge +done +export PATH=$OLD_PATH +''' + +DEFAULT_X86_IMAGES = [ + "quay.io/pypa/manylinux2014_x86_64:latest", + # "quay.io/pypa/manylinux_2_28_x86_64:latest", + + # bazel binaries do not work with musl + # "quay.io/pypa/musllinux_1_2_x86_64:latest", +] + +DEFAULT_AARCH64_IMAGES = [ + "quay.io/pypa/manylinux2014_aarch64:latest", + # "quay.io/pypa/manylinux_2_28_aarch64:latest", + + # bazel binaries do not work with musl + # "quay.io/pypa/musllinux_1_2_aarch64:latest", +] + +ARCH_ALIASES = { + "X86": "x86", + "X64": "x86", + "X86_64": "x86", + "AMD64": "x86", + "ARM": "arm64", + "ARM64": "arm64", + "AARCH64": "arm64", +} + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--arch", required=True, help="Architecture (e.g. X86, X64, AARCH64)") + p.add_argument("--release", action="store_true", help="Run full test suite for release") + p.add_argument("--dry-run", action="store_true", help="Print docker commands without running") + return p.parse_args() + +def normalize_arch(raw: str) -> str: + key = raw.strip().upper() + return ARCH_ALIASES.get(key, raw.strip().lower()) + +def collect_images_for_arch(arch_normalized: str) -> List[str]: + if arch_normalized == "x86": + imgs = DEFAULT_X86_IMAGES # dedupe preserving order + elif arch_normalized == "arm64": + imgs = DEFAULT_AARCH64_IMAGES + else: + raise SystemExit(f"Unsupported arch: {arch_normalized!r}") + return imgs + +def build_docker_cmd(workspace: str, image: str) -> List[str]: + workspace = os.path.abspath(workspace) + return [ + "docker", "run", "-i", "--rm", + "-v", f"{workspace}:/work", + "-w", "/work", + image, + "bash", "-s", "--" + ] + +def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int: + rc_overall = 0 + for image in images: + docker_cmd = build_docker_cmd(workspace, image) + printable = " ".join(shlex.quote(c) for c in docker_cmd) + print(f"+ {printable}") + if dry_run: + continue + try: + completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8")) + if completed.returncode != 0: + print(f"Container {image} exited with {completed.returncode}", file=sys.stderr) + rc_overall = completed.returncode if rc_overall == 0 else rc_overall + else: + print(f"Container {image} completed successfully.") + except KeyboardInterrupt: + print("Interrupted by user", file=sys.stderr) + return 130 + except FileNotFoundError as e: + print(f"Error running docker: {e}", file=sys.stderr) + return 2 + return rc_overall + +def main() -> int: + args = parse_args() + arch = normalize_arch(args.arch) + images = collect_images_for_arch(arch) + if not images: + print(f"No images configured for arch {arch}", file=sys.stderr) + return 2 + workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd()) + print(f"Selected images for arch {args.arch}: {images}") + return run_for_images(images, workspace, args.dry_run) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build_manylinux_wheel.sh b/ci/build_manylinux_wheel.sh deleted file mode 100755 index 58c8347ddf..0000000000 --- a/ci/build_manylinux_wheel.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Usage: -# ./build_manylinux_wheel.sh --os --python \ -# --arch [--workspace ] [--x86-image ] \ -# [--aarch64-image ] [--docker-image ] [--dry-run] -# -# Examples: -# ./build_manylinux_wheel.sh --os ubuntu-latest --python 3.10 --arch X64 -# ./build_manylinux_wheel.sh --os ubuntu-24.04-arm --python 3.11 --arch ARM64 \ -# --aarch64-image quay.io/pypa/manylinux_2014_aarch64:latest -# -# Notes: -# --arch accepts values: X86, X64, ARM, or ARM64 (case-insensitive). -# This script requires --arch to be provided explicitly. -set -euo pipefail - -print_usage() { - cat < --python --arch [options] - -Required: - --os matrix.os value (e.g. ubuntu-latest or ubuntu-24.04-arm) - --python Python version (e.g. 3.10) - --arch Architecture (X86, X64, ARM, or ARM64) - -Optional: - --workspace Path to workspace to mount into container (default: cwd) - --x86-image manylinux x86_64 docker image (overrides default env) - --aarch64-image manylinux aarch64 docker image (overrides default env) - --docker-image Explicit docker image to use (skips auto selection) - --dry-run Print the docker command without executing it - -h, --help Show this help -EOF -} - -# Defaults - can be overridden by options -WORKSPACE="${GITHUB_WORKSPACE:-$(pwd)}" -MANYLINUX_X86_64_IMAGE="${MANYLINUX_X86_64_IMAGE:-quay.io/pypa/manylinux_2_28_x86_64:latest}" -MANYLINUX_AARCH64_IMAGE="${MANYLINUX_AARCH64_IMAGE:-quay.io/pypa/manylinux_2_28_aarch64:latest}" -DOCKER_IMAGE="" -ARCH="" -DRY_RUN=0 - -# Parse args -while [[ $# -gt 0 ]]; do - case "$1" in - --os) MATRIX_OS="$2"; shift 2;; - --python) PY_VERSION_RAW="$2"; shift 2;; - --workspace) WORKSPACE="$2"; shift 2;; - --x86-image) MANYLINUX_X86_64_IMAGE="$2"; shift 2;; - --aarch64-image) MANYLINUX_AARCH64_IMAGE="$2"; shift 2;; - --docker-image) DOCKER_IMAGE="$2"; shift 2;; - --arch) ARCH="$2"; shift 2;; - --dry-run) DRY_RUN=1; shift;; - -h|--help) print_usage; exit 0;; - *) echo "Unknown argument: $1"; print_usage; exit 2;; - esac -done - -if [[ -z "${MATRIX_OS:-}" ]] || [[ -z "${PY_VERSION_RAW:-}" ]] || [[ -z "${ARCH:-}" ]]; then - echo "Error: --os, --python and --arch are required." - print_usage - exit 2 -fi - -# Normalize ARCH to uppercase -ARCH="${ARCH^^}" - -# Normalize Python version: remove dots (e.g. 3.10 -> 310) -PY_VERSION_NO_DOTS="${PY_VERSION_RAW//./}" - -# Determine DOCKER_IMAGE and PLAT strictly from ARCH (unless --docker-image supplied) -PLAT="" -case "$ARCH" in - X86|X64) - PLAT="manylinux_2_28_x86_64" - DOCKER_IMAGE="${DOCKER_IMAGE:-$MANYLINUX_X86_64_IMAGE}" - ;; - ARM|ARM64) - PLAT="manylinux_2_28_aarch64" - DOCKER_IMAGE="${DOCKER_IMAGE:-$MANYLINUX_AARCH64_IMAGE}" - ;; - *) - echo "Error: Unsupported ARCH '$ARCH'. Use one of: X86, X64, ARM, ARM64." - exit 2 - ;; -esac - -echo "Matrix OS: $MATRIX_OS" -echo "Arch (input): $ARCH" -echo "Selected docker image: $DOCKER_IMAGE" -echo "Platform (PLAT): $PLAT" -echo "Python version (raw): $PY_VERSION_RAW" -echo "PY_VERSION without dots: $PY_VERSION_NO_DOTS" -echo "Workspace: $WORKSPACE" - -# Basic checks -if ! command -v docker >/dev/null 2>&1; then - echo "Error: docker is required but not installed or not on PATH." - exit 3 -fi - -SCRIPT='set -e -yum install -y git sudo wget || true -git config --global --add safe.directory /work -ls -alh /opt/python || true -echo "PY_VERSION: $PY_VERSION" -ls /opt/python/cp${PY_VERSION}-cp${PY_VERSION} || true -ls /opt/python/cp${PY_VERSION}-cp${PY_VERSION}/bin || true -export PATH=/opt/python/cp${PY_VERSION}-cp${PY_VERSION}/bin:$PATH -echo "PATH: $PATH" -echo "Using Python from: $(which python || echo not-found)" -echo "Python version: $(python -V 2>&1 || true)" -bash ci/run_ci.sh install_bazel -bash ci/deploy.sh build_pyfory' - -DOCKER_CMD=(docker run --rm - -e "PY_VERSION=$PY_VERSION_NO_DOTS" - -e "PLAT=$PLAT" - -v "$WORKSPACE":/work - -w /work - "$DOCKER_IMAGE" - bash -lc "$SCRIPT" -) - -# Show the final command (joined) for clarity -echo -echo "Docker command to be executed:" -printf ' %q' "${DOCKER_CMD[@]}" -echo -echo - -if [[ $DRY_RUN -eq 1 ]]; then - echo "Dry run enabled; not executing docker command." - exit 0 -fi - -# Execute -"${DOCKER_CMD[@]}" diff --git a/ci/deploy.sh b/ci/deploy.sh index 9a71fceb9d..58648a3fe8 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -18,16 +18,20 @@ # under the License. +# Print commands and their arguments as they are executed. set -x # Cause the script to exit if a single command fails. set -e -# configure ~/.pypirc before run this script -#if [ ! -f ~/.pypirc ]; then -# echo "Please configure .pypirc before run this script" -# exit 1 -#fi +# Prefer Python from $PYTHON_PATH if it exists, otherwise use default python +if [ -n "$PYTHON_PATH" ] && [ -x "$PYTHON_PATH" ]; then + PYTHON_CMD="$PYTHON_PATH" + PIP_CMD="$PYTHON_PATH -m pip" +else + PYTHON_CMD="python" + PIP_CMD="pip" +fi ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" @@ -63,34 +67,32 @@ deploy_jars() { } build_pyfory() { - echo "Python version $(python -V), path $(which python)" + echo "$($PYTHON_CMD -V), path $(which "$PYTHON_CMD")" install_pyarrow - pip install Cython wheel pytest auditwheel + $PIP_CMD install cython wheel pytest pushd "$ROOT/python" - pip list + $PIP_CMD list echo "Install pyfory" # Fix strange installed deps not found - pip install setuptools -U + $PIP_CMD install setuptools -U if [[ "$OSTYPE" == "darwin"* ]]; then MACOS_VERSION=$(sw_vers -productVersion | cut -d. -f1-2) echo "MACOS_VERSION: $MACOS_VERSION" if [[ "$MACOS_VERSION" == "13"* ]]; then export MACOSX_DEPLOYMENT_TARGET=10.13 - python setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist else - python setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist fi else - python setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist fi - ls -l ../dist - if [ -n "$PLAT" ]; then # In manylinux container, repair the wheel to embed shared libraries # and rename the wheel with the manylinux tag. - PYARROW_LIB_DIR=$(python -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') + PYARROW_LIB_DIR=$($PYTHON_CMD -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH" auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w ../dist/ rm ../dist/pyfory-*-linux_*.whl @@ -99,17 +101,19 @@ build_pyfory() { elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then echo "Skip windows wheel repair" fi + + echo "Wheels for $PYTHON_CMD:" ls -l ../dist popd } install_pyarrow() { - pyversion=$(python -V | cut -d' ' -f2) + pyversion=$($PYTHON_CMD -V | cut -d' ' -f2) if [[ $pyversion == 3.13* ]]; then - pip install pyarrow==18.0.0 - pip install numpy + $PIP_CMD install pyarrow==18.0.0 + $PIP_CMD install numpy else - pip install pyarrow==15.0.0 + $PIP_CMD install pyarrow==15.0.0 # Automatically install numpy fi } diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 0c4fb52f64..63a1330d14 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -44,12 +44,11 @@ export FORY_CI=true install_python() { wget -q https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -O Miniconda3.sh bash Miniconda3.sh -b -p $HOME/miniconda && rm -f miniconda.* - which python - echo "Python version $(python -V), path $(which python)" + echo "$(python -V), path $(which python)" } install_pyfory() { - echo "Python version $(python -V), path $(which python)" + echo "$(python -V), path $(which python)" "$ROOT"/ci/deploy.sh install_pyarrow pip install Cython wheel pytest pushd "$ROOT/python" @@ -90,15 +89,15 @@ install_bazel() { esac BAZEL_VERSION=$(get_bazel_version) - BAZEL_DIR="/usr/local/bin" + BAZEL_DIR="$HOME/.local/bin" + mkdir -p "$BAZEL_DIR" # Construct platform-specific URL BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}" echo "Downloading bazel from: $BINARY_URL" - sudo wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } - - sudo chmod +x "$BAZEL_DIR/bazel" + curl -L -sSf -o "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } + chmod +x "$BAZEL_DIR/bazel" # Add to current shell's PATH export PATH="$BAZEL_DIR:$PATH" @@ -108,7 +107,7 @@ install_bazel() { bazel version || { echo "Bazel installation verification failed"; exit 1; } # Configure number of jobs based on memory - if [[ "$MACHINE" == linux ]]; then + if [[ "$OS" == linux ]]; then MEM=$(grep MemTotal < /proc/meminfo | awk '{print $2}') JOBS=$(( MEM / 1024 / 1024 / 3 )) echo "build --jobs=$JOBS" >> ~/.bazelrc diff --git a/ci/tasks/python.py b/ci/tasks/python.py index 252d238d50..ea53e94332 100644 --- a/ci/tasks/python.py +++ b/ci/tasks/python.py @@ -25,7 +25,7 @@ def install_pyfory(): logging.info("Installing pyfory package") python_version = common.exec_cmd("python -V") python_path = common.exec_cmd("which python") - logging.info(f"Python version {python_version}, path {python_path}") + logging.info(f"{python_version}, path {python_path}") # Install PyArrow common.exec_cmd(f"{common.PROJECT_ROOT_DIR}/ci/deploy.sh install_pyarrow") diff --git a/python/setup.py b/python/setup.py index 9c488e10e3..87123efa8d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,7 +30,7 @@ os.environ["CFLAGS"] = "-O0" BAZEL_BUILD_EXT = False -print(f"DEBUG = {DEBUG}, BAZEL_BUILD_EXT = {BAZEL_BUILD_EXT}") +print(f"DEBUG = {DEBUG}, BAZEL_BUILD_EXT = {BAZEL_BUILD_EXT}, PATH = {os.environ.get('PATH')}") setup_dir = abspath(os.path.dirname(__file__)) project_dir = abspath(pjoin(setup_dir, os.pardir)) @@ -39,7 +39,6 @@ print(f"setup_dir: {setup_dir}") print(f"fory_cpp_src_dir: {fory_cpp_src_dir}") - class BinaryDistribution(Distribution): def __init__(self, attrs=None): super().__init__(attrs=attrs) @@ -58,6 +57,4 @@ def has_ext_modules(self): if __name__ == "__main__": - setup( - distclass=BinaryDistribution, - ) + setup(distclass=BinaryDistribution)