From f412c247b2714a5497261b7482f95d3d9bade5c9 Mon Sep 17 00:00:00 2001 From: Asnowww <23301095@bjtu.edu.cn> Date: Sun, 24 Aug 2025 16:12:07 +0800 Subject: [PATCH 01/14] chore: translate Chinese comments into English (#2503) ## What does this PR do? chore: translate Chinese comments into English ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- java/benchmark/src/main/proto/bench.proto | 2 +- .../java/org/apache/fory/codegen/Expression.java | 3 +-- javascript/packages/fory/lib/meta/MetaString.ts | 14 +++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/java/benchmark/src/main/proto/bench.proto b/java/benchmark/src/main/proto/bench.proto index e10e13689f..0e1ec75979 100644 --- a/java/benchmark/src/main/proto/bench.proto +++ b/java/benchmark/src/main/proto/bench.proto @@ -37,7 +37,7 @@ message Bar { optional int64 f6 = 6; optional float f7 = 7; optional double f8 = 8; - repeated int32 f9 = 9; // proto不支持int16 + repeated int32 f9 = 9; // proto does not support int16 repeated int64 f10 = 10; } diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 0d9e236e22..f5135bcb67 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2432,8 +2432,7 @@ public ExprCode doGenCode(CodegenContext ctx) { action.apply( new Reference(i), new Reference(leftElemValue, leftElemType, true), - // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is - // false. + // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is false. new Reference(rightElemValue, rightElemType, false)); ExprCode elementExprCode = elemExpr.genCode(ctx); diff --git a/javascript/packages/fory/lib/meta/MetaString.ts b/javascript/packages/fory/lib/meta/MetaString.ts index 92453d2169..f7678d369f 100644 --- a/javascript/packages/fory/lib/meta/MetaString.ts +++ b/javascript/packages/fory/lib/meta/MetaString.ts @@ -201,7 +201,7 @@ export class MetaStringDecoder { capitalize(str: string) { if (typeof str !== "string" || str.length === 0) { - return str; // 如果不是字符串或为空,直接返回原值 + return str; // If not a string or empty, return the original value } return str.charAt(0).toUpperCase() + str.slice(1); } @@ -366,22 +366,22 @@ export class MetaStringEncoder { } isUpperCase(str: string) { - // 检查字符串是否为空 + // Check whether the string is empty if (typeof str !== "string" || str.length === 0) { - return false; // 如果不是字符串或为空,返回 false + return false; // If not a string or empty, return false } - // 使用正则表达式检查是否所有字母字符都是大写的 + // Use a regular expression to check whether all alphabetic characters are uppercase return /^[^a-z]*$/.test(str); } isDigit(str: string) { - // 检查字符串是否为空 + // Check whether the string is empty if (typeof str !== "string" || str.length === 0) { - return false; // 如果不是字符串或为空,返回 false + return false; // If not a string or empty, return false } - // 使用正则表达式检查是否所有字母字符都是大写的 + // Use a regular expression to check whether all alphabetic characters are uppercase return /^[^0-9]*$/.test(str); } From a278b7e6acb7dbd6014c3392ebd465c17f419040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 24 Aug 2025 10:17:06 -0400 Subject: [PATCH 02/14] fix(ci): Use $ROOT/dist for wheel distribution (#2506) ## Why The python wheels were not being found. ## What does this PR do? * Update deploy.sh to use the $ROOT/dist path for storing wheels. * This change ensures consistent wheel distribution across different platforms. * Fixes an issue where wheels were being placed in relative ../dist directories. ## Related issues Fixes #2505 --- ci/deploy.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 58648a3fe8..cb966b57fd 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -81,12 +81,12 @@ build_pyfory() { echo "MACOS_VERSION: $MACOS_VERSION" if [[ "$MACOS_VERSION" == "13"* ]]; then export MACOSX_DEPLOYMENT_TARGET=10.13 - $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir="$ROOT/dist" else - $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi else - $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi if [ -n "$PLAT" ]; then @@ -94,8 +94,8 @@ build_pyfory() { # and rename the wheel with the manylinux tag. PYARROW_LIB_DIR=$($PYTHON_CMD -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH" - auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w ../dist/ - rm ../dist/pyfory-*-linux_*.whl + auditwheel repair "$ROOT/dist"/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w "$ROOT/dist" + rm "$ROOT/dist"/pyfory-*-linux_*.whl elif [[ "$OSTYPE" == "darwin"* ]]; then echo "Skip macos wheel repair" elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then @@ -103,7 +103,7 @@ build_pyfory() { fi echo "Wheels for $PYTHON_CMD:" - ls -l ../dist + ls -l "$ROOT/dist" popd } From 8838eb0f510112c91b79d863bc83a13e0fd15b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Sun, 24 Aug 2025 23:56:54 -0400 Subject: [PATCH 03/14] ci: Fix Windows wheel creation (#2511) * Wheels were not created on Windows because it does not use bash by default * Once that works, downloading them will fail because `actions/download-artifact` requires the `run-id` to access artifacts from different workflows. * Add `actions: read` permission to release workflow. * Configure `download-artifact` action to use `run-id` and `pattern` for downloading wheels from a previous workflow run. * Enable `merge-multiple` for downloading artifacts. * Add `shell: bash` to build step in native release workflow. --- .github/workflows/build-native-release.yml | 1 + .github/workflows/release-python.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index f6c43662cf..d6f013b964 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -42,6 +42,7 @@ jobs: run: ./ci/run_ci.sh install_bazel_windows shell: bash - name: Build wheel + shell: bash run: ./ci/deploy.sh build_pyfory - name: Install and verify wheel shell: bash diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index c66d65e705..435b35c7e3 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -25,6 +25,7 @@ on: permissions: contents: read id-token: write + actions: read # for accessing workflow run artifacts jobs: publish-wheels: @@ -35,7 +36,10 @@ jobs: - name: Download all wheel artifacts uses: actions/download-artifact@v5 with: + run-id: ${{ github.event.workflow_run.id }} + pattern: "pyfory-wheels-*" path: downloaded_wheels + merge-multiple: true - name: Move wheels to a single directory shell: bash From 03452a3653424c023fbb593a75f0a40f6d16386f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 25 Aug 2025 01:07:55 -0400 Subject: [PATCH 04/14] ci: Pass the GH token to actions/download-artifact (#2514) ## Why? `actions/download-artifact` is [failing to find artifacts](https://github.com/apache/fory/actions/runs/17198889531/job/48785824375) despite [artifacts being created](https://github.com/apache/fory/actions/runs/17198794761). ## What does this PR do? Pass the github token so the action can find artifacts from other workflow runs (ours are separate). --- .github/workflows/release-python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 435b35c7e3..02474e68de 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -37,6 +37,7 @@ jobs: uses: actions/download-artifact@v5 with: run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} pattern: "pyfory-wheels-*" path: downloaded_wheels merge-multiple: true From abc70f24562d6a457708d988a79f06a1016685ab Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 25 Aug 2025 15:17:26 +0800 Subject: [PATCH 05/14] fix(python): fix publich tag for workflow (#2517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? In a workflow_run event GitHub gives the Publish workflow a brand-new run context. The variables in that context describe this run, not the upstream one. Unless you push the tag again while the Publish workflow is running, the new context will not contain that tag: github.ref → refs/heads/main (or whatever the default branch is) github.ref_name → main github.ref_type → branch ## What does this PR do? Use the payload that GitHub attaches to the workflow_run event: github.event.workflow_run.head_branch # the tag (or branch) that the upstream workflow ran for github.event.workflow_run.head_sha # the commit SHA These are the values that correspond to the tag push. ## Related issues Closes #2515 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/release-python.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 02474e68de..82ead349af 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -51,7 +51,7 @@ jobs: - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') + if: startsWith(github.event.workflow_run.head_branch, 'refs/tags/') && contains(github.event.workflow_run.head_branch, '-') with: repository-url: https://test.pypi.org/legacy/ skip-existing: true @@ -61,7 +61,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') + if: startsWith(github.event.workflow_run.head_branch, 'refs/tags/') && !contains(github.event.workflow_run.head_branch, '-') with: skip-existing: true verify-metadata: false From 837789dd8311bf71e37f58822c730c395389c7a7 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 25 Aug 2025 15:43:39 +0800 Subject: [PATCH 06/14] fix(ci): print workflow head branch (#2518) ## Why? ## What does this PR do? ## Related issues #2515 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/release-python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 82ead349af..209278436c 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -48,6 +48,7 @@ jobs: mkdir dist find downloaded_wheels -type f -name "*.whl" -exec mv {} dist/ \; ls -R dist + echo "head_branch: ${{ github.event.workflow_run.head_branch }}" - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 From 3d281b4ee6ca489d7fab25cfc13efc3bd1f7ec2d Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Mon, 25 Aug 2025 16:19:52 +0800 Subject: [PATCH 07/14] fix(python): fix release python wheel head_branch tag (#2519) ## Why? ## What does this PR do? ## Related issues Close #2515 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/release-python.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 209278436c..9062f995e2 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -52,7 +52,7 @@ jobs: - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.event.workflow_run.head_branch, 'refs/tags/') && contains(github.event.workflow_run.head_branch, '-') + if: startsWith(github.event.workflow_run.head_branch, 'v') && contains(github.event.workflow_run.head_branch, '-') with: repository-url: https://test.pypi.org/legacy/ skip-existing: true @@ -62,7 +62,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.event.workflow_run.head_branch, 'refs/tags/') && !contains(github.event.workflow_run.head_branch, '-') + if: startsWith(github.event.workflow_run.head_branch, 'v') && !contains(github.event.workflow_run.head_branch, '-') with: skip-existing: true verify-metadata: false From 3efc9fc65d2d46c81f9afb9f79790e871f8ceca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 25 Aug 2025 22:35:37 -0400 Subject: [PATCH 08/14] fix(ci): Pass tag to bump_py_version, set PLAT inside manylinux container (#2522) ## Why? * TestPyPi was running in the release workflow, making it difficult to test. * The wheel platform tag was rejected #2520 * The wheel version did not match the tag #2521 ## What does this PR do? * Remove TestPyPI publishing from release-python.yaml. * Pass tag name to bump_py_version script in build-native-release.yml and build-containerized-release.yml. * Install auditwheel in build_linux_wheels.py. * Add explicit PLAT variable for aarch64 in build_linux_wheels.py. * Add upload-artifact and publish-to-testpypi jobs to build-containerized-pr.yml. * Add upload-artifact and publish-to-testpypi jobs to build-native-pr.yml. * Add permissions for contents and id-token to build-containerized-pr.yml and build-native-pr.yml. ## Related issues Closes #2520, closes #2521 --- .github/workflows/build-containerized-pr.yml | 5 +++++ .github/workflows/build-containerized-release.yml | 3 ++- .github/workflows/build-native-pr.yml | 5 +++++ .github/workflows/build-native-release.yml | 3 ++- ci/build_linux_wheels.py | 8 +++++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-containerized-pr.yml b/.github/workflows/build-containerized-pr.yml index 5d144bc60b..3f1240f1d7 100644 --- a/.github/workflows/build-containerized-pr.yml +++ b/.github/workflows/build-containerized-pr.yml @@ -33,3 +33,8 @@ jobs: - uses: actions/checkout@v5 - name: Build and test wheels run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-containerized-${{ matrix.os }} + path: dist/*.whl diff --git a/.github/workflows/build-containerized-release.yml b/.github/workflows/build-containerized-release.yml index 7551ec047b..4637c577be 100644 --- a/.github/workflows/build-containerized-release.yml +++ b/.github/workflows/build-containerized-release.yml @@ -29,7 +29,8 @@ jobs: steps: - uses: actions/checkout@v5 - name: Bump version - run: ./ci/deploy.sh bump_py_version + # Pass the tag name from the push (e.g. "v0.12.1b1"); deploy.sh will strip leading "v". + run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}" - name: Install bazel run: ./ci/run_ci.sh install_bazel - name: Build and test wheels diff --git a/.github/workflows/build-native-pr.yml b/.github/workflows/build-native-pr.yml index 3c0d2c06c0..9a5150235e 100644 --- a/.github/workflows/build-native-pr.yml +++ b/.github/workflows/build-native-pr.yml @@ -51,3 +51,8 @@ jobs: python -m pip install --upgrade pip pip install dist/*.whl python -c "import pyfory; print(pyfory.__version__)" + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-native-${{ matrix.os }}-${{ matrix.python-version }} + path: dist/*.whl diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index d6f013b964..b4e0afda98 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -30,7 +30,8 @@ jobs: steps: - uses: actions/checkout@v5 - name: Bump version - run: ./ci/deploy.sh bump_py_version + # Pass the tag name from the push so the python package version matches the tag. + run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}" - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py index 02c1754e67..4e0a67b0ef 100755 --- a/ci/build_linux_wheels.py +++ b/ci/build_linux_wheels.py @@ -53,7 +53,13 @@ export PYTHON_PATH="/opt/python/$PY/bin/python" export PATH="/opt/python/$PY/bin:$OLD_PATH" echo "Using $PYTHON_PATH" - python -m pip install cython wheel pytest + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ]; then + export PLAT="manylinux2014_aarch64" + else + export PLAT="manylinux2014_x86_64" + fi + python -m pip install cython wheel pytest auditwheel ci/deploy.sh build_pyfory latest_wheel=$(ls -t dist/*.whl | head -n1) From 132309627d0bda029284187ca4eb7c6be3641fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Wed, 27 Aug 2025 00:05:51 -0400 Subject: [PATCH 09/14] fix(ci): Use ref_name tag in Windows, name release runs, verify wheel versions, use all python versions for release (#2532) ## Why? The Python release pipeline is broken: * Windows wheels do not use the ref tag (pyfory-0.12.1-cp39-cp39-win_amd64.whl vs pyfory-0.12.1a3-cp310-cp310-macosx_15_0_universal2.whl) * Linux does not release wheels for the full set of Python versions. ## What does this PR do? * Introduce `ci/tasks/python_container_build_script.sh` for containerized wheel builds. * Update `ci/build_linux_wheels.py` to use the new container script and define Python versions. * Enhance `ci/deploy.sh` to force wheel version on Windows. * Add version verification logic to `.github/workflows/build-native-pr.yml`. * Update GitHub Actions workflows (`release-python.yaml`, `build-native-release.yml`, `build-containerized-release.yml`, `build-native-pr.yml`) for consistency and artifact naming. * Refactor `ci/build_linux_wheels.py` to pass `GITHUB_REF_NAME` to Docker for version verification. ## Related issues #2527 --- .../workflows/build-containerized-release.yml | 2 +- .github/workflows/build-native-release.yml | 19 +++- .github/workflows/release-python.yaml | 1 + ci/build_linux_wheels.py | 84 ++++++++---------- ci/deploy.sh | 33 +++++-- ci/tasks/python_container_build_script.sh | 87 +++++++++++++++++++ 6 files changed, 169 insertions(+), 57 deletions(-) create mode 100644 ci/tasks/python_container_build_script.sh diff --git a/.github/workflows/build-containerized-release.yml b/.github/workflows/build-containerized-release.yml index 4637c577be..c3d1d4afc8 100644 --- a/.github/workflows/build-containerized-release.yml +++ b/.github/workflows/build-containerized-release.yml @@ -35,7 +35,7 @@ jobs: run: ./ci/run_ci.sh install_bazel - name: Build and test wheels run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release - - name: Upload artifacts + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }} diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index b4e0afda98..7baf25f407 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -45,13 +45,28 @@ jobs: - name: Build wheel shell: bash run: ./ci/deploy.sh build_pyfory + env: + GITHUB_REF_NAME: ${{ github.ref_name }} - name: Install and verify wheel shell: bash run: | python -m pip install --upgrade pip pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - name: Upload artifacts + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + echo "Installed version: $INSTALLED_VERSION" + # Verify version matches the tag + EXPECTED_VERSION="${{ github.ref_name }}" + EXPECTED_VERSION=${EXPECTED_VERSION#v} + EXPECTED_VERSION="${EXPECTED_VERSION//-alpha/a}" + EXPECTED_VERSION="${EXPECTED_VERSION//-beta/b}" + EXPECTED_VERSION="${EXPECTED_VERSION//-rc/rc}" + echo "Expected version: $EXPECTED_VERSION" + if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then + echo "Version mismatch: Expected $EXPECTED_VERSION but got $INSTALLED_VERSION" + exit 1 + fi + echo "Version verification successful" + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }} diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 9062f995e2..eb97c05641 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -16,6 +16,7 @@ # under the License. name: Publish Python +run-name: "Python Release: ${{ contains(github.event.workflow_run.name, 'Containerized') && 'Containerized' || 'Native' }} (Run ID: ${{ github.event.workflow_run.id }})" on: workflow_run: diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py index 4e0a67b0ef..4f4d6010ae 100755 --- a/ci/build_linux_wheels.py +++ b/ci/build_linux_wheels.py @@ -32,45 +32,12 @@ import sys from typing import List -SCRIPT = r'''set -e -yum install -y git sudo wget || true - -git config --global --add safe.directory /work - -# Determine Python versions to test -if [ "$RELEASE" = "1" ]; then - PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" -else - PYTHON_VERSIONS="cp38-cp38 cp313-cp313" -fi - -ci/run_ci.sh install_bazel -export PATH="$HOME/.local/bin:$PATH" - -# use the python interpreters preinstalled in manylinux -OLD_PATH=$PATH -for PY in $PYTHON_VERSIONS; do - export PYTHON_PATH="/opt/python/$PY/bin/python" - export PATH="/opt/python/$PY/bin:$OLD_PATH" - echo "Using $PYTHON_PATH" - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ]; then - export PLAT="manylinux2014_aarch64" - else - export PLAT="manylinux2014_x86_64" - fi - python -m pip install cython wheel pytest auditwheel - ci/deploy.sh build_pyfory - - latest_wheel=$(ls -t dist/*.whl | head -n1) - echo "Attempting to install $latest_wheel" - python -m pip install "$latest_wheel" - python -c "import pyfory; print(pyfory.__version__)" - - bazel clean --expunge -done -export PATH=$OLD_PATH -''' +# Define Python version sets directly in the Python script +RELEASE_PYTHON_VERSIONS = "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" +DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313" + +# Path to the container build script +CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh" DEFAULT_X86_IMAGES = [ "quay.io/pypa/manylinux2014_x86_64:latest", @@ -118,26 +85,38 @@ def collect_images_for_arch(arch_normalized: str) -> List[str]: raise SystemExit(f"Unsupported arch: {arch_normalized!r}") return imgs -def build_docker_cmd(workspace: str, image: str) -> List[str]: +def build_docker_cmd(workspace: str, image: str, release: bool = False) -> List[str]: workspace = os.path.abspath(workspace) - return [ + python_versions = RELEASE_PYTHON_VERSIONS if release else DEFAULT_PYTHON_VERSIONS + + # Get GitHub reference name from environment + github_ref_name = os.environ.get("GITHUB_REF_NAME", "") + + cmd = [ "docker", "run", "-i", "--rm", - "-v", f"{workspace}:/work", - "-w", "/work", - image, - "bash", "-s", "--" + "-v", f"{workspace}:/work", # (v)olume + "-w", "/work", # (w)orking directory + "-e", f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables + "-e", f"RELEASE_BUILD={'1' if release else '0'}" ] -def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int: + # Pass GitHub reference name if available + if github_ref_name: + cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"]) + + cmd.extend([image, "bash", CONTAINER_SCRIPT_PATH]) + return cmd + +def run_for_images(images: List[str], workspace: str, dry_run: bool, release: bool = False) -> int: rc_overall = 0 for image in images: - docker_cmd = build_docker_cmd(workspace, image) + docker_cmd = build_docker_cmd(workspace, image, release=release) printable = " ".join(shlex.quote(c) for c in docker_cmd) print(f"+ {printable}") if dry_run: continue try: - completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8")) + completed = subprocess.run(docker_cmd) if completed.returncode != 0: print(f"Container {image} exited with {completed.returncode}", file=sys.stderr) rc_overall = completed.returncode if rc_overall == 0 else rc_overall @@ -159,8 +138,15 @@ def main() -> int: print(f"No images configured for arch {arch}", file=sys.stderr) return 2 workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd()) + + # Check if the container script exists + script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH) + if not os.path.exists(script_path): + print(f"Container script not found at {script_path}", file=sys.stderr) + return 2 + print(f"Selected images for arch {args.arch}: {images}") - return run_for_images(images, workspace, args.dry_run) + return run_for_images(images, workspace, args.dry_run, release=args.release) if __name__ == "__main__": sys.exit(main()) diff --git a/ci/deploy.sh b/ci/deploy.sh index cb966b57fd..5ec402e05a 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -44,16 +44,26 @@ bump_java_version() { python "$ROOT/ci/release.py" bump_version -l java -version "$1" } -bump_py_version() { +# Replicates the behavior of _update_python_version in ci/release.py +parse_py_version() { local version="$1" if [ -z "$version" ]; then # Get the latest tag from the current Git repository version=$(git describe --tags --abbrev=0) - # Check if the tag starts with 'v' and strip it - if [[ $version == v* ]]; then - version="${version:1}" - fi fi + # Check if the tag starts with 'v' and strip it + if [[ $version == v* ]]; then + version="${version:1}" + fi + version="${version//-alpha/a}" + version="${version//-beta/b}" + version="${version//-rc/rc}" + echo "$version" +} + +bump_py_version() { + local version + version=$(parse_py_version "$1") python "$ROOT/ci/release.py" bump_version -l python -version "$version" } @@ -85,6 +95,19 @@ build_pyfory() { else $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi + elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then + + # Windows tends to drop alpha/beta markers - force it through setup.cfg + if [ -n "$GITHUB_REF_NAME" ]; then + version=$(parse_py_version "$GITHUB_REF_NAME") + echo "Using version from GITHUB_REF_NAME: $version" + echo "[metadata]" > setup.cfg + echo "version = $version" >> setup.cfg + fi + + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" + # Clean up + rm setup.cfg else $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi diff --git a/ci/tasks/python_container_build_script.sh b/ci/tasks/python_container_build_script.sh new file mode 100644 index 0000000000..c89e961f96 --- /dev/null +++ b/ci/tasks/python_container_build_script.sh @@ -0,0 +1,87 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +yum install -y git sudo wget || true + +git config --global --add safe.directory /work + +ci/run_ci.sh install_bazel +export PATH="$HOME/.local/bin:$PATH" + +# Function to verify the installed version against expected version +verify_version() { + local installed_version=$1 + + echo "Installed version: $installed_version" + + # Check if GITHUB_REF_NAME is available and use it for verification + if [ -n "$GITHUB_REF_NAME" ]; then + # Strip leading 'v' if present + local expected_version + expected_version="$(ci/deploy.sh parse_py_version $GITHUB_REF_NAME)" + echo "Expected version: $expected_version" + + if [ "$installed_version" != "$expected_version" ]; then + echo "Version mismatch: Expected $expected_version but got $installed_version" + exit 1 + fi + echo "Version verification successful" + else + echo "GITHUB_REF_NAME not available, skipping version verification" + fi +} + +# use the python interpreters preinstalled in manylinux +OLD_PATH=$PATH +for PY in $PYTHON_VERSIONS; do + export PYTHON_PATH="/opt/python/$PY/bin/python" + export PATH="/opt/python/$PY/bin:$OLD_PATH" + echo "Using $PYTHON_PATH" + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ]; then + export PLAT="manylinux2014_aarch64" + else + export PLAT="manylinux2014_x86_64" + fi + python -m pip install cython wheel pytest auditwheel + ci/deploy.sh build_pyfory + + latest_wheel=$(find dist -maxdepth 1 -type f -name '*.whl' -print0 | xargs -0 ls -t | head -n1) + if [ -z "$latest_wheel" ]; then + echo "No wheel found" >&2 + exit 1 + fi + + echo "Attempting to install $latest_wheel" + python -m pip install "$latest_wheel" + + # Verify the installed version matches the expected version + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + + # Only run version verification for release builds + if [ "${RELEASE_BUILD:-0}" = "1" ]; then + echo "Running version verification for release build" + verify_version "$INSTALLED_VERSION" + else + echo "Skipping version verification for test build" + fi + + bazel clean --expunge +done +export PATH=$OLD_PATH From f6a7273113b558a0999ea77cc64ee5b98e5e6a00 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Wed, 27 Aug 2025 12:25:08 +0800 Subject: [PATCH 10/14] add bash for windows --- .github/workflows/build-native-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index 7baf25f407..b72c1c380f 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -32,6 +32,7 @@ jobs: - name: Bump version # Pass the tag name from the push so the python package version matches the tag. run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}" + shell: bash - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From cf11083d869eb392933420b3264d3d65b51ffdf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Wed, 27 Aug 2025 01:10:56 -0400 Subject: [PATCH 11/14] ci: Fix verify_version() by capturing version properly (#2535) ## Why? [Containerized builds were failing at the version verification step](https://github.com/apache/fory/actions/runs/17257282639/job/48971637440#step:5:1997). ## What does this PR do? * Refine script to capture only the actual version output from deploy.sh. --- ci/deploy.sh | 4 +++- ci/tasks/python_container_build_script.sh | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/deploy.sh b/ci/deploy.sh index 5ec402e05a..cb9a23cf72 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -19,7 +19,9 @@ # Print commands and their arguments as they are executed. -set -x +if [ "${DEPLOY_QUIET:-0}" != "1" ]; then + set -x +fi # Cause the script to exit if a single command fails. set -e diff --git a/ci/tasks/python_container_build_script.sh b/ci/tasks/python_container_build_script.sh index c89e961f96..da4032d6e7 100644 --- a/ci/tasks/python_container_build_script.sh +++ b/ci/tasks/python_container_build_script.sh @@ -32,9 +32,9 @@ verify_version() { # Check if GITHUB_REF_NAME is available and use it for verification if [ -n "$GITHUB_REF_NAME" ]; then - # Strip leading 'v' if present + # Strip leading 'v' if present and capture only the actual output, not debug messages local expected_version - expected_version="$(ci/deploy.sh parse_py_version $GITHUB_REF_NAME)" + expected_version="$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $GITHUB_REF_NAME)" echo "Expected version: $expected_version" if [ "$installed_version" != "$expected_version" ]; then From e676ee7a80dea2fb12416417e222151bcb2a0f7c Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Wed, 27 Aug 2025 13:14:12 +0800 Subject: [PATCH 12/14] fix(python): fix bump_wheel version (#2534) ## Why? ## What does this PR do? - Set bash for windows - Remove `-` from version ## Related issues ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- ci/deploy.sh | 1 + ci/release.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/deploy.sh b/ci/deploy.sh index cb9a23cf72..007ccb8a58 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -60,6 +60,7 @@ parse_py_version() { version="${version//-alpha/a}" version="${version//-beta/b}" version="${version//-rc/rc}" + version="${version//-//}" echo "$version" } diff --git a/ci/release.py b/ci/release.py index 5f913089e8..347cad0d85 100644 --- a/ci/release.py +++ b/ci/release.py @@ -245,6 +245,7 @@ def _update_python_version(lines, v: str): v = v.replace("-alpha", "a") v = v.replace("-beta", "b") v = v.replace("-rc", "rc") + v = v.replace("-", "") lines[index] = f'__version__ = "{v}"\n' break From adfe96c4a3a34dd5f224ac967e46eb4d88075b99 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Wed, 27 Aug 2025 15:11:23 +0800 Subject: [PATCH 13/14] fix(python): fix bump version (#2536) ## Why? https://github.com/apache/fory/actions/runs/17257989186/job/48973541484 ## What does this PR do? ## Related issues #2534 #2535 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/build-native-release.yml | 5 +---- ci/deploy.sh | 2 +- ci/tasks/python_container_build_script.sh | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index b72c1c380f..5131c0686b 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -57,10 +57,7 @@ jobs: echo "Installed version: $INSTALLED_VERSION" # Verify version matches the tag EXPECTED_VERSION="${{ github.ref_name }}" - EXPECTED_VERSION=${EXPECTED_VERSION#v} - EXPECTED_VERSION="${EXPECTED_VERSION//-alpha/a}" - EXPECTED_VERSION="${EXPECTED_VERSION//-beta/b}" - EXPECTED_VERSION="${EXPECTED_VERSION//-rc/rc}" + EXPECTED_VERSION=$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $EXPECTED_VERSION | tail -n1) echo "Expected version: $EXPECTED_VERSION" if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then echo "Version mismatch: Expected $EXPECTED_VERSION but got $INSTALLED_VERSION" diff --git a/ci/deploy.sh b/ci/deploy.sh index 007ccb8a58..be0c385956 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -60,7 +60,7 @@ parse_py_version() { version="${version//-alpha/a}" version="${version//-beta/b}" version="${version//-rc/rc}" - version="${version//-//}" + version="${version//-/}" echo "$version" } diff --git a/ci/tasks/python_container_build_script.sh b/ci/tasks/python_container_build_script.sh index da4032d6e7..6ba1f7d806 100644 --- a/ci/tasks/python_container_build_script.sh +++ b/ci/tasks/python_container_build_script.sh @@ -34,7 +34,7 @@ verify_version() { if [ -n "$GITHUB_REF_NAME" ]; then # Strip leading 'v' if present and capture only the actual output, not debug messages local expected_version - expected_version="$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $GITHUB_REF_NAME)" + expected_version="$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $GITHUB_REF_NAME | tail -n1)" echo "Expected version: $expected_version" if [ "$installed_version" != "$expected_version" ]; then From 97b8ddb807e6e9f314a60ee822cadb180101cb89 Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Wed, 27 Aug 2025 15:38:06 +0800 Subject: [PATCH 14/14] fix(python): add macos 13 release (#2538) ## Why? ## What does this PR do? ## Related issues Closes #2537 ## Does this PR introduce any user-facing change? - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark --- .github/workflows/build-native-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index 5131c0686b..2ef8248c8f 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -25,7 +25,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, windows-latest] + os: [macos-13, macos-latest, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v5