diff --git a/.github/workflows/build-containerized-pr.yml b/.github/workflows/build-containerized-pr.yml index 5d144bc60b..3f1240f1d7 100644 --- a/.github/workflows/build-containerized-pr.yml +++ b/.github/workflows/build-containerized-pr.yml @@ -33,3 +33,8 @@ jobs: - uses: actions/checkout@v5 - name: Build and test wheels run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-containerized-${{ matrix.os }} + path: dist/*.whl diff --git a/.github/workflows/build-containerized-release.yml b/.github/workflows/build-containerized-release.yml index 7551ec047b..c3d1d4afc8 100644 --- a/.github/workflows/build-containerized-release.yml +++ b/.github/workflows/build-containerized-release.yml @@ -29,12 +29,13 @@ jobs: steps: - uses: actions/checkout@v5 - name: Bump version - run: ./ci/deploy.sh bump_py_version + # Pass the tag name from the push (e.g. "v0.12.1b1"); deploy.sh will strip leading "v". + run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}" - name: Install bazel run: ./ci/run_ci.sh install_bazel - name: Build and test wheels run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release - - name: Upload artifacts + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }} diff --git a/.github/workflows/build-native-pr.yml b/.github/workflows/build-native-pr.yml index 3c0d2c06c0..9a5150235e 100644 --- a/.github/workflows/build-native-pr.yml +++ b/.github/workflows/build-native-pr.yml @@ -51,3 +51,8 @@ jobs: python -m pip install --upgrade pip pip install dist/*.whl python -c "import pyfory; print(pyfory.__version__)" + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-native-${{ matrix.os }}-${{ matrix.python-version }} + path: dist/*.whl diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index f6c43662cf..2ef8248c8f 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -25,12 +25,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, windows-latest] + os: [macos-13, macos-latest, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v5 - name: Bump version - run: ./ci/deploy.sh bump_py_version + # Pass the tag name from the push so the python package version matches the tag. + run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}" + shell: bash - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -42,14 +44,27 @@ jobs: run: ./ci/run_ci.sh install_bazel_windows shell: bash - name: Build wheel + shell: bash run: ./ci/deploy.sh build_pyfory + env: + GITHUB_REF_NAME: ${{ github.ref_name }} - name: Install and verify wheel shell: bash run: | python -m pip install --upgrade pip pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - name: Upload artifacts + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + echo "Installed version: $INSTALLED_VERSION" + # Verify version matches the tag + EXPECTED_VERSION="${{ github.ref_name }}" + EXPECTED_VERSION=$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $EXPECTED_VERSION | tail -n1) + echo "Expected version: $EXPECTED_VERSION" + if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then + echo "Version mismatch: Expected $EXPECTED_VERSION but got $INSTALLED_VERSION" + exit 1 + fi + echo "Version verification successful" + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }} diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index c66d65e705..eb97c05641 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -16,6 +16,7 @@ # under the License. name: Publish Python +run-name: "Python Release: ${{ contains(github.event.workflow_run.name, 'Containerized') && 'Containerized' || 'Native' }} (Run ID: ${{ github.event.workflow_run.id }})" on: workflow_run: @@ -25,6 +26,7 @@ on: permissions: contents: read id-token: write + actions: read # for accessing workflow run artifacts jobs: publish-wheels: @@ -35,7 +37,11 @@ jobs: - name: Download all wheel artifacts uses: actions/download-artifact@v5 with: + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: "pyfory-wheels-*" path: downloaded_wheels + merge-multiple: true - name: Move wheels to a single directory shell: bash @@ -43,10 +49,11 @@ jobs: mkdir dist find downloaded_wheels -type f -name "*.whl" -exec mv {} dist/ \; ls -R dist + echo "head_branch: ${{ github.event.workflow_run.head_branch }}" - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') + if: startsWith(github.event.workflow_run.head_branch, 'v') && contains(github.event.workflow_run.head_branch, '-') with: repository-url: https://test.pypi.org/legacy/ skip-existing: true @@ -56,7 +63,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 - if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') + if: startsWith(github.event.workflow_run.head_branch, 'v') && !contains(github.event.workflow_run.head_branch, '-') with: skip-existing: true verify-metadata: false diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py index 02c1754e67..4f4d6010ae 100755 --- a/ci/build_linux_wheels.py +++ b/ci/build_linux_wheels.py @@ -32,39 +32,12 @@ import sys from typing import List -SCRIPT = r'''set -e -yum install -y git sudo wget || true - -git config --global --add safe.directory /work - -# Determine Python versions to test -if [ "$RELEASE" = "1" ]; then - PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" -else - PYTHON_VERSIONS="cp38-cp38 cp313-cp313" -fi - -ci/run_ci.sh install_bazel -export PATH="$HOME/.local/bin:$PATH" - -# use the python interpreters preinstalled in manylinux -OLD_PATH=$PATH -for PY in $PYTHON_VERSIONS; do - export PYTHON_PATH="/opt/python/$PY/bin/python" - export PATH="/opt/python/$PY/bin:$OLD_PATH" - echo "Using $PYTHON_PATH" - python -m pip install cython wheel pytest - ci/deploy.sh build_pyfory - - latest_wheel=$(ls -t dist/*.whl | head -n1) - echo "Attempting to install $latest_wheel" - python -m pip install "$latest_wheel" - python -c "import pyfory; print(pyfory.__version__)" - - bazel clean --expunge -done -export PATH=$OLD_PATH -''' +# Define Python version sets directly in the Python script +RELEASE_PYTHON_VERSIONS = "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" +DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313" + +# Path to the container build script +CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh" DEFAULT_X86_IMAGES = [ "quay.io/pypa/manylinux2014_x86_64:latest", @@ -112,26 +85,38 @@ def collect_images_for_arch(arch_normalized: str) -> List[str]: raise SystemExit(f"Unsupported arch: {arch_normalized!r}") return imgs -def build_docker_cmd(workspace: str, image: str) -> List[str]: +def build_docker_cmd(workspace: str, image: str, release: bool = False) -> List[str]: workspace = os.path.abspath(workspace) - return [ + python_versions = RELEASE_PYTHON_VERSIONS if release else DEFAULT_PYTHON_VERSIONS + + # Get GitHub reference name from environment + github_ref_name = os.environ.get("GITHUB_REF_NAME", "") + + cmd = [ "docker", "run", "-i", "--rm", - "-v", f"{workspace}:/work", - "-w", "/work", - image, - "bash", "-s", "--" + "-v", f"{workspace}:/work", # (v)olume + "-w", "/work", # (w)orking directory + "-e", f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables + "-e", f"RELEASE_BUILD={'1' if release else '0'}" ] -def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int: + # Pass GitHub reference name if available + if github_ref_name: + cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"]) + + cmd.extend([image, "bash", CONTAINER_SCRIPT_PATH]) + return cmd + +def run_for_images(images: List[str], workspace: str, dry_run: bool, release: bool = False) -> int: rc_overall = 0 for image in images: - docker_cmd = build_docker_cmd(workspace, image) + docker_cmd = build_docker_cmd(workspace, image, release=release) printable = " ".join(shlex.quote(c) for c in docker_cmd) print(f"+ {printable}") if dry_run: continue try: - completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8")) + completed = subprocess.run(docker_cmd) if completed.returncode != 0: print(f"Container {image} exited with {completed.returncode}", file=sys.stderr) rc_overall = completed.returncode if rc_overall == 0 else rc_overall @@ -153,8 +138,15 @@ def main() -> int: print(f"No images configured for arch {arch}", file=sys.stderr) return 2 workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd()) + + # Check if the container script exists + script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH) + if not os.path.exists(script_path): + print(f"Container script not found at {script_path}", file=sys.stderr) + return 2 + print(f"Selected images for arch {args.arch}: {images}") - return run_for_images(images, workspace, args.dry_run) + return run_for_images(images, workspace, args.dry_run, release=args.release) if __name__ == "__main__": sys.exit(main()) diff --git a/ci/deploy.sh b/ci/deploy.sh index 58648a3fe8..be0c385956 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -19,7 +19,9 @@ # Print commands and their arguments as they are executed. -set -x +if [ "${DEPLOY_QUIET:-0}" != "1" ]; then + set -x +fi # Cause the script to exit if a single command fails. set -e @@ -44,16 +46,27 @@ bump_java_version() { python "$ROOT/ci/release.py" bump_version -l java -version "$1" } -bump_py_version() { +# Replicates the behavior of _update_python_version in ci/release.py +parse_py_version() { local version="$1" if [ -z "$version" ]; then # Get the latest tag from the current Git repository version=$(git describe --tags --abbrev=0) - # Check if the tag starts with 'v' and strip it - if [[ $version == v* ]]; then - version="${version:1}" - fi fi + # Check if the tag starts with 'v' and strip it + if [[ $version == v* ]]; then + version="${version:1}" + fi + version="${version//-alpha/a}" + version="${version//-beta/b}" + version="${version//-rc/rc}" + version="${version//-/}" + echo "$version" +} + +bump_py_version() { + local version + version=$(parse_py_version "$1") python "$ROOT/ci/release.py" bump_version -l python -version "$version" } @@ -81,12 +94,25 @@ build_pyfory() { echo "MACOS_VERSION: $MACOS_VERSION" if [[ "$MACOS_VERSION" == "13"* ]]; then export MACOSX_DEPLOYMENT_TARGET=10.13 - $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir="$ROOT/dist" else - $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi + elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then + + # Windows tends to drop alpha/beta markers - force it through setup.cfg + if [ -n "$GITHUB_REF_NAME" ]; then + version=$(parse_py_version "$GITHUB_REF_NAME") + echo "Using version from GITHUB_REF_NAME: $version" + echo "[metadata]" > setup.cfg + echo "version = $version" >> setup.cfg + fi + + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" + # Clean up + rm setup.cfg else - $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi if [ -n "$PLAT" ]; then @@ -94,8 +120,8 @@ build_pyfory() { # and rename the wheel with the manylinux tag. PYARROW_LIB_DIR=$($PYTHON_CMD -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH" - auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w ../dist/ - rm ../dist/pyfory-*-linux_*.whl + auditwheel repair "$ROOT/dist"/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w "$ROOT/dist" + rm "$ROOT/dist"/pyfory-*-linux_*.whl elif [[ "$OSTYPE" == "darwin"* ]]; then echo "Skip macos wheel repair" elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then @@ -103,7 +129,7 @@ build_pyfory() { fi echo "Wheels for $PYTHON_CMD:" - ls -l ../dist + ls -l "$ROOT/dist" popd } diff --git a/ci/release.py b/ci/release.py index 5f913089e8..347cad0d85 100644 --- a/ci/release.py +++ b/ci/release.py @@ -245,6 +245,7 @@ def _update_python_version(lines, v: str): v = v.replace("-alpha", "a") v = v.replace("-beta", "b") v = v.replace("-rc", "rc") + v = v.replace("-", "") lines[index] = f'__version__ = "{v}"\n' break diff --git a/ci/tasks/python_container_build_script.sh b/ci/tasks/python_container_build_script.sh new file mode 100644 index 0000000000..6ba1f7d806 --- /dev/null +++ b/ci/tasks/python_container_build_script.sh @@ -0,0 +1,87 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +yum install -y git sudo wget || true + +git config --global --add safe.directory /work + +ci/run_ci.sh install_bazel +export PATH="$HOME/.local/bin:$PATH" + +# Function to verify the installed version against expected version +verify_version() { + local installed_version=$1 + + echo "Installed version: $installed_version" + + # Check if GITHUB_REF_NAME is available and use it for verification + if [ -n "$GITHUB_REF_NAME" ]; then + # Strip leading 'v' if present and capture only the actual output, not debug messages + local expected_version + expected_version="$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $GITHUB_REF_NAME | tail -n1)" + echo "Expected version: $expected_version" + + if [ "$installed_version" != "$expected_version" ]; then + echo "Version mismatch: Expected $expected_version but got $installed_version" + exit 1 + fi + echo "Version verification successful" + else + echo "GITHUB_REF_NAME not available, skipping version verification" + fi +} + +# use the python interpreters preinstalled in manylinux +OLD_PATH=$PATH +for PY in $PYTHON_VERSIONS; do + export PYTHON_PATH="/opt/python/$PY/bin/python" + export PATH="/opt/python/$PY/bin:$OLD_PATH" + echo "Using $PYTHON_PATH" + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ]; then + export PLAT="manylinux2014_aarch64" + else + export PLAT="manylinux2014_x86_64" + fi + python -m pip install cython wheel pytest auditwheel + ci/deploy.sh build_pyfory + + latest_wheel=$(find dist -maxdepth 1 -type f -name '*.whl' -print0 | xargs -0 ls -t | head -n1) + if [ -z "$latest_wheel" ]; then + echo "No wheel found" >&2 + exit 1 + fi + + echo "Attempting to install $latest_wheel" + python -m pip install "$latest_wheel" + + # Verify the installed version matches the expected version + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + + # Only run version verification for release builds + if [ "${RELEASE_BUILD:-0}" = "1" ]; then + echo "Running version verification for release build" + verify_version "$INSTALLED_VERSION" + else + echo "Skipping version verification for test build" + fi + + bazel clean --expunge +done +export PATH=$OLD_PATH diff --git a/java/benchmark/src/main/proto/bench.proto b/java/benchmark/src/main/proto/bench.proto index e10e13689f..0e1ec75979 100644 --- a/java/benchmark/src/main/proto/bench.proto +++ b/java/benchmark/src/main/proto/bench.proto @@ -37,7 +37,7 @@ message Bar { optional int64 f6 = 6; optional float f7 = 7; optional double f8 = 8; - repeated int32 f9 = 9; // proto不支持int16 + repeated int32 f9 = 9; // proto does not support int16 repeated int64 f10 = 10; } diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 0d9e236e22..f5135bcb67 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2432,8 +2432,7 @@ public ExprCode doGenCode(CodegenContext ctx) { action.apply( new Reference(i), new Reference(leftElemValue, leftElemType, true), - // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is - // false. + // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is false. new Reference(rightElemValue, rightElemType, false)); ExprCode elementExprCode = elemExpr.genCode(ctx); diff --git a/javascript/packages/fory/lib/meta/MetaString.ts b/javascript/packages/fory/lib/meta/MetaString.ts index 92453d2169..f7678d369f 100644 --- a/javascript/packages/fory/lib/meta/MetaString.ts +++ b/javascript/packages/fory/lib/meta/MetaString.ts @@ -201,7 +201,7 @@ export class MetaStringDecoder { capitalize(str: string) { if (typeof str !== "string" || str.length === 0) { - return str; // 如果不是字符串或为空，直接返回原值 + return str; // If not a string or empty, return the original value } return str.charAt(0).toUpperCase() + str.slice(1); } @@ -366,22 +366,22 @@ export class MetaStringEncoder { } isUpperCase(str: string) { - // 检查字符串是否为空 + // Check whether the string is empty if (typeof str !== "string" || str.length === 0) { - return false; // 如果不是字符串或为空，返回 false + return false; // If not a string or empty, return false } - // 使用正则表达式检查是否所有字母字符都是大写的 + // Use a regular expression to check whether all alphabetic characters are uppercase return /^[^a-z]*$/.test(str); } isDigit(str: string) { - // 检查字符串是否为空 + // Check whether the string is empty if (typeof str !== "string" || str.length === 0) { - return false; // 如果不是字符串或为空，返回 false + return false; // If not a string or empty, return false } - // 使用正则表达式检查是否所有字母字符都是大写的 + // Use a regular expression to check whether all alphabetic characters are uppercase return /^[^0-9]*$/.test(str); }