diff --git a/.github/workflows/build-containerized-pr.yml b/.github/workflows/build-containerized-pr.yml new file mode 100644 index 0000000000..5d144bc60b --- /dev/null +++ b/.github/workflows/build-containerized-pr.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Containerized PR Wheels +on: + pull_request: + paths: [ci/**, python/**, .github/workflows/**] + push: + branches: [main] + paths: [ci/**, python/**, .github/workflows/**] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v5 + - name: Build and test wheels + run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} diff --git a/.github/workflows/build-containerized-release.yml b/.github/workflows/build-containerized-release.yml new file mode 100644 index 0000000000..7551ec047b --- /dev/null +++ b/.github/workflows/build-containerized-release.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Containerized Release Wheels +on: + push: + tags: ['v*'] # NO PATH FILTER - critical for releases + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v5 + - name: Bump version + run: ./ci/deploy.sh bump_py_version + - name: Install bazel + run: ./ci/run_ci.sh install_bazel + - name: Build and test wheels + run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }} + path: dist/*.whl diff --git a/.github/workflows/build-native-pr.yml b/.github/workflows/build-native-pr.yml new file mode 100644 index 0000000000..3c0d2c06c0 --- /dev/null +++ b/.github/workflows/build-native-pr.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native PR Wheels +on: + pull_request: + paths: [ci/**, python/**, .github/workflows/**] + push: + branches: [main] + paths: [ci/**, python/**, .github/workflows/**] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.8', '3.13'] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install bazel + if: runner.os != 'Windows' + run: ./ci/run_ci.sh install_bazel + - name: Install bazel (Windows) + if: runner.os == 'Windows' + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + - name: Build wheel + run: ./ci/deploy.sh build_pyfory + shell: bash + - name: Install and verify wheel + shell: bash + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml new file mode 100644 index 0000000000..f6c43662cf --- /dev/null +++ b/.github/workflows/build-native-release.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build Native Release Wheels +on: + push: + tags: ['v*'] # NO PATH FILTER - critical for releases + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + steps: + - uses: actions/checkout@v5 + - name: Bump version + run: ./ci/deploy.sh bump_py_version + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install bazel + if: runner.os != 'Windows' + run: ./ci/run_ci.sh install_bazel + - name: Install bazel (Windows) + if: runner.os == 'Windows' + run: ./ci/run_ci.sh install_bazel_windows + shell: bash + - name: Build wheel + run: ./ci/deploy.sh build_pyfory + - name: Install and verify wheel + shell: bash + run: | + python -m pip install --upgrade pip + pip install dist/*.whl + python -c "import pyfory; print(pyfory.__version__)" + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }} + path: dist/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11c92b32b4..db261c1a6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: matrix: java-version: ["8", "11", "17", "21", "24"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -81,7 +81,7 @@ jobs: # String in openj9 1.8 share byte array by offset, fory doesn't allow it. java-version: ["21"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -107,7 +107,7 @@ jobs: matrix: java-version: ["21"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -128,7 +128,7 @@ jobs: matrix: java-version: ["17", "21", "23"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: graalvm/setup-graalvm@v1 with: java-version: ${{ matrix.java-version }} @@ -152,7 +152,7 @@ jobs: matrix: java-version: ["8", "11", "17", "21"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -171,7 +171,7 @@ jobs: name: Scala CI runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK8 uses: actions/setup-java@v4 with: @@ -189,7 +189,7 @@ jobs: name: Integration Tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK8 uses: actions/setup-java@v4 with: @@ -210,7 +210,7 @@ jobs: os: [ubuntu-latest, macos-13, windows-2022] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: @@ -237,7 +237,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.11 uses: actions/setup-python@v5 with: @@ -252,7 +252,7 @@ jobs: os: [ubuntu-latest, macos-13, macos-14, windows-2022] # macos-13: x86, macos-14: arm64 runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python 3.11 uses: actions/setup-python@v5 with: @@ -267,7 +267,7 @@ jobs: python-version: [3.8, 3.12, 3.13.3] os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, windows-2022] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -286,7 +286,7 @@ jobs: matrix: go-version: ["1.13", "1.18"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Go ${{ matrix.go-version }} uses: actions/setup-go@v4 with: @@ -308,7 +308,7 @@ jobs: name: Code Style Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release-java-snapshot.yaml b/.github/workflows/release-java-snapshot.yaml index 1292ccc1f3..631975aac4 100644 --- a/.github/workflows/release-java-snapshot.yaml +++ b/.github/workflows/release-java-snapshot.yaml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'apache/fory' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Maven Central Repository uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml new file mode 100644 index 0000000000..c66d65e705 --- /dev/null +++ b/.github/workflows/release-python.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Publish Python + +on: + workflow_run: + workflows: ["Build Containerized Release Wheels", "Build Native Release Wheels"] + types: [completed] + +permissions: + contents: read + id-token: write + +jobs: + publish-wheels: + name: Publish Wheels + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Download all wheel artifacts + uses: actions/download-artifact@v5 + with: + path: downloaded_wheels + + - name: Move wheels to a single directory + shell: bash + run: | + mkdir dist + find downloaded_wheels -type f -name "*.whl" -exec mv {} dist/ \; + ls -R dist + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + verbose: true + verify-metadata: false + packages-dir: dist + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') + with: + skip-existing: true + verify-metadata: false + packages-dir: dist diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 0d3728421b..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Publish Fory - -on: - push: - tags: - - "v*" - -permissions: - contents: read - -jobs: - build-wheels: - name: Build Wheels - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.8, 3.9, "3.10", 3.11, 3.12, 3.13] - os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, windows-2022] # macos-13: x86, macos-14: arm64 - env: - manylinux_x86_64_image: quay.io/pypa/manylinux_2_28_x86_64 - manylinux_aarch64_image: quay.io/pypa/manylinux_2_28_aarch64 - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install bazel (for macOS and Windows) - if: "!startsWith(matrix.os, 'ubuntu')" - shell: bash - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - ./ci/run_ci.sh install_bazel_windows - else - ./ci/run_ci.sh install_bazel - fi - - name: Update version in setup.py - shell: bash - run: ci/deploy.sh bump_py_version - # --------- Use manylinux for Linux wheels --------- - - name: Build a binary wheel (Linux, manylinux) - if: startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - DOCKER_IMAGE="" - PLAT="" - if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - DOCKER_IMAGE="${{ env.manylinux_x86_64_image }}" - PLAT="manylinux_2_28_x86_64" - elif [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then - DOCKER_IMAGE="${{ env.manylinux_aarch64_image }}" - PLAT="manylinux_2_28_aarch64" - fi - PY_VERSION=${{ matrix.python-version }} - echo "PY_VERSION: $PY_VERSION" - PY_VERSION=${PY_VERSION//./} - echo "PY_VERSION without dots: $PY_VERSION" - docker run --rm -e PY_VERSION="$PY_VERSION" -e PLAT="$PLAT" \ - -v ${{ github.workspace }}:/work \ - -w /work "$DOCKER_IMAGE" \ - bash -c " - set -e - yum install -y git sudo wget - git config --global --add safe.directory /work - ls -alh /opt/python - echo \"PY_VERSION: \$PY_VERSION\" - ls /opt/python/cp\${PY_VERSION}-cp\${PY_VERSION} - ls /opt/python/cp\${PY_VERSION}-cp\${PY_VERSION}/bin - export PATH=/opt/python/cp\${PY_VERSION}-cp\${PY_VERSION}/bin:\$PATH - echo \"PATH: \$PATH\" - echo \"Using Python from: \$(which python)\" - echo \"Python version: \$(python -V)\" - bash ci/run_ci.sh install_bazel - bash ci/deploy.sh build_pyfory - " - - # --------- Native (not in container) for macOS and Windows --------- - - name: Build a binary wheel (native) - if: "!startsWith(matrix.os, 'ubuntu')" - shell: bash - run: | - ci/deploy.sh build_pyfory - - name: Upload Wheel Artifact - uses: actions/upload-artifact@v4 - with: - name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }} - path: dist/*.whl - - publish-wheels: - name: Publish Wheels - runs-on: ubuntu-latest - needs: build-wheels - permissions: - contents: read - id-token: write - steps: - - name: Download Wheel Artifacts - uses: actions/download-artifact@v4 - with: - path: downloaded_wheels/ - merge-multiple: true - - name: Display structure of downloaded files - run: ls -R downloaded_wheels - - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: ${{ startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') }} - with: - repository-url: https://test.pypi.org/legacy/ - skip-existing: true - verbose: true - verify-metadata: false - packages-dir: downloaded_wheels - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }} - with: - skip-existing: true - verify-metadata: false - packages-dir: downloaded_wheels diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 46ed08d178..9f4c1b0fbe 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'apache/fory' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Sync files uses: BetaHuhn/repo-file-sync-action@v1 with: diff --git a/README.md b/README.md index 22937b0c6d..cd38e3ea6f 100644 --- a/README.md +++ b/README.md @@ -106,13 +106,13 @@ Nightly snapshot: org.apache.fory fory-core - 0.12.0-SNAPSHOT + 0.13.0-SNAPSHOT ``` @@ -122,13 +122,13 @@ Release version: org.apache.fory fory-core - 0.11.2 + 0.12.0 ``` @@ -137,13 +137,13 @@ Release version: Scala2: ```sbt -libraryDependencies += "org.apache.fory" % "fory-scala_2.13" % "0.11.2" +libraryDependencies += "org.apache.fory" % "fory-scala_2.13" % "0.12.0" ``` Scala3: ```sbt -libraryDependencies += "org.apache.fory" % "fory-scala_3" % "0.11.2" +libraryDependencies += "org.apache.fory" % "fory-scala_3" % "0.12.0" ``` ### Kotlin @@ -152,7 +152,7 @@ libraryDependencies += "org.apache.fory" % "fory-scala_3" % "0.11.2" org.apache.fory fory-kotlin - 0.11.2 + 0.12.0 ``` diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py new file mode 100755 index 0000000000..02c1754e67 --- /dev/null +++ b/ci/build_linux_wheels.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Host-side wrapper: workflow provides only --arch. +Images are defined as regular Python lists (no env vars). + +Environment: + - GITHUB_WORKSPACE (optional; defaults to cwd) +""" +from __future__ import annotations +import argparse +import os +import shlex +import subprocess +import sys +from typing import List + +SCRIPT = r'''set -e +yum install -y git sudo wget || true + +git config --global --add safe.directory /work + +# Determine Python versions to test +if [ "$RELEASE" = "1" ]; then + PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" +else + PYTHON_VERSIONS="cp38-cp38 cp313-cp313" +fi + +ci/run_ci.sh install_bazel +export PATH="$HOME/.local/bin:$PATH" + +# use the python interpreters preinstalled in manylinux +OLD_PATH=$PATH +for PY in $PYTHON_VERSIONS; do + export PYTHON_PATH="/opt/python/$PY/bin/python" + export PATH="/opt/python/$PY/bin:$OLD_PATH" + echo "Using $PYTHON_PATH" + python -m pip install cython wheel pytest + ci/deploy.sh build_pyfory + + latest_wheel=$(ls -t dist/*.whl | head -n1) + echo "Attempting to install $latest_wheel" + python -m pip install "$latest_wheel" + python -c "import pyfory; print(pyfory.__version__)" + + bazel clean --expunge +done +export PATH=$OLD_PATH +''' + +DEFAULT_X86_IMAGES = [ + "quay.io/pypa/manylinux2014_x86_64:latest", + # "quay.io/pypa/manylinux_2_28_x86_64:latest", + + # bazel binaries do not work with musl + # "quay.io/pypa/musllinux_1_2_x86_64:latest", +] + +DEFAULT_AARCH64_IMAGES = [ + "quay.io/pypa/manylinux2014_aarch64:latest", + # "quay.io/pypa/manylinux_2_28_aarch64:latest", + + # bazel binaries do not work with musl + # "quay.io/pypa/musllinux_1_2_aarch64:latest", +] + +ARCH_ALIASES = { + "X86": "x86", + "X64": "x86", + "X86_64": "x86", + "AMD64": "x86", + "ARM": "arm64", + "ARM64": "arm64", + "AARCH64": "arm64", +} + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--arch", required=True, help="Architecture (e.g. X86, X64, AARCH64)") + p.add_argument("--release", action="store_true", help="Run full test suite for release") + p.add_argument("--dry-run", action="store_true", help="Print docker commands without running") + return p.parse_args() + +def normalize_arch(raw: str) -> str: + key = raw.strip().upper() + return ARCH_ALIASES.get(key, raw.strip().lower()) + +def collect_images_for_arch(arch_normalized: str) -> List[str]: + if arch_normalized == "x86": + imgs = DEFAULT_X86_IMAGES # dedupe preserving order + elif arch_normalized == "arm64": + imgs = DEFAULT_AARCH64_IMAGES + else: + raise SystemExit(f"Unsupported arch: {arch_normalized!r}") + return imgs + +def build_docker_cmd(workspace: str, image: str) -> List[str]: + workspace = os.path.abspath(workspace) + return [ + "docker", "run", "-i", "--rm", + "-v", f"{workspace}:/work", + "-w", "/work", + image, + "bash", "-s", "--" + ] + +def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int: + rc_overall = 0 + for image in images: + docker_cmd = build_docker_cmd(workspace, image) + printable = " ".join(shlex.quote(c) for c in docker_cmd) + print(f"+ {printable}") + if dry_run: + continue + try: + completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8")) + if completed.returncode != 0: + print(f"Container {image} exited with {completed.returncode}", file=sys.stderr) + rc_overall = completed.returncode if rc_overall == 0 else rc_overall + else: + print(f"Container {image} completed successfully.") + except KeyboardInterrupt: + print("Interrupted by user", file=sys.stderr) + return 130 + except FileNotFoundError as e: + print(f"Error running docker: {e}", file=sys.stderr) + return 2 + return rc_overall + +def main() -> int: + args = parse_args() + arch = normalize_arch(args.arch) + images = collect_images_for_arch(arch) + if not images: + print(f"No images configured for arch {arch}", file=sys.stderr) + return 2 + workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd()) + print(f"Selected images for arch {args.arch}: {images}") + return run_for_images(images, workspace, args.dry_run) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/deploy.sh b/ci/deploy.sh index 9a71fceb9d..58648a3fe8 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -18,16 +18,20 @@ # under the License. +# Print commands and their arguments as they are executed. set -x # Cause the script to exit if a single command fails. set -e -# configure ~/.pypirc before run this script -#if [ ! -f ~/.pypirc ]; then -# echo "Please configure .pypirc before run this script" -# exit 1 -#fi +# Prefer Python from $PYTHON_PATH if it exists, otherwise use default python +if [ -n "$PYTHON_PATH" ] && [ -x "$PYTHON_PATH" ]; then + PYTHON_CMD="$PYTHON_PATH" + PIP_CMD="$PYTHON_PATH -m pip" +else + PYTHON_CMD="python" + PIP_CMD="pip" +fi ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" @@ -63,34 +67,32 @@ deploy_jars() { } build_pyfory() { - echo "Python version $(python -V), path $(which python)" + echo "$($PYTHON_CMD -V), path $(which "$PYTHON_CMD")" install_pyarrow - pip install Cython wheel pytest auditwheel + $PIP_CMD install cython wheel pytest pushd "$ROOT/python" - pip list + $PIP_CMD list echo "Install pyfory" # Fix strange installed deps not found - pip install setuptools -U + $PIP_CMD install setuptools -U if [[ "$OSTYPE" == "darwin"* ]]; then MACOS_VERSION=$(sw_vers -productVersion | cut -d. -f1-2) echo "MACOS_VERSION: $MACOS_VERSION" if [[ "$MACOS_VERSION" == "13"* ]]; then export MACOSX_DEPLOYMENT_TARGET=10.13 - python setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist else - python setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist fi else - python setup.py bdist_wheel --dist-dir=../dist + $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist fi - ls -l ../dist - if [ -n "$PLAT" ]; then # In manylinux container, repair the wheel to embed shared libraries # and rename the wheel with the manylinux tag. - PYARROW_LIB_DIR=$(python -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') + PYARROW_LIB_DIR=$($PYTHON_CMD -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))') export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH" auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w ../dist/ rm ../dist/pyfory-*-linux_*.whl @@ -99,17 +101,19 @@ build_pyfory() { elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then echo "Skip windows wheel repair" fi + + echo "Wheels for $PYTHON_CMD:" ls -l ../dist popd } install_pyarrow() { - pyversion=$(python -V | cut -d' ' -f2) + pyversion=$($PYTHON_CMD -V | cut -d' ' -f2) if [[ $pyversion == 3.13* ]]; then - pip install pyarrow==18.0.0 - pip install numpy + $PIP_CMD install pyarrow==18.0.0 + $PIP_CMD install numpy else - pip install pyarrow==15.0.0 + $PIP_CMD install pyarrow==15.0.0 # Automatically install numpy fi } diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 0c4fb52f64..63a1330d14 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -44,12 +44,11 @@ export FORY_CI=true install_python() { wget -q https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -O Miniconda3.sh bash Miniconda3.sh -b -p $HOME/miniconda && rm -f miniconda.* - which python - echo "Python version $(python -V), path $(which python)" + echo "$(python -V), path $(which python)" } install_pyfory() { - echo "Python version $(python -V), path $(which python)" + echo "$(python -V), path $(which python)" "$ROOT"/ci/deploy.sh install_pyarrow pip install Cython wheel pytest pushd "$ROOT/python" @@ -90,15 +89,15 @@ install_bazel() { esac BAZEL_VERSION=$(get_bazel_version) - BAZEL_DIR="/usr/local/bin" + BAZEL_DIR="$HOME/.local/bin" + mkdir -p "$BAZEL_DIR" # Construct platform-specific URL BINARY_URL="https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-${OS}-${ARCH}" echo "Downloading bazel from: $BINARY_URL" - sudo wget -q -O "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } - - sudo chmod +x "$BAZEL_DIR/bazel" + curl -L -sSf -o "$BAZEL_DIR/bazel" "$BINARY_URL" || { echo "Failed to download bazel"; exit 1; } + chmod +x "$BAZEL_DIR/bazel" # Add to current shell's PATH export PATH="$BAZEL_DIR:$PATH" @@ -108,7 +107,7 @@ install_bazel() { bazel version || { echo "Bazel installation verification failed"; exit 1; } # Configure number of jobs based on memory - if [[ "$MACHINE" == linux ]]; then + if [[ "$OS" == linux ]]; then MEM=$(grep MemTotal < /proc/meminfo | awk '{print $2}') JOBS=$(( MEM / 1024 / 1024 / 3 )) echo "build --jobs=$JOBS" >> ~/.bazelrc diff --git a/ci/tasks/python.py b/ci/tasks/python.py index 252d238d50..ea53e94332 100644 --- a/ci/tasks/python.py +++ b/ci/tasks/python.py @@ -25,7 +25,7 @@ def install_pyfory(): logging.info("Installing pyfory package") python_version = common.exec_cmd("python -V") python_path = common.exec_cmd("which python") - logging.info(f"Python version {python_version}, path {python_path}") + logging.info(f"{python_version}, path {python_path}") # Install PyArrow common.exec_cmd(f"{common.PROJECT_ROOT_DIR}/ci/deploy.sh install_pyarrow") diff --git a/docs/guide/scala_guide.md b/docs/guide/scala_guide.md index c0ca649637..563e1e65d2 100644 --- a/docs/guide/scala_guide.md +++ b/docs/guide/scala_guide.md @@ -34,7 +34,7 @@ Scala 2 and 3 are both supported. To add a dependency on Fory scala for with sbt, use the following: ```sbt -libraryDependencies += "org.apache.fory" %% "fory-scala" % "0.11.2" +libraryDependencies += "org.apache.fory" %% "fory-scala" % "0.12.0" ``` ## Quick Start diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md index 51fa304801..4efae21923 100644 --- a/docs/specification/java_serialization_spec.md +++ b/docs/specification/java_serialization_spec.md @@ -136,8 +136,8 @@ Class meta are encoded from parent class to leaf class, only class with serializ Meta header is a 64 bits number value encoded in little endian order. -- lower 12 bits are used to encode meta size. If meta size `>= 0b111_1111_1111`, then write - `meta_ size - 0b111_1111_1111` next. +- lower 12 bits are used to encode meta size. If meta size `>= 0b1111_1111_1111`, then write + `meta_ size - 0b1111_1111_1111` next. - 13rd bit is used to indicate whether to write fields meta. When this class is schema-consistent or use registered serializer, fields meta will be skipped. Class Meta will be used for share namespace + type name only. - 14rd bit is used to indicate whether meta is compressed. diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 2fcbab1db7..ba0cc0067c 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -319,8 +319,8 @@ subclass. `50 bits hash + 1bit compress flag + write fields meta + 12 bits meta size`. Right is the lower bits. -- lower 12 bits are used to encode meta size. If meta size `>= 0b111_1111_1111`, then write - `meta_ size - 0b111_1111_1111` next. +- lower 12 bits are used to encode meta size. If meta size `>= 0b1111_1111_1111`, then write + `meta_ size - 0b1111_1111_1111` next. - 13rd bit is used to indicate whether to write fields meta. When this class is schema-consistent or use registered serializer, fields meta will be skipped. Class Meta will be used for share namespace + type name only. - 14rd bit is used to indicate whether meta is compressed. diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index 01c5e648fc..236bcf6254 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 ../../java 4.0.0 diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index 46b38388ba..891782a14c 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 ../../java 4.0.0 diff --git a/integration_tests/jpms_tests/pom.xml b/integration_tests/jpms_tests/pom.xml index 911c27a2da..63d9b522d4 100644 --- a/integration_tests/jpms_tests/pom.xml +++ b/integration_tests/jpms_tests/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 ../../java 4.0.0 diff --git a/integration_tests/latest_jdk_tests/pom.xml b/integration_tests/latest_jdk_tests/pom.xml index b9b10358e0..94f4d1b687 100644 --- a/integration_tests/latest_jdk_tests/pom.xml +++ b/integration_tests/latest_jdk_tests/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 ../../java 4.0.0 diff --git a/java/benchmark/pom.xml b/java/benchmark/pom.xml index ef449a66ae..bc8eccfa93 100644 --- a/java/benchmark/pom.xml +++ b/java/benchmark/pom.xml @@ -26,7 +26,7 @@ fory-parent org.apache.fory - 0.12.0 + 0.12.1 benchmark diff --git a/java/fory-core/pom.xml b/java/fory-core/pom.xml index b5ef9e22bb..79f0ac1dba 100644 --- a/java/fory-core/pom.xml +++ b/java/fory-core/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 4.0.0 diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 582c236129..0d9e236e22 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2432,7 +2432,7 @@ public ExprCode doGenCode(CodegenContext ctx) { action.apply( new Reference(i), new Reference(leftElemValue, leftElemType, true), - // elemValue nullability check use isNullAt inside action, so elemValueRef'nullable is + // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is // false. new Reference(rightElemValue, rightElemType, false)); ExprCode elementExprCode = elemExpr.genCode(ctx); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index 507c9522d7..b49ded2e4f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -237,8 +237,9 @@ private void disallow(String classNameOrPrefix) { public void addListener(ClassResolver classResolver) { try { lock.writeLock().lock(); - } finally { listeners.put(classResolver, true); + } finally { + lock.writeLock().unlock(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index 5d35b5b5c4..afee63b3db 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -656,6 +656,17 @@ public static boolean isBean(TypeRef typeRef, TypeResolutionContext ctx) { } } + /** + * Check if a class is one of {@link Optional), {@link OptionalInt}, + * {@link OptionaLong}, or {@link OptionalDouble}. + */ + public static boolean isOptionalType(Class type) { + return type == Optional.class + || type == OptionalInt.class + || type == OptionalLong.class + || type == OptionalDouble.class; + } + private static boolean isSynthesizableInterface(Class cls) { return cls.isInterface() && !Collection.class.isAssignableFrom(cls) diff --git a/java/fory-extensions/pom.xml b/java/fory-extensions/pom.xml index 9471560a46..79d4056e66 100644 --- a/java/fory-extensions/pom.xml +++ b/java/fory-extensions/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 4.0.0 diff --git a/java/fory-format/pom.xml b/java/fory-format/pom.xml index 00b00aac55..2d9a18f441 100644 --- a/java/fory-format/pom.xml +++ b/java/fory-format/pom.xml @@ -25,7 +25,7 @@ org.apache.fory fory-parent - 0.12.0 + 0.12.1 4.0.0 diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java index 49910d9584..13e1e3d52c 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayDataForEach.java @@ -110,7 +110,7 @@ public Code.ExprCode doGenCode(CodegenContext ctx) { String i = freshNames[0]; String elemValue = freshNames[1]; String len = freshNames[2]; - // elemValue is only used in notNullAction, so set elemValueRef'nullable to false. + // elemValue is only used in notNullAction, so set elemValueRef's nullable to false. Reference elemValueRef = new Reference(elemValue, elemType); Code.ExprCode notNullElemExprCode = notNullAction.apply(new Reference(i), elemValueRef).genCode(ctx); diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java index 18845507b8..522f54b27b 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/Encoders.java @@ -474,7 +474,7 @@ public static MapEncoder mapEncoder(TypeRef token, Fory fo TypeRef keyToken = token4BeanLoad(set1, tuple2.f0); TypeRef valToken = token4BeanLoad(set2, tuple2.f1); - MapEncoder encoder = mapEncoder(token, keyToken, valToken, fory); + MapEncoder encoder = mapEncoder0(token, keyToken, valToken, fory); return createMapEncoder(encoder); } @@ -495,6 +495,22 @@ public static MapEncoder mapEncoder( Preconditions.checkNotNull(keyToken); Preconditions.checkNotNull(valToken); + Set> set1 = beanSet(keyToken); + Set> set2 = beanSet(valToken); + LOG.info("Find beans to load: {}, {}", set1, set2); + + token4BeanLoad(set1, keyToken); + token4BeanLoad(set2, valToken); + + return mapEncoder0(mapToken, keyToken, valToken, fory); + } + + private static MapEncoder mapEncoder0( + TypeRef mapToken, TypeRef keyToken, TypeRef valToken, Fory fory) { + Preconditions.checkNotNull(mapToken); + Preconditions.checkNotNull(keyToken); + Preconditions.checkNotNull(valToken); + Schema schema = TypeInference.inferSchema(mapToken, false); Field field = DataTypes.fieldOfSchema(schema, 0); Field keyField = DataTypes.keyArrayFieldForMap(field); @@ -685,6 +701,9 @@ public static Class loadOrGenRowCodecClass(Class beanClass) { TypeUtils.listBeansRecursiveInclusive( beanClass, new TypeResolutionContext(CustomTypeEncoderRegistry.customTypeHandler(), true)); + if (classes.isEmpty()) { + return null; + } LOG.info("Create RowCodec for classes {}", classes); CompileUnit[] compileUnits = classes.stream() diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java index 45f65fff93..49ec2e134f 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java @@ -285,10 +285,7 @@ public Expression buildDecodeExpression() { private static Expression nullValue(TypeRef fieldType) { Class rawType = fieldType.getRawType(); - if (rawType == Optional.class - || rawType == OptionalInt.class - || rawType == OptionalLong.class - || rawType == OptionalDouble.class) { + if (TypeUtils.isOptionalType(rawType)) { return new Expression.StaticInvoke(rawType, "empty", "", fieldType, false, true); } return new Expression.Reference(TypeUtils.defaultValue(rawType), fieldType); @@ -361,7 +358,7 @@ private CodegenContext buildImplClass() { Expression storeValue = new Expression.SetField(new Expression.Reference("this"), fieldName, decodeValue); Expression shouldLoad; - if (rawFieldType == Optional.class) { + if (TypeUtils.isOptionalType(rawFieldType)) { shouldLoad = new Expression.Not( Expression.Invoke.inlineInvoke(fieldRef, "isPresent", TypeUtils.BOOLEAN_TYPE)); diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java index cdd38be6cf..b942d5dc01 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java @@ -22,7 +22,11 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; import java.util.TreeSet; + import lombok.Data; import org.apache.arrow.vector.types.pojo.Field; import org.apache.fory.annotation.ForyField; @@ -141,35 +145,56 @@ public PoisonPill decode(final byte[] value) { public interface OptionalType { Optional f1(); + OptionalInt f2(); + OptionalLong f3(); + OptionalDouble f4(); } static class OptionalTypeImpl implements OptionalType { - private final Optional f1; - - OptionalTypeImpl(final Optional f1) { - this.f1 = f1; - } + Optional f1; + OptionalInt f2; + OptionalLong f3; + OptionalDouble f4; @Override public Optional f1() { return f1; } + + @Override + public OptionalInt f2() { + return f2; + } + + @Override + public OptionalLong f3() { + return f3; + } + + @Override + public OptionalDouble f4() { + return f4; + } } @Test public void testNullOptional() { - final OptionalType bean1 = new OptionalTypeImpl(null); + final OptionalType bean1 = new OptionalTypeImpl(); final RowEncoder encoder = Encoders.bean(OptionalType.class); final BinaryRow row = encoder.toRow(bean1); final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); row.pointTo(buffer, 0, buffer.size()); final OptionalType deserializedBean = encoder.fromRow(row); Assert.assertEquals(deserializedBean.f1(), Optional.empty()); + Assert.assertEquals(deserializedBean.f2(), OptionalInt.empty()); + Assert.assertEquals(deserializedBean.f3(), OptionalLong.empty()); + Assert.assertEquals(deserializedBean.f4(), OptionalDouble.empty()); } @Test public void testPresentOptional() { - final OptionalType bean1 = new OptionalTypeImpl(Optional.of("42")); + final OptionalTypeImpl bean1 = new OptionalTypeImpl(); + bean1.f1 = Optional.of("42"); final RowEncoder encoder = Encoders.bean(OptionalType.class); final BinaryRow row = encoder.toRow(bean1); final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); @@ -178,6 +203,42 @@ public void testPresentOptional() { Assert.assertEquals(deserializedBean.f1(), Optional.of("42")); } + @Test + public void testPresentOptionalInteger() { + final OptionalTypeImpl bean1 = new OptionalTypeImpl(); + bean1.f2 = OptionalInt.of(42); + final RowEncoder encoder = Encoders.bean(OptionalType.class); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final OptionalType deserializedBean = encoder.fromRow(row); + Assert.assertEquals(deserializedBean.f2(), OptionalInt.of(42)); + } + + @Test + public void testPresentOptionalLong() { + final OptionalTypeImpl bean1 = new OptionalTypeImpl(); + bean1.f3 = OptionalLong.of(42); + final RowEncoder encoder = Encoders.bean(OptionalType.class); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final OptionalType deserializedBean = encoder.fromRow(row); + Assert.assertEquals(deserializedBean.f3(), OptionalLong.of(42)); + } + + @Test + public void testPresentOptionalDouble() { + final OptionalTypeImpl bean1 = new OptionalTypeImpl(); + bean1.f4 = OptionalDouble.of(42.42); + final RowEncoder encoder = Encoders.bean(OptionalType.class); + final BinaryRow row = encoder.toRow(bean1); + final MemoryBuffer buffer = MemoryUtils.wrap(row.toBytes()); + row.pointTo(buffer, 0, buffer.size()); + final OptionalType deserializedBean = encoder.fromRow(row); + Assert.assertEquals(deserializedBean.f4(), OptionalDouble.of(42.42)); + } + public static class Id { byte id; diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/MapEncoderTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/MapEncoderTest.java index cf40f70ac4..3b0fdea73e 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/MapEncoderTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/MapEncoderTest.java @@ -134,9 +134,9 @@ public void testSimpleNestStructWithMapEncoder() { @Test public void testKVStructMap() { Map map = ImmutableMap.of(SimpleFoo.create(), SimpleFoo.create()); - MapEncoder encoder = Encoders.mapEncoder(new TypeRef>() {}); + var encoder = Encoders.mapEncoder(new TypeRef>() {}); testStreamingEncode(encoder, map); - MapEncoder encoder1 = Encoders.mapEncoder(new TypeRef>() {}); + var encoder1 = Encoders.mapEncoder(new TypeRef>() {}); testStreamingEncode(encoder1, ImmutableMap.of(Foo.create(), Foo.create())); } @@ -192,4 +192,18 @@ public void testNestArrayWithMapEncoder() { testStreamingEncode(encoder, lmap); } + + @Test + public void testDynamicTypeDeclaration() { + Encoders.mapEncoder( + new TypeRef>() {}, + TypeRef.of(Integer.class), + TypeRef.of(Bean.class), + null) + .encode(new HashMap<>()); + } + + public static class Bean { + int f1; + } } diff --git a/java/fory-test-core/pom.xml b/java/fory-test-core/pom.xml index a8b8191f41..67a6f90d0d 100644 --- a/java/fory-test-core/pom.xml +++ b/java/fory-test-core/pom.xml @@ -25,7 +25,7 @@ fory-parent org.apache.fory - 0.12.0 + 0.12.1 4.0.0 diff --git a/java/fory-testsuite/pom.xml b/java/fory-testsuite/pom.xml index bbdd339508..9a6f3c63fa 100644 --- a/java/fory-testsuite/pom.xml +++ b/java/fory-testsuite/pom.xml @@ -25,7 +25,7 @@ fory-parent org.apache.fory - 0.12.0 + 0.12.1 4.0.0 diff --git a/java/pom.xml b/java/pom.xml index 557f5004ae..b4f7a42c56 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -33,7 +33,7 @@ org.apache.fory fory-parent pom - 0.12.0 + 0.12.1 Fory Project Parent POM Apache Fory™ is a blazingly fast multi-language serialization framework powered by jit and zero-copy. diff --git a/javascript/packages/fory/package.json b/javascript/packages/fory/package.json index 0e57b04f05..82bd60d843 100644 --- a/javascript/packages/fory/package.json +++ b/javascript/packages/fory/package.json @@ -1,6 +1,6 @@ { "name": "@foryjs/fory", - "version": "0.12.0", + "version": "0.12.1", "description": "Apache Fory™ is a blazingly fast multi-language serialization framework powered by jit and zero-copy", "main": "dist/index.js", "scripts": { diff --git a/javascript/packages/hps/package.json b/javascript/packages/hps/package.json index 183c420066..394afd738c 100644 --- a/javascript/packages/hps/package.json +++ b/javascript/packages/hps/package.json @@ -1,6 +1,6 @@ { "name": "@foryjs/hps", - "version": "0.12.0", + "version": "0.12.1", "description": "Apache Fory™ nodejs high-performance suite", "main": "dist/index.js", "files": [ diff --git a/kotlin/pom.xml b/kotlin/pom.xml index 89a028d573..a74178c13c 100644 --- a/kotlin/pom.xml +++ b/kotlin/pom.xml @@ -30,7 +30,7 @@ org.apache.fory fory-kotlin - 0.12.0 + 0.12.1 4.0.0 diff --git a/python/CONTRIBUTING.md b/python/CONTRIBUTING.md new file mode 100644 index 0000000000..fbe24c77dd --- /dev/null +++ b/python/CONTRIBUTING.md @@ -0,0 +1,73 @@ +# Contributing to Apache Fory Python + +This document provides instructions for building and testing the `pyfory` package. + +## Building + +```bash +cd python +# Uninstall numpy first so that when we install pyarrow, it will install the correct numpy version automatically. +# For Python versions less than 3.13, numpy 2 is not currently supported. +pip uninstall -y numpy +# Install necessary environment for Python < 3.13. +pip install pyarrow==15.0.0 Cython wheel pytest +# For Python 3.13, pyarrow 18.0.0 is available and requires numpy version greater than 2. +# pip install pyarrow==18.0.0 Cython wheel pytest +pip install -v -e . +``` + +If the last steps fails with an error like `libarrow_python.dylib: No such file or directory`, +you are probably suffering from bazel's aggressive caching; the sought library is longer at the +temporary directory it was the last time bazel ran. To remedy this run + +> bazel clean --expunge + +In this situation, you might also find it fruitful to run bazel yourself before pip: + +> bazel build -s //:cp_fory_so + +### Environment Requirements + +- python 3.8+ + +## Testing + +```bash +cd python +pytest -v -s . +``` + +## Formatting + +```bash +cd python +pip install ruff +ruff format python +``` + +## Debugging + +```bash +cd python +python setup.py develop +``` + +- Use `cython --cplus -a pyfory/_serialization.pyx` to produce an annotated HTML file of the source code. Then you can + analyze interaction between Python objects and Python's C API. +- Read more: + +```bash +FORY_DEBUG=true python setup.py build_ext --inplace +# For linux +cygdb build +``` + +### Debugging with lldb + +```bash +lldb +(lldb) target create -- python +(lldb) settings set -- target.run-args "-c" "from pyfory.tests.test_serializer import test_enum; test_enum()" +(lldb) run +(lldb) bt +``` diff --git a/python/README.md b/python/README.md index fba1679573..963c8409a0 100644 --- a/python/README.md +++ b/python/README.md @@ -1,73 +1,178 @@ # Apache Fory™ Python -Fory is a blazingly-fast multi-language serialization framework powered by just-in-time compilation and zero-copy. +[![Build Status](https://img.shields.io/github/actions/workflow/status/apache/fory/ci.yml?branch=main&style=for-the-badge&label=GITHUB%20ACTIONS&logo=github)](https://github.com/apache/fory/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/pyfory.svg?logo=PyPI)](https://pypi.org/project/pyfory/) +[![Slack Channel](https://img.shields.io/badge/slack-join-3f0e40?logo=slack&style=for-the-badge)](https://join.slack.com/t/fory-project/shared_invite/zt-36g0qouzm-kcQSvV_dtfbtBKHRwT5gsw) +[![X](https://img.shields.io/badge/@ApacheFory-follow-blue?logo=x&style=for-the-badge)](https://x.com/ApacheFory) -## Build Fory Python +**Apache Fory** (formerly _Fury_) is a blazing fast multi-language serialization framework powered by **JIT** (just-in-time compilation) and **zero-copy**, providing up to 170x performance and ease of use. + +This package provides the Python bindings for Apache Fory. + +## Installation + +You can install `pyfory` using pip: ```bash -cd python -# Uninstall numpy first so that when we install pyarrow, it will install the correct numpy version automatically. -# For Python versions less than 3.13, numpy 2 is not currently supported. -pip uninstall -y numpy -# Install necessary environment for Python < 3.13. -pip install pyarrow==15.0.0 Cython wheel pytest -# For Python 3.13, pyarrow 18.0.0 is available and requires numpy version greater than 2. -# pip install pyarrow==18.0.0 Cython wheel pytest -pip install -v -e . +pip install pyfory ``` -If the last steps fails with an error like `libarrow_python.dylib: No such file or directory`, -you are probably suffering from bazel's aggressive caching; the sought library is longer at the -temporary directory it was the last time bazel ran. To remedy this run +## Quickstart -> bazel clean --expunge +Here are a few examples of how to use `pyfory` for serialization. -In this situation, you might also find it fruitful to run bazel yourself before pip: +### Basic Serialization -> bazel build -s //:cp_fory_so +This example shows how to serialize and deserialize a simple Python object. -### Environment Requirements +```python +from typing import Dict +import pyfory -- python 3.8+ +class SomeClass: + f1: "SomeClass" + f2: Dict[str, str] + f3: Dict[str, str] -## Testing - -```bash -cd python -pytest -v -s . +fory = pyfory.Fory(ref_tracking=True) +fory.register_type(SomeClass, typename="example.SomeClass") +obj = SomeClass() +obj.f2 = {"k1": "v1", "k2": "v2"} +obj.f1, obj.f3 = obj, obj.f2 +data = fory.serialize(obj) +# bytes can be data serialized by other languages. +print(fory.deserialize(data)) ``` -## Code Style +### Cross-language Serialization -```bash -cd python -pip install ruff -ruff format python -``` +Apache Fory excels at cross-language serialization. You can serialize data in Python and deserialize it in another language like Java or Go, and vice-versa. -## Debug +Here's an example of how to serialize an object in Python and deserialize it in Java: -```bash -cd python -python setup.py develop -``` +**Python** -- Use `cython --cplus -a pyfory/_serialization.pyx` to produce an annotated HTML file of the source code. Then you can - analyze interaction between Python objects and Python's C API. -- Read more: +```python +from typing import Dict +import pyfory -```bash -FORY_DEBUG=true python setup.py build_ext --inplace -# For linux -cygdb build +class SomeClass: + f1: "SomeClass" + f2: Dict[str, str] + f3: Dict[str, str] + +fory = pyfory.Fory(ref_tracking=True) +fory.register_type(SomeClass, typename="example.SomeClass") +obj = SomeClass() +obj.f2 = {"k1": "v1", "k2": "v2"} +obj.f1, obj.f3 = obj, obj.f2 +data = fory.serialize(obj) +# `data` can now be sent to a Java application ``` -## Debug with lldb +**Java** + +```java +import org.apache.fory.*; +import org.apache.fory.config.*; +import java.util.*; + +public class ReferenceExample { + public static class SomeClass { + SomeClass f1; + Map f2; + Map f3; + } + + public static void main(String[] args) { + Fory fory = Fory.builder().withLanguage(Language.XLANG) + .withRefTracking(true).build(); + fory.register(SomeClass.class, "example.SomeClass"); + // `bytes` would be the data received from the Python application + byte[] bytes = ... + System.out.println(fory.deserialize(bytes)); + } +} +``` -```bash -lldb -(lldb) target create -- python -(lldb) settings set -- target.run-args "-c" "from pyfory.tests.test_serializer import test_enum; test_enum()" -(lldb) run -(lldb) bt +### Row Format Zero-Copy Partial Serialzation + +Apache Fory provide a random-access row format, which supports map a typed nested struct into a binary and read its nested element without deserializing the whole binary. This can be used to minimize teh deserialization overhead for huge objects in the case where you only needs to access part of the data. You can even encode huge objects into binary and write to file, then mmap that file into memory to reduce memory overhead too. + +**Python** + +```python +@dataclass +class Bar: + f1: str + f2: List[pa.int64] +@dataclass +class Foo: + f1: pa.int32 + f2: List[pa.int32] + f3: Dict[str, pa.int32] + f4: List[Bar] + +encoder = pyfory.encoder(Foo) +foo = Foo(f1=10, f2=list(range(1000_000)), + f3={f"k{i}": i for i in range(1000_000)}, + f4=[Bar(f1=f"s{i}", f2=list(range(10))) for i in range(1000_000)]) +binary: bytes = encoder.to_row(foo).to_bytes() +foo_row = pyfory.RowData(encoder.schema, binary) +print(foo_row.f2[100000], foo_row.f4[100000].f1, foo_row.f4[200000].f2[5]) ``` + +**Java** + +```java +public class Bar { + String f1; + List f2; +} + +public class Foo { + int f1; + List f2; + Map f3; + List f4; +} + +RowEncoder encoder = Encoders.bean(Foo.class); +Foo foo = new Foo(); +foo.f1 = 10; +foo.f2 = IntStream.range(0, 1000000).boxed().collect(Collectors.toList()); +foo.f3 = IntStream.range(0, 1000000).boxed().collect(Collectors.toMap(i -> "k"+i, i->i)); +List bars = new ArrayList<>(1000000); +for (int i = 0; i < 1000000; i++) { + Bar bar = new Bar(); + bar.f1 = "s"+i; + bar.f2 = LongStream.range(0, 10).boxed().collect(Collectors.toList()); + bars.add(bar); +} +foo.f4 = bars; +// Can be zero-copy read by python +BinaryRow binaryRow = encoder.toRow(foo); +// can be data from python +Foo newFoo = encoder.fromRow(binaryRow); +// zero-copy read List f2 +BinaryArray binaryArray2 = binaryRow.getArray(1); +// zero-copy read List f4 +BinaryArray binaryArray4 = binaryRow.getArray(3); +// zero-copy read 11th element of `readList f4` +BinaryRow barStruct = binaryArray4.getStruct(10); + +// zero-copy read 6th of f2 of 11th element of `readList f4` +barStruct.getArray(1).getInt64(5); +RowEncoder barEncoder = Encoders.bean(Bar.class); +// deserialize part of data. +Bar newBar = barEncoder.fromRow(barStruct); +Bar newBar2 = barEncoder.fromRow(binaryArray4.getStruct(20)); +``` + +## Useful Links + +- **[Project Website](https://fory.apache.org)** +- **[Documentation](https://fory.apache.org/docs/latest/python_guide/)** +- **[GitHub Repository](https://github.com/apache/fory)** +- **[Issue Tracker](https://github.com/apache/fory/issues)** +- **[Slack Channel](https://join.slack.com/t/fory-project/shared_invite/zt-36g0qouzm-kcQSvV_dtfbtBKHRwT5gsw)** diff --git a/python/pyfory/__init__.py b/python/pyfory/__init__.py index eb94670f3b..6383048bdb 100644 --- a/python/pyfory/__init__.py +++ b/python/pyfory/__init__.py @@ -61,4 +61,4 @@ except (AttributeError, ImportError): pass -__version__ = "0.12.0" +__version__ = "0.12.1" diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index c78e03746e..35ab78d36b 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -117,7 +117,7 @@ class Fory: def __init__( self, - language=Language.XLANG, + language=Language.PYTHON, ref_tracking: bool = False, require_type_registration: bool = True, ): diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx index 156693bdc0..d98a248d53 100644 --- a/python/pyfory/_serialization.pyx +++ b/python/pyfory/_serialization.pyx @@ -388,9 +388,9 @@ cdef class TypeInfo: for python `int`: `Int8/1632/64/128Serializer` for `int8/16/32/64/128` each, and another `IntSerializer` for `int` which will dispatch to different `int8/16/32/64/128` type according the actual value. - We do not get the acutal type here, because it will introduce extra computing. + We do not get the actual type here, because it will introduce extra computing. For example, we have want to get actual `Int8/16/32/64Serializer`, we must check and - extract the actutal here which will introduce cost, and we will do same thing again + extract the actual here which will introduce cost, and we will do same thing again when serializing the actual data. """ cdef public object cls @@ -602,7 +602,7 @@ cdef class Fory: def __init__( self, - language=Language.XLANG, + language=Language.PYTHON, ref_tracking: bool = False, require_type_registration: bool = True, ): @@ -650,14 +650,26 @@ cdef class Fory: def register_serializer(self, cls: Union[type, TypeVar], Serializer serializer): self.type_resolver.register_serializer(cls, serializer) + def register( + self, + cls: Union[type, TypeVar], + *, + type_id: int = None, + namespace: str = None, + typename: str = None, + serializer=None, + ): + self.type_resolver.register_type( + cls, type_id=type_id, namespace=namespace, typename=typename, serializer=serializer) + def register_type( - self, - cls: Union[type, TypeVar], - *, - type_id: int = None, - namespace: str = None, - typename: str = None, - serializer=None, + self, + cls: Union[type, TypeVar], + *, + type_id: int = None, + namespace: str = None, + typename: str = None, + serializer=None, ): self.type_resolver.register_type( cls, type_id=type_id, namespace=namespace, typename=typename, serializer=serializer) @@ -1543,7 +1555,7 @@ cdef inline get_next_element( typeinfo = type_resolver.read_typeinfo(buffer) cdef int32_t type_id = typeinfo.type_id # Note that all read operations in fast paths of list/tuple/set/dict/sub_dict - # ust match corresponding writing operations. Otherwise, ref tracking will + # must match corresponding writing operations. Otherwise, ref tracking will # error. if type_id == TypeId.STRING: return buffer.read_string() diff --git a/python/setup.py b/python/setup.py index 9c488e10e3..87123efa8d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,7 +30,7 @@ os.environ["CFLAGS"] = "-O0" BAZEL_BUILD_EXT = False -print(f"DEBUG = {DEBUG}, BAZEL_BUILD_EXT = {BAZEL_BUILD_EXT}") +print(f"DEBUG = {DEBUG}, BAZEL_BUILD_EXT = {BAZEL_BUILD_EXT}, PATH = {os.environ.get('PATH')}") setup_dir = abspath(os.path.dirname(__file__)) project_dir = abspath(pjoin(setup_dir, os.pardir)) @@ -39,7 +39,6 @@ print(f"setup_dir: {setup_dir}") print(f"fory_cpp_src_dir: {fory_cpp_src_dir}") - class BinaryDistribution(Distribution): def __init__(self, attrs=None): super().__init__(attrs=attrs) @@ -58,6 +57,4 @@ def has_ext_modules(self): if __name__ == "__main__": - setup( - distclass=BinaryDistribution, - ) + setup(distclass=BinaryDistribution) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 174302d0b0..d2032b22f1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.12.0" +version = "0.12.1" rust-version = "1.70" license = "Apache-2.0" readme = "README.md" diff --git a/scala/build.sbt b/scala/build.sbt index 5edbb0974f..7646cac8a5 100644 --- a/scala/build.sbt +++ b/scala/build.sbt @@ -16,7 +16,7 @@ * limitations under the License. */ -val foryVersion = "0.12.0" +val foryVersion = "0.12.1" val scala213Version = "2.13.15" ThisBuild / apacheSonatypeProjectProfile := "fory" version := foryVersion