chore: cherry-pick fix for 0.12.1

.github/workflows/build-containerized-pr.yml

@@ -33,3 +33,8 @@ jobs:
       - uses: actions/checkout@v5
       - name: Build and test wheels
         run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }}
+      - name: Upload wheels as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pyfory-wheels-containerized-${{ matrix.os }}
+          path: dist/*.whl

.github/workflows/build-containerized-release.yml

@@ -29,12 +29,13 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Bump version
-        run: ./ci/deploy.sh bump_py_version
+        # Pass the tag name from the push (e.g. "v0.12.1b1"); deploy.sh will strip leading "v".
+        run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}"
       - name: Install bazel
         run: ./ci/run_ci.sh install_bazel
       - name: Build and test wheels
         run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release
-      - name: Upload artifacts
+      - name: Upload wheels as artifacts
         uses: actions/upload-artifact@v4
         with:
           name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }}

.github/workflows/build-native-pr.yml

@@ -51,3 +51,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install dist/*.whl
           python -c "import pyfory; print(pyfory.__version__)"
+      - name: Upload wheels as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pyfory-wheels-native-${{ matrix.os }}-${{ matrix.python-version }}
+          path: dist/*.whl

.github/workflows/build-native-release.yml

@@ -25,12 +25,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        os: [macos-13, macos-latest, windows-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
       - uses: actions/checkout@v5
       - name: Bump version
-        run: ./ci/deploy.sh bump_py_version
+        # Pass the tag name from the push so the python package version matches the tag.
+        run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}"
+        shell: bash
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -42,14 +44,27 @@ jobs:
         run: ./ci/run_ci.sh install_bazel_windows
         shell: bash
       - name: Build wheel
+        shell: bash
         run: ./ci/deploy.sh build_pyfory
+        env:
+          GITHUB_REF_NAME: ${{ github.ref_name }}
       - name: Install and verify wheel
         shell: bash
         run: |
           python -m pip install --upgrade pip
           pip install dist/*.whl
-          python -c "import pyfory; print(pyfory.__version__)"
-      - name: Upload artifacts
+          INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)")
+          echo "Installed version: $INSTALLED_VERSION"
+          # Verify version matches the tag
+          EXPECTED_VERSION="${{ github.ref_name }}"
+          EXPECTED_VERSION=$(DEPLOY_QUIET=1 ci/deploy.sh parse_py_version $EXPECTED_VERSION | tail -n1)
+          echo "Expected version: $EXPECTED_VERSION"
+          if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then
+            echo "Version mismatch: Expected $EXPECTED_VERSION but got $INSTALLED_VERSION"
+            exit 1
+          fi
+          echo "Version verification successful"
+      - name: Upload wheels as artifacts
         uses: actions/upload-artifact@v4
         with:
           name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }}

.github/workflows/release-python.yaml

@@ -16,6 +16,7 @@
 # under the License.
 
 name: Publish Python
+run-name: "Python Release: ${{ contains(github.event.workflow_run.name, 'Containerized') && 'Containerized' || 'Native' }} (Run ID: ${{ github.event.workflow_run.id }})"
 
 on:
   workflow_run:
@@ -25,6 +26,7 @@ on:
 permissions:
   contents: read
   id-token: write
+  actions: read  # for accessing workflow run artifacts
 
 jobs:
   publish-wheels:
@@ -35,18 +37,23 @@ jobs:
       - name: Download all wheel artifacts
         uses: actions/download-artifact@v5
         with:
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          pattern: "pyfory-wheels-*"
           path: downloaded_wheels
+          merge-multiple: true
 
       - name: Move wheels to a single directory
         shell: bash
         run: |
           mkdir dist
           find downloaded_wheels -type f -name "*.whl" -exec mv {} dist/ \;
           ls -R dist
+          echo "head_branch: ${{ github.event.workflow_run.head_branch }}"
 
       - name: Publish to TestPyPI
         uses: pypa/gh-action-pypi-publish@release/v1
-        if: startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-')
+        if: startsWith(github.event.workflow_run.head_branch, 'v') && contains(github.event.workflow_run.head_branch, '-')
         with:
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
@@ -56,7 +63,7 @@ jobs:
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
-        if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-')
+        if: startsWith(github.event.workflow_run.head_branch, 'v') && !contains(github.event.workflow_run.head_branch, '-')
         with:
           skip-existing: true
           verify-metadata: false

ci/build_linux_wheels.py

@@ -32,39 +32,12 @@
 import sys
 from typing import List
 
-SCRIPT = r'''set -e
-yum install -y git sudo wget || true
-
-git config --global --add safe.directory /work
-
-# Determine Python versions to test
-if [ "$RELEASE" = "1" ]; then
-    PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313"
-else
-    PYTHON_VERSIONS="cp38-cp38 cp313-cp313"
-fi
-
-ci/run_ci.sh install_bazel
-export PATH="$HOME/.local/bin:$PATH"
-
-# use the python interpreters preinstalled in manylinux
-OLD_PATH=$PATH
-for PY in $PYTHON_VERSIONS; do
-    export PYTHON_PATH="/opt/python/$PY/bin/python"
-    export PATH="/opt/python/$PY/bin:$OLD_PATH"
-    echo "Using $PYTHON_PATH"
-    python -m pip install cython wheel pytest
-    ci/deploy.sh build_pyfory
-
-    latest_wheel=$(ls -t dist/*.whl | head -n1)
-    echo "Attempting to install $latest_wheel"
-    python -m pip install "$latest_wheel"
-    python -c "import pyfory; print(pyfory.__version__)"
-
-    bazel clean --expunge
-done
-export PATH=$OLD_PATH
-'''
+# Define Python version sets directly in the Python script
+RELEASE_PYTHON_VERSIONS = "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313"
+DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313"
+
+# Path to the container build script
+CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh"
 
 DEFAULT_X86_IMAGES = [
     "quay.io/pypa/manylinux2014_x86_64:latest",
@@ -112,26 +85,38 @@ def collect_images_for_arch(arch_normalized: str) -> List[str]:
         raise SystemExit(f"Unsupported arch: {arch_normalized!r}")
     return imgs
 
-def build_docker_cmd(workspace: str, image: str) -> List[str]:
+def build_docker_cmd(workspace: str, image: str, release: bool = False) -> List[str]:
     workspace = os.path.abspath(workspace)
-    return [
+    python_versions = RELEASE_PYTHON_VERSIONS if release else DEFAULT_PYTHON_VERSIONS
+
+    # Get GitHub reference name from environment
+    github_ref_name = os.environ.get("GITHUB_REF_NAME", "")
+
+    cmd = [
         "docker", "run", "-i", "--rm",
-        "-v", f"{workspace}:/work",
-        "-w", "/work",
-        image,
-        "bash", "-s", "--"
+        "-v", f"{workspace}:/work", # (v)olume
+        "-w", "/work",  # (w)orking directory
+        "-e", f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables
+        "-e", f"RELEASE_BUILD={'1' if release else '0'}"
     ]
 
-def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int:
+    # Pass GitHub reference name if available
+    if github_ref_name:
+        cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"])
+
+    cmd.extend([image, "bash", CONTAINER_SCRIPT_PATH])
+    return cmd
+
+def run_for_images(images: List[str], workspace: str, dry_run: bool, release: bool = False) -> int:
     rc_overall = 0
     for image in images:
-        docker_cmd = build_docker_cmd(workspace, image)
+        docker_cmd = build_docker_cmd(workspace, image, release=release)
         printable = " ".join(shlex.quote(c) for c in docker_cmd)
         print(f"+ {printable}")
         if dry_run:
             continue
         try:
-            completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8"))
+            completed = subprocess.run(docker_cmd)
             if completed.returncode != 0:
                 print(f"Container {image} exited with {completed.returncode}", file=sys.stderr)
                 rc_overall = completed.returncode if rc_overall == 0 else rc_overall
@@ -153,8 +138,15 @@ def main() -> int:
         print(f"No images configured for arch {arch}", file=sys.stderr)
         return 2
     workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd())
+
+    # Check if the container script exists
+    script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH)
+    if not os.path.exists(script_path):
+        print(f"Container script not found at {script_path}", file=sys.stderr)
+        return 2
+
     print(f"Selected images for arch {args.arch}: {images}")
-    return run_for_images(images, workspace, args.dry_run)
+    return run_for_images(images, workspace, args.dry_run, release=args.release)
 
 if __name__ == "__main__":
     sys.exit(main())

ci/deploy.sh

@@ -19,7 +19,9 @@
 
 
 # Print commands and their arguments as they are executed.
-set -x
+if [ "${DEPLOY_QUIET:-0}" != "1" ]; then
+    set -x
+fi
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -44,16 +46,27 @@ bump_java_version() {
   python "$ROOT/ci/release.py" bump_version -l java -version "$1"
 }
 
-bump_py_version() {
+# Replicates the behavior of _update_python_version in ci/release.py
+parse_py_version() {
   local version="$1"
   if [ -z "$version" ]; then
     # Get the latest tag from the current Git repository
     version=$(git describe --tags --abbrev=0)
-    # Check if the tag starts with 'v' and strip it
-    if [[ $version == v* ]]; then
-      version="${version:1}"
-    fi
   fi
+  # Check if the tag starts with 'v' and strip it
+  if [[ $version == v* ]]; then
+    version="${version:1}"
+  fi
+  version="${version//-alpha/a}"
+  version="${version//-beta/b}"
+  version="${version//-rc/rc}"
+  version="${version//-/}"
+  echo "$version"
+}
+
+bump_py_version() {
+  local version
+  version=$(parse_py_version "$1")
   python "$ROOT/ci/release.py" bump_version -l python -version "$version"
 }
 
@@ -81,29 +94,42 @@ build_pyfory() {
     echo "MACOS_VERSION: $MACOS_VERSION"
     if [[ "$MACOS_VERSION" == "13"* ]]; then
       export MACOSX_DEPLOYMENT_TARGET=10.13
-      $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir=../dist
+      $PYTHON_CMD setup.py bdist_wheel --plat-name macosx_10_13_x86_64 --dist-dir="$ROOT/dist"
     else
-      $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist
+      $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
     fi
+  elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
+
+    # Windows tends to drop alpha/beta markers - force it through setup.cfg
+    if [ -n "$GITHUB_REF_NAME" ]; then
+      version=$(parse_py_version "$GITHUB_REF_NAME")
+      echo "Using version from GITHUB_REF_NAME: $version"
+      echo "[metadata]" > setup.cfg
+      echo "version = $version" >> setup.cfg
+    fi
+
+    $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
+    # Clean up
+    rm setup.cfg
   else
-    $PYTHON_CMD setup.py bdist_wheel --dist-dir=../dist
+    $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
   fi
 
   if [ -n "$PLAT" ]; then
     # In manylinux container, repair the wheel to embed shared libraries
     # and rename the wheel with the manylinux tag.
     PYARROW_LIB_DIR=$($PYTHON_CMD -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))')
     export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH"
-    auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w ../dist/
-    rm ../dist/pyfory-*-linux_*.whl
+    auditwheel repair "$ROOT/dist"/pyfory-*-linux_*.whl --plat "$PLAT" --exclude '*arrow*' --exclude '*parquet*' --exclude '*numpy*' -w "$ROOT/dist"
+    rm "$ROOT/dist"/pyfory-*-linux_*.whl
   elif [[ "$OSTYPE" == "darwin"* ]]; then
     echo "Skip macos wheel repair"
   elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
     echo "Skip windows wheel repair"
   fi
 
   echo "Wheels for $PYTHON_CMD:"
-  ls -l ../dist
+  ls -l "$ROOT/dist"
   popd
 }
 

ci/release.py

@@ -245,6 +245,7 @@ def _update_python_version(lines, v: str):
             v = v.replace("-alpha", "a")
             v = v.replace("-beta", "b")
             v = v.replace("-rc", "rc")
+            v = v.replace("-", "")
             lines[index] = f'__version__ = "{v}"\n'
             break