ci: rework tests to break out ios/android tests (#2519)

henryiii · Copilot · mayeut · web-flow · commit cade3864dec5 · 2025-08-13T08:52:27.000-04:00
* ci: rework tests to break out ios/android tests Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Apply suggestions from code review * ci: fix check for wheels Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Update .github/workflows/test.yml * tests: pass via flag instead of envvar Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: apply to serial/parallel Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: join together marks Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: join together marks Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: break up azure job Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: use matrix for azure pipelines Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Try installing Java on macos as well * Revert "Try installing Java on macos as well" This reverts commit aedf0f5. * Try to get the most basic android test running first * Rename platform to test_select, move headers in run_tests Maybe it's a personal thing, but the header location was really making me have to think more than I needed to look at that file! * Only run docker unit tests on machines that are testing linux * Update the flag name in azure-pipelines too * Update azure-pipelines.yml --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> Co-authored-by: Joe Rickerby <joerick@mac.com>
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -47,19 +47,44 @@ jobs:
       run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github
 
   test:
-    name: Test on ${{ matrix.os }} (${{ matrix.python_version }})
+    name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) ${{ matrix.test_select }}
     needs: lint
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15]
-        python_version: ['3.13']
         include:
+        # Min Python
         - os: ubuntu-latest
           python_version: '3.11'
+        # Max Python
         - os: ubuntu-latest
           python_version: '3.14'
+        - os: ubuntu-latest
+          python_version: '3.13'
+          test_select: android
+        - os: ubuntu-24.04-arm
+          python_version: '3.13'
+        - os: windows-latest
+          python_version: '3.13'
+        - os: windows-11-arm
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+        - os: macos-15
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-15
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-13
+          python_version: '3.13'
+          test_select: android
+        - os: macos-15
+          python_version: '3.13'
+          test_select: android
     timeout-minutes: 180
     steps:
     - uses: actions/checkout@v5
@@ -129,10 +154,12 @@ jobs:
         output-dir: wheelhouse
       env:
         CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-        CIBW_BUILD_FRONTEND: 'build[uv]'
+        CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }}
+        CIBW_PLATFORM: ${{ matrix.test_select }}
 
     - name: Run a sample build (GitHub Action, only)
       uses: ./
+      if: matrix.test_select == ''
       with:
         package-dir: sample_proj
         output-dir: wheelhouse_only
@@ -152,6 +179,8 @@ jobs:
 
     - name: Run a sample build (GitHub Action, config-file)
       uses: ./
+      env:
+        CIBW_PLATFORM: ${{ matrix.test_select }}
       with:
         package-dir: sample_proj
         output-dir: wheelhouse_config_file
@@ -161,17 +190,22 @@ jobs:
       shell: bash
       run: |
         test $(find wheelhouse -name '*.whl' | wc -l) -ge 1
-        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
         test $(find wheelhouse_config_file -name '*.whl' | wc -l) -eq 1
 
+    - name: Check Action artifacts (native build only)
+      if: matrix.test_select == ''
+      shell: bash
+      run: |
+        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
+
     - uses: actions/upload-artifact@v4
       with:
         name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
         path: wheelhouse/*.whl
 
     - name: Test cibuildwheel
       run: |
-        uv run --no-sync bin/run_tests.py ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
+        uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
 
   emulated-archs:
     name: Get qemu emulated architectures
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -16,62 +16,61 @@ pr:
       - noxfile.py
 
 jobs:
-- job: linux_311
+- job: tests
+  strategy:
+    matrix:
+      linux_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      android_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      ios_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "ios"
+      android_macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      windows_311:
+        imageName: "windows-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
   timeoutInMinutes: 180
-  pool: {vmImage: 'ubuntu-latest'}
+  pool:
+    vmImage: $(imageName)
+
   steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: '3.11'
+        versionSpec: $(pythonVersion)
+
     - task: JavaToolInstaller@0
+      condition: and(eq(variables['testSelect'], 'android'), eq(variables['Agent.OS'], 'Linux'))
       inputs:
         versionSpec: '17'
         jdkArchitectureOption: 'x64'
         jdkSourceOption: 'PreInstalled'
-    - bash: |
-        docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
 
-- job: macos_311
-  pool: {vmImage: 'macOS-latest'}
-  timeoutInMinutes: 120
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.11'
-    - bash: |
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
+    - bash: docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
+      condition: and(eq(variables['imageName'], 'ubuntu-latest'), eq(variables['testSelect'], 'native'))
+      displayName: 'Install binfmt on Linux'
+
+    - bash: python -m pip install -U pip && python -m pip install -e. --group test
+      displayName: 'Update pip and install cibuildwheel --group test'
+
+    - bash: echo "##vso[task.setvariable variable=CIBW_ENABLE;]all"
+      condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
+      displayName: Set CIBW_ENABLE to all (main branch)
 
-- job: windows_311
-  pool: {vmImage: 'windows-latest'}
-  timeoutInMinutes: 180
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.11'
     - bash: |
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
+        echo "CIBW_ENABLE = $CIBW_ENABLE"
+        python ./bin/run_tests.py --test-select $(testSelect)
+      displayName: 'Run tests'
diff --git a/bin/run_tests.py b/bin/run_tests.py
@@ -27,6 +27,12 @@
         default=default_cpu_count,
         help="number of processes to use for testing",
     )
+    parser.add_argument(
+        "--test-select",
+        choices={"all", "native", "android", "ios", "pyodide"},
+        default="all",
+        help="Either 'native' or 'android'/'ios'/'pyodide'",
+    )
     args = parser.parse_args()
 
     # move cwd to the project root
@@ -50,48 +56,65 @@
         )
 
     # unit tests
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
     unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]
 
-    if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux":
+    if (
+        sys.platform.startswith("linux")
+        and os.environ.get("CIBW_PLATFORM", "linux") == "linux"
+        and args.test_select in ["all", "native"]
+    ):
         # run the docker unit tests only on Linux
         unit_test_args += ["--run-docker"]
 
         if args.run_podman:
             unit_test_args += ["--run-podman"]
 
+    subprocess.run(unit_test_args, check=True)
+
     print(
-        "\n\n================================== UNIT TESTS ==================================",
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
         flush=True,
     )
-    subprocess.run(unit_test_args, check=True)
+
+    match args.test_select:
+        case "all":
+            marks = []
+        case "native":
+            marks = ["not pyodide", "not android", "not ios"]
+        case mark:
+            marks = [f"{mark}"]
 
     # Run the serial integration tests without multiple processes
     serial_integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
         "-m",
-        "serial",
+        f"{' and '.join(['serial', *marks])}",
         "-x",
         "--durations",
         "0",
         "--timeout=2400",
         "test",
         "-vv",
     ]
+
+    subprocess.run(serial_integration_test_args, check=True)
+
     print(
-        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
         flush=True,
     )
-    subprocess.run(serial_integration_test_args, check=True)
-
-    # Non-serial integration tests
     integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
         "-m",
-        "not serial",
+        f"{' and '.join(['not serial', *marks])}",
         f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
@@ -104,8 +127,4 @@
     if sys.platform.startswith("linux") and args.run_podman:
         integration_test_args += ["--run-podman"]
 
-    print(
-        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
-        flush=True,
-    )
     subprocess.run(integration_test_args, check=True)
diff --git a/test/test_android.py b/test/test_android.py
@@ -90,8 +90,17 @@ def test_android_home(tmp_path, capfd):
     assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err
 
 
-# Can fail to setup
+# the first build can fail to setup - mark as flaky, and serial to make sure it runs first
+@pytest.mark.serial
 @pytest.mark.flaky(reruns=2)
+def test_expected_wheels(tmp_path):
+    new_c_project().generate(tmp_path)
+    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
+    assert wheels == expected_wheels(
+        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
+    )
+
+
 def test_frontend_good(tmp_path):
     new_c_project().generate(tmp_path)
     wheels = cibuildwheel_run(
@@ -112,14 +121,6 @@ def test_frontend_bad(frontend, tmp_path, capfd):
     assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err
 
 
-def test_expected_wheels(tmp_path):
-    new_c_project().generate(tmp_path)
-    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
-    assert wheels == expected_wheels(
-        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
-    )
-
-
 @needs_emulator
 def test_archs(tmp_path, capfd):
     new_c_project().generate(tmp_path)
