Add support for prebuilt wheels

Author: T-Dynamos

pythonforandroid/build.py

@@ -21,7 +21,7 @@
 from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
 from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore)
 from pythonforandroid.pythonpackage import get_package_name
-from pythonforandroid.recipe import CythonRecipe, Recipe
+from pythonforandroid.recipe import CythonRecipe, Recipe, PyProjectRecipe
 from pythonforandroid.recommendations import (
     check_ndk_version, check_target_api, check_ndk_api,
     RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
@@ -101,6 +101,12 @@ class Context:
 
     java_build_tool = 'auto'
 
+    skip_prebuilt = False
+
+    extra_index_urls = []
+
+    use_prebuilt_version_for = []
+
     @property
     def packages_path(self):
         '''Where packages are downloaded before being unpacked'''
@@ -667,7 +673,17 @@ def is_wheel_platform_independent(whl_name):
     return all(tag.platform == "any" for tag in tags)
 
 
-def process_python_modules(ctx, modules):
+def is_wheel_compatible(whl_name, arch, ctx):
+    name, version, build, tags = parse_wheel_filename(whl_name)
+    supported_tags = PyProjectRecipe.get_wheel_platform_tag(None, arch.arch, ctx=ctx)
+    supported_tags.append("any")
+    result = all(tag.platform in supported_tags for tag in tags)
+    if not result:
+        warning(f"Incompatible module : {whl_name}")
+    return result
+
+
+def process_python_modules(ctx, modules, arch):
     """Use pip --dry-run to resolve dependencies and filter for pure-Python packages
     """
     modules = list(modules)
@@ -702,6 +718,7 @@ def process_python_modules(ctx, modules):
 
     # setup hostpython recipe
     env = environ.copy()
+    host_recipe = None
     try:
         host_recipe = Recipe.get_recipe("hostpython3", ctx)
         _python_path = host_recipe.get_path_to_python()
@@ -713,11 +730,28 @@ def process_python_modules(ctx, modules):
         # hostpython3 non available so we use system pip (like in tests)
         pip = sh.Command("pip")
 
+    # add platform tags
+    platforms = []
+    tags = PyProjectRecipe.get_wheel_platform_tag(None, arch.arch, ctx=ctx)
+    for tag in tags:
+        platforms.append(f"--platform={tag}")
+
+    if host_recipe is not None:
+        platforms.extend(["--python-version", host_recipe.version])
+    else:
+        # tests?
+        platforms.extend(["--python-version", "3.13.4"])
+
+    indices = []
+    # add extra index urls
+    for index in ctx.extra_index_urls:
+        indices.extend(["--extra-index-url", index])
     try:
         shprint(
             pip, 'install', *modules,
             '--dry-run', '--break-system-packages', '--ignore-installed',
-            '--report', path, '-q', _env=env
+            '--disable-pip-version-check', '--only-binary=:all:',
+            '--report', path, '-q', *platforms, *indices, _env=env
         )
     except Exception as e:
         warning(f"Auto module resolution failed: {e}")
@@ -751,7 +785,9 @@ def process_python_modules(ctx, modules):
         filename = basename(module["download_info"]["url"])
         pure_python = True
 
-        if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)):
+        if (
+                filename.endswith(".whl") and not is_wheel_compatible(filename, arch, ctx)
+        ):
             any_not_pure_python = True
             pure_python = False
 
@@ -793,7 +829,7 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,
 
     info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))
 
-    modules = process_python_modules(ctx, modules)
+    modules = process_python_modules(ctx, modules, arch)
 
     modules = [m for m in modules if ctx.not_has_package(m, arch)]
 

pythonforandroid/recipe.py

@@ -923,8 +923,7 @@ def real_hostpython_location(self):
         if host_name == 'hostpython3':
             return self._host_recipe.python_exe
         else:
-            python_recipe = self.ctx.python_recipe
-            return 'python{}'.format(python_recipe.version)
+            return 'python{}'.format(self.ctx.python_recipe.version)
 
     @property
     def hostpython_location(self):
@@ -1248,6 +1247,55 @@ class PyProjectRecipe(PythonRecipe):
     extra_build_args = []
     call_hostpython_via_targetpython = False
 
+    def get_pip_name(self):
+        name_str = self.name
+        if self.name not in self.ctx.use_prebuilt_version_for:
+            name_str += f"=={self.version}"
+        return name_str
+
+    def get_pip_install_args(self, arch):
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        opts = [
+            "install",
+            self.get_pip_name(),
+            "--disable-pip-version-check",
+            "--python-version",
+            python_recipe.version,
+            "--only-binary=:all:",
+        ]
+        # add platform tags
+        tags = self.get_wheel_platform_tag(arch.arch)
+        for tag in tags:
+            opts.append(f"--platform={tag}")
+
+        # add extra index urls
+        for index in self.ctx.extra_index_urls:
+            opts.extend(["--extra-index-url", index])
+
+        return opts
+
+    def lookup_prebuilt(self, arch):
+        pip_options = self.get_pip_install_args(arch)
+        # do not install
+        pip_options.extend(["--dry-run"])
+        pip_env = self.get_hostrecipe_env()
+        try:
+            shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
+        except Exception as e:
+            warning(f"Lookup fail result: {e}")
+            return False
+        return True
+
+    def check_prebuilt(self, arch, msg=""):
+        if self.ctx.skip_prebuilt:
+            return False
+
+        if self.lookup_prebuilt(arch):
+            if msg != "":
+                info(f"Prebuilt pip wheel found, {msg}")
+            return True
+        return
+
     def get_recipe_env(self, arch, **kwargs):
         # Custom hostpython
         self.ctx.python_recipe.python_exe = join(
@@ -1259,24 +1307,40 @@ def get_recipe_env(self, arch, **kwargs):
 
         with open(build_opts, "w") as file:
             file.write("[bdist_wheel]\nplat_name={}".format(
-                self.get_wheel_platform_tag(arch)
+                self.get_wheel_platform_tag(arch.arch)[0]
             ))
             file.close()
 
         env["DIST_EXTRA_CONFIG"] = build_opts
         return env
 
-    def get_wheel_platform_tag(self, arch):
+    def get_wheel_platform_tag(self, arch, ctx=None):
+        if ctx is None:
+            ctx = self.ctx
         # https://peps.python.org/pep-0738/#packaging
         # official python only supports 64 bit:
         # android_21_arm64_v8a
         # android_21_x86_64
-        return f"android_{self.ctx.ndk_api}_" + {
-            "arm64-v8a": "arm64_v8a",
-            "x86_64": "x86_64",
-            "armeabi-v7a": "arm",
-            "x86": "i686",
-        }[arch.arch]
+        _suffix = {
+            "arm64-v8a": ["arm64_v8a", "aarch64"],
+            "x86_64": ["x86_64"],
+            "armeabi-v7a": ["arm"],
+            "x86": ["i686"],
+        }[arch]
+        return [f"android_{ctx.ndk_api}_" + _ for _ in _suffix]
+
+    def install_prebuilt_wheel(self, arch):
+        info("Installing prebuilt built wheel")
+        destination = self.ctx.get_python_install_dir(arch.arch)
+        pip_options = self.get_pip_install_args(arch)
+        pip_options.extend(["--target", destination])
+        pip_options.append("--upgrade")
+        pip_env = self.get_hostrecipe_env()
+        try:
+            shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
+        except Exception:
+            return False
+        return True
 
     def install_wheel(self, arch, built_wheels):
         with patch_wheel_setuptools_logging():
@@ -1287,7 +1351,7 @@ def install_wheel(self, arch, built_wheels):
         # Fix wheel platform tag
         wheel_tag = wheel_tags(
             _wheel,
-            platform_tags=self.get_wheel_platform_tag(arch),
+            platform_tags=self.get_wheel_platform_tag(arch.arch)[0],
             remove=True,
         )
         selected_wheel = join(built_wheel_dir, wheel_tag)
@@ -1305,6 +1369,11 @@ def install_wheel(self, arch, built_wheels):
             wf.close()
 
     def build_arch(self, arch):
+        if self.check_prebuilt(arch, "skipping build_arch") is not None:
+            result = self.install_prebuilt_wheel(arch)
+            if result:
+                return
+            warning("Failed to install prebuilt wheel, falling back to build_arch")
 
         build_dir = self.get_build_dir(arch.arch)
         if not (isfile(join(build_dir, "pyproject.toml")) or isfile(join(build_dir, "setup.py"))):

pythonforandroid/recipes/hostpython3/__init__.py

@@ -6,7 +6,7 @@
 from os.path import join
 
 from packaging.version import Version
-from pythonforandroid.logger import shprint
+from pythonforandroid.logger import shprint, error
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import (
     BuildInterruptingException,
@@ -48,6 +48,16 @@ class HostPython3Recipe(Recipe):
 
     patches = ["fix_ensurepip.patch"]
 
+    # apply version guard
+    def download(self):
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        if python_recipe.version != self.version:
+            error(
+                f"python3 should have same version as hostpython3, {python_recipe.version} != {self.version}"
+            )
+            exit(1)
+        super().download()
+
     @property
     def _exe_name(self):
         '''

pythonforandroid/recipes/python3/__init__.py

@@ -55,36 +55,45 @@ class Python3Recipe(TargetPythonRecipe):
     '''
 
     version = '3.14.2'
-    _p_version = Version(version)
     url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
     name = 'python3'
 
-    patches = [
-        'patches/pyconfig_detection.patch',
-        'patches/reproducible-buildinfo.diff',
-    ]
+    @property
+    def _p_version(self):
+        # as version is dynamic
+        return Version(self.version)
 
-    if _p_version.major == 3 and _p_version.minor == 7:
-        patches += [
-            'patches/py3.7.1_fix-ctypes-util-find-library.patch',
-            'patches/py3.7.1_fix-zlib-version.patch',
+    @property
+    def patches(self):
+        patches = [
+            'patches/pyconfig_detection.patch',
+            'patches/reproducible-buildinfo.diff',
         ]
+        _p_version = self._p_version
 
-    if 8 <= _p_version.minor <= 10:
-        patches.append('patches/py3.8.1.patch')
+        if _p_version.major == 3 and _p_version.minor == 7:
+            patches += [
+                'patches/py3.7.1_fix-ctypes-util-find-library.patch',
+                'patches/py3.7.1_fix-zlib-version.patch',
+            ]
 
-    if _p_version.minor >= 11:
-        patches.append('patches/cpython-311-ctypes-find-library.patch')
+        if 8 <= _p_version.minor <= 10:
+            patches.append('patches/py3.8.1.patch')
 
-    if _p_version.minor >= 14:
-        patches.append('patches/3.14_armv7l_fix.patch')
-        patches.append('patches/3.14_fix_remote_debug.patch')
+        if _p_version.minor >= 11:
+            patches.append('patches/cpython-311-ctypes-find-library.patch')
 
-    if shutil.which('lld') is not None:
-        if _p_version.minor == 7:
-            patches.append("patches/py3.7.1_fix_cortex_a8.patch")
-        elif _p_version.minor >= 8:
-            patches.append("patches/py3.8.1_fix_cortex_a8.patch")
+        if _p_version.minor >= 14:
+            patches.append('patches/3.14_armv7l_fix.patch')
+            patches.append('patches/3.14_fix_remote_debug.patch')
+
+        if shutil.which('lld') is not None:
+            if _p_version.minor == 7:
+                patches.append("patches/py3.7.1_fix_cortex_a8.patch")
+            elif _p_version.minor >= 8:
+                patches.append("patches/py3.8.1_fix_cortex_a8.patch")
+
+        return patches
 
     depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
     # those optional depends allow us to build python compression modules:
@@ -116,11 +125,6 @@ class Python3Recipe(TargetPythonRecipe):
         'ac_cv_header_bzlib_h=no',
     ]
 
-    if _p_version.minor >= 11:
-        configure_args.extend([
-            '--with-build-python={python_host_bin}',
-        ])
-
     '''The configure arguments needed to build the python recipe. Those are
     used in method :meth:`build_arch` (if not overwritten like python3's
     recipe does).
@@ -317,6 +321,9 @@ def add_flags(include_flags, link_dirs, link_libs):
         env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
         add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')
 
+        if self._p_version.minor >= 11:
+            self.configure_args.append('--with-build-python={python_host_bin}')
+
         if self._p_version.minor >= 13 and self.disable_gil:
             self.configure_args.append("--disable-gil")