fix PYTHONPATH hacks (#3301)

* fix PYTHONPATH hacks

* fix android recipe

* fix numpy and matplotlib build

* no shebang patching required now

* fix kivy master build

* fix numpy build

* more wrappers for meson recipe

* flake8 fix

* add back pandas host preq

* fix bug in logger

* add back cython for pandas

Author: T-Dynamos

pythonforandroid/build.py

@@ -728,10 +728,6 @@ def process_python_modules(ctx, modules, arch):
     host_recipe = None
     try:
         host_recipe = Recipe.get_recipe("hostpython3", ctx)
-        _python_path = host_recipe.get_path_to_python()
-        libdir = glob.glob(join(_python_path, "build", "lib*"))
-        env['PYTHONPATH'] = host_recipe.site_dir + ":" + join(
-            _python_path, "Modules") + ":" + (libdir[0] if libdir else "")
         pip = host_recipe.pip
     except Exception:
         # hostpython3 is unavailable, so fall back to system pip

pythonforandroid/logger.py

@@ -135,6 +135,13 @@ def shprint(command, *args, **kwargs):
     kwargs["_out_bufsize"] = 1
     kwargs["_err_to_out"] = True
     kwargs["_bg"] = True
+
+    # silent mode
+    silent = kwargs.get("silent", False)
+    kwargs.pop("silent", False)
+    if silent:
+        kwargs["_out"] = None
+
     is_critical = kwargs.pop('_critical', False)
     tail_n = kwargs.pop('_tail', None)
     full_debug = False
@@ -190,6 +197,11 @@ def shprint(command, *args, **kwargs):
                 Err_Style.RESET_ALL, ' ', width=(columns - 1)))
             stdout.flush()
     except sh.ErrorReturnCode as err:
+        if silent:
+            if is_critical:
+                exit(1)
+            else:
+                raise
         if need_closing_newline:
             stdout.write('{}\r{:>{width}}\r'.format(
                 Err_Style.RESET_ALL, ' ', width=(columns - 1)))

pythonforandroid/meson_python.py

@@ -0,0 +1,76 @@
+import sys
+import json
+import subprocess
+import importlib.util
+from os.path import join
+from glob import glob
+
+
+class C:
+    TARGET_PYTHON_PREFIX = globals().get("TARGET_PYTHON_PREFIX")
+    PYTHON_MAJOR_VERSION = globals().get("PYTHON_MAJOR_VERSION", sys.version_info.major)
+    PYTHON_MINOR_VERSION = globals().get("PYTHON_MINOR_VERSION", sys.version_info.minor)
+    PLATFORM_TAG = globals().get("PLATFORM_TAG", sys.platform)
+    PYTHON_SUFFIX = globals().get("PYTHON_SUFFIX", "")
+
+
+def handle_python_info():
+
+    prefix = C.TARGET_PYTHON_PREFIX
+
+    sysconfig_file = glob(join(prefix, "lib/python3.14/_sysconfigdata*.py"))[0]
+
+    spec = importlib.util.spec_from_file_location("_android_cfg", sysconfig_file)
+    cfg = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(cfg)
+
+    ver_python = f"python{C.PYTHON_MAJOR_VERSION}.{C.PYTHON_MINOR_VERSION}"
+
+    site_path = join(prefix, f"lib/{ver_python}/site-packages")
+
+    android_paths = {
+        "stdlib": join(prefix, "lib"),
+        "platstdlib": join(prefix, "lib"),
+        "purelib": site_path,
+        "platlib": site_path,
+        "include": join(prefix, f"include/{ver_python}/"),
+        "platinclude": join(prefix, f"include/{ver_python}/"),
+        "scripts": join(prefix, "bin"),
+        "data": prefix,
+    }
+
+    print(
+        json.dumps(
+            {
+                "variables": cfg.build_time_vars,
+                "paths": android_paths,
+                "sysconfig_paths": android_paths,
+                "install_paths": android_paths,
+                "version": f"{C.PYTHON_MAJOR_VERSION}.{C.PYTHON_MINOR_VERSION}",
+                "platform": C.PLATFORM_TAG,
+                "is_pypy": False,
+                "is_venv": False,
+                "link_libpython": True,
+                "suffix": C.PYTHON_SUFFIX,
+                "limited_api_suffix": ".abi3.so",
+                "is_freethreaded": False,
+            }
+        )
+    )
+
+
+WHITELIST = {
+    "python_info.py": handle_python_info,
+}
+
+
+args = sys.argv[1:]
+
+if args:
+    cmd = args[0]
+    name = cmd.split("/")[-1]
+
+    if name in WHITELIST:
+        sys.exit(WHITELIST[name]())
+
+sys.exit(subprocess.run([sys.executable] + args).returncode)

pythonforandroid/recipe.py

@@ -5,13 +5,15 @@
 from re import match
 
 import sh
+import subprocess
 import shutil
 import fnmatch
 import zipfile
 import urllib.request
 from urllib.request import urlretrieve
-from os import listdir, unlink, environ, curdir, walk
+from os import listdir, unlink, environ, curdir, walk, chmod
 from sys import stdout
+from packaging.version import Version
 from multiprocessing import cpu_count
 import time
 try:
@@ -966,7 +968,8 @@ def patch_shebangs(self, path, original_bin):
         # set correct shebang
         for file in listdir(path):
             _file = join(path, file)
-            self.patch_shebang(_file, original_bin)
+            if isfile(_file):
+                self.patch_shebang(_file, original_bin)
 
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         if self._host_recipe is None:
@@ -976,11 +979,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         # Set the LANG, this isn't usually important but is a better default
         # as it occasionally matters how Python e.g. reads files
         env['LANG'] = "en_GB.UTF-8"
-
-        # Binaries made by packages installed by pip
-        self.patch_shebangs(self._host_recipe.local_bin, self._host_recipe.python_exe)
         env["PATH"] = self._host_recipe.local_bin + ":" + self._host_recipe.site_bin + ":" + env["PATH"]
-
         host_env = self.get_hostrecipe_env(arch)
         env['PYTHONPATH'] = host_env["PYTHONPATH"]
 
@@ -1031,10 +1030,8 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
 
     def get_hostrecipe_env(self, arch=None):
         env = environ.copy()
-        _python_path = self._host_recipe.get_path_to_python()
-        libdir = glob.glob(join(_python_path, "build", "lib*"))
-        env['PYTHONPATH'] = self._host_recipe.site_dir + ":" + join(
-            _python_path, "Modules") + ":" + (libdir[0] if libdir else "")
+        env['PYTHONPATH'] = ''
+        env['HOME'] = '/tmp'
         return env
 
     @property
@@ -1063,14 +1060,10 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
         pip_options = [
             "install",
             *packages,
-            "--target", self._host_recipe.site_dir, "--python-version",
-            self.ctx.python_recipe.version,
-            # Don't use sources, instead wheels
-            "--only-binary=:all:",
+            "-q",
         ]
         if force_upgrade:
             pip_options.append("--upgrade")
-        # Use system's pip
         pip_env = self.get_hostrecipe_env()
         shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
 
@@ -1284,7 +1277,7 @@ def lookup_prebuilt(self, arch):
         pip_options.extend(["--dry-run", "-q"])
         pip_env = self.get_hostrecipe_env()
         try:
-            shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
+            shprint(self._host_recipe.pip, *pip_options, _env=pip_env, silent=True)
         except Exception:
             return False
         return True
@@ -1301,9 +1294,6 @@ def check_prebuilt(self, arch, msg=""):
         return False
 
     def get_recipe_env(self, arch, **kwargs):
-        # Custom hostpython
-        self.ctx.python_recipe.python_exe = join(
-            self.ctx.python_recipe.get_build_dir(arch), "android-build", "python3")
         env = super().get_recipe_env(arch, **kwargs)
         build_dir = self.get_build_dir(arch)
         ensure_dir(build_dir)
@@ -1316,6 +1306,8 @@ def get_recipe_env(self, arch, **kwargs):
             file.close()
 
         env["DIST_EXTRA_CONFIG"] = build_opts
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        env["INCLUDEPY"] = python_recipe.include_root(arch.arch)
         return env
 
     @staticmethod
@@ -1387,18 +1379,14 @@ def build_arch(self, arch):
         if not (isfile(join(build_dir, "pyproject.toml")) or isfile(join(build_dir, "setup.py"))):
             warning("Skipping build because it does not appear to be a Python project.")
             return
-
         self.install_hostpython_prerequisites(
             packages=["build[virtualenv]", "pip", "setuptools", "patchelf"] + self.hostpython_prerequisites
         )
-        self.patch_shebangs(self._host_recipe.site_bin, self.real_hostpython_location)
 
         env = self.get_recipe_env(arch, with_flags_in_cc=True)
         # make build dir separately
         sub_build_dir = join(build_dir, "p4a_android_build")
         ensure_dir(sub_build_dir)
-        # copy hostpython to built python to ensure correct selection of libs and includes
-        shprint(sh.cp, self.real_hostpython_location, self.ctx.python_recipe.python_exe)
 
         build_args = [
             "-m",
@@ -1411,7 +1399,7 @@ def build_arch(self, arch):
         built_wheels = []
         with current_directory(build_dir):
             shprint(
-                sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env
+                sh.Command(self.real_hostpython_location), *build_args, _env=env
             )
             built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")]
         self.install_wheel(arch, built_wheels)
@@ -1421,7 +1409,7 @@ class MesonRecipe(PyProjectRecipe):
     '''Recipe for projects which uses meson as build system'''
 
     meson_version = "1.4.0"
-    ninja_version = "1.11.1.1"
+    pybind_version = "3.3.0"
 
     skip_python = False
     '''If true, skips all Python build and installation steps.
@@ -1430,10 +1418,98 @@ class MesonRecipe(PyProjectRecipe):
     def sanitize_flags(self, *flag_strings):
         return " ".join(flag_strings).strip().split(" ")
 
+    def get_wrapper_dir(self, arch):
+        return join(self.get_build_dir(arch.arch), "p4a_wrappers")
+
+    def write_wrapper(self, arch, name, content):
+        wrapper_dir = self.get_wrapper_dir(arch)
+        ensure_dir(wrapper_dir)
+        wrapper_path = join(wrapper_dir, name)
+        with open(wrapper_path, "w") as f:
+            f.write(content)
+        chmod(wrapper_path, 0o755)
+        return wrapper_path
+
+    def get_python_wrapper(self, arch):
+        """
+        Meson Python introspection runs on the host interpreter, but the
+        target Python (Android) cannot be executed on the build machine.
+
+        We therefore run host Python and override sysconfig data to emulate
+        the target Android Python environment during Meson introspection.
+        """
+        python_recipe = Recipe.get_recipe('python3', self.ctx)
+        target_prefix = python_recipe.get_python_root(arch)
+        python_file = join(self.ctx.root_dir, 'meson_python.py')
+        _arch = {
+            "arm64-v8a": ["aarch64"],
+            "x86_64": ["x86_64"],
+            "armeabi-v7a": ["arm"],
+            "x86": ["i686"],
+        }[arch.arch][0]
+
+        # Real values pulled from android
+        # PYTHON_MAJOR_VERSION -> 3
+        # PYTHON_MINOR_VERSION -> 14
+        # PLATFORM_TAG eg -> 'android-24-arm64_v8a'
+        # PYTHON_SUFFIX eg -> '.cpython-314-aarch64-linux-android.so'
+
+        _p_version = Version(python_recipe.version)
+        file_data = f"#!{self.real_hostpython_location}"
+        file_data += f"\nTARGET_PYTHON_PREFIX='{target_prefix}'"
+        file_data += f"\nPYTHON_MAJOR_VERSION='{_p_version.major}'"
+        file_data += f"\nPYTHON_MINOR_VERSION='{_p_version.minor}'"
+        file_data += f"\nPLATFORM_TAG='{self.get_wheel_platform_tags(arch.arch, self.ctx)[0]}'"
+        file_data += f"\nPYTHON_SUFFIX='.cpython-{_p_version.major}{_p_version.minor}-{_arch}-linux-android.so'"
+
+        with open(python_file, "r") as f:
+            file_data += "\n" + f.read()
+
+        return self.write_wrapper(arch, "python", file_data)
+
+    def get_config_wrappers(self, arch, w_type: str):
+        wrapper_name = ""
+        version = ""
+        include_path = ""
+
+        if w_type == "pybind11":
+            wrapper_name = "pybind11-config"
+            include_path = join(self._host_recipe.site_dir, "pybind11/include")
+
+            version = None
+            try:
+                command = [self._host_recipe.real_hostpython_location, "-c", "import pybind11; print(pybind11.__version__)"]
+                version = subprocess.check_output(command).decode('utf-8').strip()
+            except Exception:
+                warning("Unable to get pybind11 version")
+            if version is None:
+                version = self.pybind_version
+
+        elif w_type == "numpy":
+            wrapper_name = "numpy-config"
+            recipe = Recipe.get_recipe("numpy", self.ctx)
+            include_path = recipe.get_include(arch)
+            version = recipe.version
+        else:
+            raise ValueError(f"Unknown wrapper type: {w_type}")
+
+        content = (
+            f"#!/bin/sh\n"
+            f"if [ \"$1\" = \"--version\" ]; then\n"
+            f"    echo '{version}'\n"
+            f"else\n"
+            f"    echo '-I{include_path}'\n"
+            f"fi\n"
+        )
+        return self.write_wrapper(arch, wrapper_name, content)
+
     def get_recipe_meson_options(self, arch):
         env = self.get_recipe_env(arch, with_flags_in_cc=True)
         return {
             "binaries": {
+                "pybind11-config": self.get_config_wrappers(arch, "pybind11"),
+                "numpy-config": self.get_config_wrappers(arch, "numpy"),
+                "python": self.get_python_wrapper(arch),
                 "c": arch.get_clang_exe(with_target=True),
                 "cpp": arch.get_clang_exe(with_target=True, plus_plus=True),
                 "ar": self.ctx.ndk.llvm_ar,
@@ -1504,13 +1580,18 @@ def build_arch(self, arch):
         self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file))
         # ensure ninja and meson
         for dep in [
-            "ninja=={}".format(self.ninja_version),
+            "ninja",
             "meson=={}".format(self.meson_version),
         ]:
             if dep not in self.hostpython_prerequisites:
                 self.hostpython_prerequisites.append(dep)
+
         if not self.skip_python:
             super().build_arch(arch)
+        else:
+            self.install_hostpython_prerequisites(
+                packages=["build[virtualenv]", "pip", "setuptools", "patchelf"] + self.hostpython_prerequisites
+            )
 
 
 class RustCompiledComponentsRecipe(PyProjectRecipe):
@@ -1523,8 +1604,6 @@ class RustCompiledComponentsRecipe(PyProjectRecipe):
         "x86": "i686-linux-android",
     }
 
-    call_hostpython_via_targetpython = False
-
     def get_recipe_env(self, arch, **kwargs):
         env = super().get_recipe_env(arch, **kwargs)
 
@@ -1563,7 +1642,7 @@ def get_recipe_env(self, arch, **kwargs):
         env["PATH"] = ("{hostpython_dir}:{old_path}").format(
             hostpython_dir=Recipe.get_recipe(
                 "hostpython3", self.ctx
-            ).get_path_to_python(),
+            ).local_bin,
             old_path=env["PATH"],
         )
         return env

pythonforandroid/recipes/android/src/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools",
+    "wheel",
+    "Cython>=0.29,<3.1"
+]
+build-backend = "setuptools.build_meta"