Merge pull request #10739 from SomberNight/202607_android_build

android build: use --no-isolation, hash-pin more deps, only one install of setuptools

Author: SomberNight

contrib/android/Dockerfile

@@ -176,6 +176,7 @@ RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends --allow-downgrades \
         libopengl-dev \
         libegl-dev \
+        ninja-build \
     && apt -y autoremove \
     && apt -y clean
 
@@ -225,7 +226,7 @@ RUN /opt/venv/bin/python3 -m pip install --no-build-isolation --no-dependencies
     -r /opt/deterministic-build/requirements-build-android.txt
 
 # install buildozer
-ENV BUILDOZER_CHECKOUT_COMMIT="4403ecf445f10b5fbf7c74f4621bf2b922ad35b5"
+ENV BUILDOZER_CHECKOUT_COMMIT="6242e3e41365e70f76bd18f30992dcc5898ad40e"
 # ^ from branch electrum_20240930 (note: careful with force-pushing! see #8162)
 RUN cd /opt \
     && git clone https://github.com/spesmilo/buildozer \
@@ -234,7 +235,7 @@ RUN cd /opt \
     && /opt/venv/bin/python3 -m pip install --no-build-isolation --no-dependencies -e .
 
 # install python-for-android
-ENV P4A_CHECKOUT_COMMIT="1098be6964cfc2156959e435e81c2c50f8398586"
+ENV P4A_CHECKOUT_COMMIT="8c0fcc9ef2e559918ca96ecde6e09fe521bb1427"
 # ^ from branch electrum_202602 (note: careful with force-pushing! see #8162)
 RUN cd /opt \
     && git clone https://github.com/spesmilo/python-for-android \

contrib/android/p4a_recipes/hostpython3/__init__.py

@@ -8,6 +8,7 @@
 
 assert HostPython3Recipe.depends == []
 assert HostPython3Recipe.python_depends == []
+assert HostPython3Recipe.patches == []
 
 
 class HostPython3RecipePinned(util.InheritedRecipeMixin, HostPython3Recipe):
@@ -18,6 +19,11 @@ class HostPython3RecipePinned(util.InheritedRecipeMixin, HostPython3Recipe):
     # use official releases from python.org that have sigs, instead of auto-generated archives from github
     url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
 
+    # TODO: remove patch once CPython >= 3.12 is used (no more bundled setuptools)
+    patches = [
+        os.path.join(os.path.dirname(__file__), "patches", "cpython-311-ensurepip-no-setuptools.patch"),
+    ]
+
     # this property overrides the default hostpython dependencies for PyProjectRecipe recipies
     pyproject_base_dependencies = [
         HashPinnedDependency(package="build==1.4.0",

contrib/android/p4a_recipes/hostpython3/patches/cpython-311-ensurepip-no-setuptools.patch

@@ -0,0 +1,33 @@
+Backport of Python 3.12's removal of setuptools from ensurepip (gh-95299,
+https://github.com/python/cpython/pull/101039) onto CPython 3.11.
+
+ensurepip is only used to bootstrap pip into hostpython. Without this patch
+it would also install its bundled, unpinned setuptools, which would later
+coexist with the hash-pinned setuptools we install ourselves: pip's
+"--target" mode cannot uninstall, so the stale setuptools-79.0.1.dist-info
+would survive next to the pinned version and which distribution gets picked
+up during the build depends on os.listdir() order, breaking reproducibility.
+
+The now-unreferenced setuptools wheel in Lib/ensurepip/_bundled/ is left in
+the source tree ("patch" cannot delete binary files); it is never installed.
+
+The deleted _SETUPTOOLS_VERSION line changes between CPython 3.11.x releases,
+so this patch is coupled to the CPython version pinned in our hostpython3
+recipe and must be refreshed together with it. Remove the patch when
+hostpython3 is >= 3.12.
+
+--- a/Lib/ensurepip/__init__.py
++++ b/Lib/ensurepip/__init__.py
+@@ -9,11 +9,9 @@
+ 
+ 
+ __all__ = ["version", "bootstrap"]
+-_PACKAGE_NAMES = ('setuptools', 'pip')
+-_SETUPTOOLS_VERSION = "79.0.1"
++_PACKAGE_NAMES = ('pip',)
+ _PIP_VERSION = "24.0"
+ _PROJECTS = [
+-    ("setuptools", _SETUPTOOLS_VERSION, "py3"),
+     ("pip", _PIP_VERSION, "py3"),
+ ]
+ 

contrib/android/p4a_recipes/packaging/__init__.py

@@ -1,4 +1,5 @@
 from pythonforandroid.recipes.packaging import PackagingRecipe
+from pythonforandroid.util import HashPinnedDependency
 
 
 assert PackagingRecipe._version == "26.0"
@@ -8,6 +9,10 @@
 
 class PackagingRecipePinned(PackagingRecipe):
     sha512sum = "27a066a7d65ba76189212973b6a0d162f3d361848b1b0c34a82865cf180b3284a837cc34206c297f002a73feae414e25a26c5960bb884a74ea337f582585f1d2"
+    hostpython_prerequisites = [
+        HashPinnedDependency(package="flit-core==3.12.0",
+                             hashes=['sha256:e7a0304069ea895172e3c7bb703292e992c5d1555dd1233ab7b5621b5b69e62c']),
+    ]
 
 
 recipe = PackagingRecipePinned()

contrib/android/p4a_recipes/pyjnius/__init__.py

@@ -13,6 +13,10 @@
 
 class PyjniusRecipePinned(util.InheritedRecipeMixin, PyjniusRecipe):
     hostpython_prerequisites = [
+        HashPinnedDependency(package="setuptools==80.9.0",
+                             hashes=['sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922']),
+        HashPinnedDependency(package="wheel==0.45.1",
+                             hashes=['sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248']),
         HashPinnedDependency(package="Cython==3.1.8",
                              hashes=['sha256:282b3c8e6abc3fea421919e862e898ffdd86fc0796009bdb5ffdf8211413219f'])
     ]

contrib/android/p4a_recipes/pyqt6sip/__init__.py

@@ -17,8 +17,6 @@ class PyQt6SipRecipePinned(util.InheritedRecipeMixin, PyQt6SipRecipe):
     hostpython_prerequisites = [
         HashPinnedDependency(package="setuptools==80.9.0",
                              hashes=['sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922']),
-        HashPinnedDependency(package="packaging==26.0",
-                             hashes=['sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529']),
     ]
 
 

contrib/android/p4a_recipes/setuptools/__init__.py

@@ -0,0 +1,18 @@
+from pythonforandroid.recipes.setuptools import SetuptoolsRecipe
+from pythonforandroid.util import HashPinnedDependency
+
+
+assert SetuptoolsRecipe._version == "80.9.0"
+assert SetuptoolsRecipe.depends == ['python3']
+assert SetuptoolsRecipe.python_depends == []
+
+
+class SetuptoolsRecipePinned(SetuptoolsRecipe):
+    sha512sum = "36eb1f219d29c6b9e135936bde2001ad70a971c8069cd0175d3a5325b450e6843a903d3f70043c9f534768ebeab8ab0c544b8f44456555d333f1ed72daa5c18b"
+    hostpython_prerequisites = [
+        HashPinnedDependency(package="setuptools==80.9.0",
+                             hashes=['sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922']),
+    ]
+
+
+recipe = SetuptoolsRecipePinned()

contrib/android/p4a_recipes/sip/__init__.py

@@ -2,7 +2,7 @@
 from pythonforandroid.util import HashPinnedDependency
 
 assert SipRecipe._version == "6.15.1"
-assert SipRecipe.depends == ["python3"], SipRecipe.depends
+assert SipRecipe.depends == ["python3", "packaging"], SipRecipe.depends
 assert SipRecipe.python_depends == []
 
 
@@ -12,6 +12,11 @@ class SipRecipePinned(SipRecipe):
     hostpython_prerequisites = [
         HashPinnedDependency(package="setuptools==80.9.0",
                              hashes=['sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922']),
+        HashPinnedDependency(package="setuptools-scm==8.3.1",
+                             hashes=['sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3']),
+        HashPinnedDependency(package="packaging==26.0",  # pulled in by setuptools-scm
+                             hashes=['sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529']),
+
     ]
 
 

contrib/deterministic-build/requirements-build-android.txt

@@ -1,5 +1,7 @@
 appdirs==1.4.4 \
     --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41
+build==1.4.0 \
+    --hash=sha256:f1b91b925aa322be454f8330c6fb48b465da993d1e7e7e6fa35027ec49f3c936
 colorama==0.4.5 \
     --hash=sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4
 Cython==0.29.37 \
@@ -10,14 +12,14 @@ MarkupSafe==3.0.3 \
     --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698
 packaging==26.2 \
     --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661
-pep517==0.13.1 \
-    --hash=sha256:1b2fa2ffd3938bb4beffe5d6146cbcb2bda996a5a4da9f31abffd8b24e07b317
 pexpect==4.9.0 \
     --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f
 pip==25.1.1 \
     --hash=sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077
 ptyprocess==0.7.0 \
     --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220
+pyproject-hooks==1.2.0 \
+    --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8
 setuptools==80.9.0 \
     --hash=sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c
 sh==2.2.2 \

contrib/requirements/requirements-build-android.txt

@@ -9,13 +9,16 @@ sh
 cython<3.0
 
 # needed by python-for-android:
+# ref https://github.com/spesmilo/python-for-android/blob/1098be6964cfc2156959e435e81c2c50f8398586/setup.py#L22
+# (meson and ninja we skip as not needed)
 appdirs
 # colorama upper bound to avoid needing hatchling
 colorama>=0.3.3,<0.4.6
 jinja2
-sh>=1.10
-pep517
+sh>=2,<3.0
 toml
+build
+packaging
 
 # needed for the Qt/QML Android GUI:
 # TODO double-check this