feat: allow manylinux 2.28 and 2.34 on python 3.12+ when compiled on a different architecture (#762)

* feat: allow manylinux 2.28 and 2.34 on python 3.12+ when compiled on a different architecture

* fix: docstring formatting

Author: alesanfra

Makefile

@@ -21,7 +21,7 @@ lint:
 	ruff check aws_lambda_builders
 
 lint-fix:
-	ruff aws_lambda_builders --fix
+	ruff check aws_lambda_builders --fix
 
 # Command to run everytime you make changes to verify everything works
 dev: lint test

aws_lambda_builders/workflows/python_pip/packager.py

@@ -2,11 +2,12 @@
 Installs packages using PIP
 """
 
+import itertools
 import logging
 import re
 import subprocess
 from email.parser import FeedParser
-from typing import Tuple
+from typing import List, Tuple
 
 from aws_lambda_builders.architecture import ARM64, X86_64
 from aws_lambda_builders.utils import extract_tarfile
@@ -171,19 +172,25 @@ class DependencyBuilder(object):
     packager.
     """
 
-    _COMPATIBLE_PLATFORM_ARM64 = {
+    _COMPATIBLE_PLATFORM_ARM64 = [
         "any",
         "linux_aarch64",
         "manylinux2014_aarch64",
-    }
+        "manylinux_2_17_aarch64",
+        "manylinux_2_28_aarch64",
+        "manylinux_2_34_aarch64",
+    ]
 
-    _COMPATIBLE_PLATFORM_X86_64 = {
+    _COMPATIBLE_PLATFORM_X86_64 = [
         "any",
         "linux_x86_64",
         "manylinux1_x86_64",
         "manylinux2010_x86_64",
         "manylinux2014_x86_64",
-    }
+        "manylinux_2_17_x86_64",
+        "manylinux_2_28_x86_64",
+        "manylinux_2_34_x86_64",
+    ]
 
     _COMPATIBLE_PLATFORMS = {
         ARM64: _COMPATIBLE_PLATFORM_ARM64,
@@ -214,6 +221,14 @@ class DependencyBuilder(object):
     # Unlikely to hit this case.
     _DEFAULT_GLIBC = (2, 17)
 
+    # Mapping of glibc version to the most recent manylinux version compatible.
+    # The offically supported manylinux versions are 2_17, 2_28 and 2_34 as per https://github.com/pypa/manylinux
+    _GLIBC_TO_LATEST_MANYLINUX = {
+        (2, 17): "manylinux_2_17",
+        (2, 26): "manylinux_2_17",
+        (2, 34): "manylinux_2_34",
+    }
+
     def __init__(self, osutils, runtime, python_exe, pip_runner=None, architecture=X86_64):
         """Initialize a DependencyBuilder.
 
@@ -379,8 +394,60 @@ def _download_binary_wheels(self, packages, directory):
         # Try to get binary wheels for each package that isn't compatible.
         LOG.debug("Downloading missing wheels: %s", packages)
         lambda_abi = get_lambda_abi(self.runtime)
-        platform = "manylinux2014_aarch64" if self.architecture == ARM64 else "manylinux2014_x86_64"
-        self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi, platform)
+        self._pip.download_manylinux_wheels(
+            [pkg.identifier for pkg in packages], directory, lambda_abi, self.compatible_platforms
+        )
+
+    @property
+    def compatible_platforms(self) -> List[str]:
+        """Get the list of all compatible platforms for the current architecture.
+
+        Examples:
+        ```python
+        # Return value with python 3.11 on x86_64
+        [
+            'any',
+            'linux_x86_64',
+            'manylinux1_x86_64',
+            'manylinux2010_x86_64',
+            'manylinux2014_x86_64',
+            'manylinux_2_17_x86_64'
+        ]
+
+        # Return value with python 3.12 on x86_64
+        [
+            'any',
+            'linux_x86_64',
+            'manylinux1_x86_64',
+            'manylinux2010_x86_64',
+            'manylinux2014_x86_64',
+            'manylinux_2_17_x86_64',
+            'manylinux_2_28_x86_64',
+            'manylinux_2_34_x86_64'
+        ]
+
+        # Return value with python 3.13 on ARM64
+        [
+            'any',
+            'linux_aarch64',
+            'manylinux2014_aarch64',
+            'manylinux_2_17_aarch64',
+            'manylinux_2_28_aarch64',
+            'manylinux_2_34_aarch64'
+        ]
+        ```
+        """
+        lambda_abi = get_lambda_abi(self.runtime)
+        manylinux_prefix = self._GLIBC_TO_LATEST_MANYLINUX.get(self._RUNTIME_GLIBC.get(lambda_abi, self._DEFAULT_GLIBC))
+        architecture = "aarch64" if self.architecture == ARM64 else "x86_64"
+
+        # Get the latest compatible platform tag for the current architecture,
+        # all the previous ones are also compatible.
+        latest_compatible_platform = f"{manylinux_prefix}_{architecture}"
+
+        all_platforms = self._COMPATIBLE_PLATFORMS[self.architecture]
+        max_index = all_platforms.index(latest_compatible_platform)
+        return all_platforms[: max_index + 1]
 
     def _build_sdists(self, sdists, directory, compile_c=True):
         LOG.debug("Build missing wheels from sdists " "(C compiling %s): %s", compile_c, sdists)
@@ -432,7 +499,7 @@ def _is_compatible_platform_tag(self, expected_abi, platform):
 
         In addition to checking the tag pattern, we also need to verify the glibc version
         """
-        if platform in self._COMPATIBLE_PLATFORMS[self.architecture]:
+        if platform in self.compatible_platforms:
             return True
 
         arch = "aarch64" if self.architecture == ARM64 else "x86_64"
@@ -832,7 +899,7 @@ def download_all_dependencies(self, requirements_filename, directory):
             # complain at deployment time.
             self.build_wheel(wheel_package_path, directory)
 
-    def download_manylinux_wheels(self, packages, directory, lambda_abi, platform="manylinux2014_x86_64"):
+    def download_manylinux_wheels(self, packages, directory, lambda_abi, platforms):
         """Download wheel files for manylinux for all the given packages."""
         # If any one of these dependencies fails pip will bail out. Since we
         # are only interested in all the ones we can download, we need to feed
@@ -846,8 +913,7 @@ def download_manylinux_wheels(self, packages, directory, lambda_abi, platform="m
             arguments = [
                 "--only-binary=:all:",
                 "--no-deps",
-                "--platform",
-                platform,
+                *list(itertools.chain.from_iterable(["--platform", element] for element in platforms)),
                 "--implementation",
                 "cp",
                 "--abi",

tests/functional/workflows/python_pip/test_packager.py

@@ -202,10 +202,10 @@ def _write_requirements_txt(self, packages, directory):
         with open(filepath, "w") as f:
             f.write(contents)
 
-    def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, **kwargs):
+    def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner, runtime="python3.9", **kwargs):
         appdir = str(_create_app_structure(tmpdir))
         self._write_requirements_txt(reqs, appdir)
-        builder = DependencyBuilder(OSUtils(), "python3.9", sys.executable, runner, **kwargs)
+        builder = DependencyBuilder(OSUtils(), runtime, sys.executable, runner, **kwargs)
         return appdir, builder
 
     def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils):
@@ -516,6 +516,74 @@ def test_can_get_arm64_whls(self, tmpdir, osutils, pip_runner):
         for req in reqs:
             assert req in installed_packages
 
+    def test_can_get_newer_platforms(self, tmpdir, osutils, pip_runner):
+        reqs = ["foo", "bar"]
+        pip, runner = pip_runner
+        appdir, builder = self._make_appdir_and_dependency_builder(reqs, tmpdir, runner, runtime="python3.12")
+        requirements_file = os.path.join(appdir, "requirements.txt")
+        pip.packages_to_download(
+            expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
+            packages=["foo-1.0-cp312-none-any.whl", "bar-1.2-cp312-cp312-manylinux_2_28_x86_64.whl"],
+        )
+        site_packages = os.path.join(appdir, ".chalice.", "site-packages")
+        with osutils.tempdir() as scratch_dir:
+            builder.build_site_packages(requirements_file, site_packages, scratch_dir)
+        installed_packages = os.listdir(site_packages)
+
+        pip.validate()
+        for req in reqs:
+            assert req in installed_packages
+
+    def test_can_get_newer_platforms_cross_compile(self, tmpdir, osutils, pip_runner):
+        reqs = ["foo", "bar"]
+        pip, runner = pip_runner
+        appdir, builder = self._make_appdir_and_dependency_builder(
+            reqs, tmpdir, runner, runtime="python3.12", architecture=ARM64
+        )
+        requirements_file = os.path.join(appdir, "requirements.txt")
+        pip.packages_to_download(
+            expected_args=["-r", requirements_file, "--dest", mock.ANY, "--exists-action", "i"],
+            packages=["foo-1.0-cp312-none-any.whl", "bar-1.2-cp312-cp312-manylinux_2_28_x86_64.whl"],
+        )
+
+        # First call returned x86_64 wheels, fallback to the second call
+        pip.packages_to_download(
+            expected_args=[
+                "--only-binary=:all:",
+                "--no-deps",
+                "--platform",
+                "any",
+                "--platform",
+                "linux_aarch64",
+                "--platform",
+                "manylinux2014_aarch64",
+                "--platform",
+                "manylinux_2_17_aarch64",
+                # It's python 3.12, so we can use newer platforms.
+                "--platform",
+                "manylinux_2_28_aarch64",
+                "--platform",
+                "manylinux_2_34_aarch64",
+                "--implementation",
+                "cp",
+                "--abi",
+                get_lambda_abi(builder.runtime),
+                "--dest",
+                mock.ANY,
+                "bar==1.2",
+            ],
+            packages=["bar-1.2-cp312-cp312-manylinux_2_28_aarch64.whl"],
+        )
+
+        site_packages = os.path.join(appdir, ".chalice.", "site-packages")
+        with osutils.tempdir() as scratch_dir:
+            builder.build_site_packages(requirements_file, site_packages, scratch_dir)
+        installed_packages = os.listdir(site_packages)
+
+        pip.validate()
+        for req in reqs:
+            assert req in installed_packages
+
     def test_does_fail_on_invalid_local_package(self, tmpdir, osutils, pip_runner):
         reqs = ["../foo"]
         pip, runner = pip_runner
@@ -629,7 +697,17 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
                 "--only-binary=:all:",
                 "--no-deps",
                 "--platform",
+                "any",
+                "--platform",
+                "linux_x86_64",
+                "--platform",
+                "manylinux1_x86_64",
+                "--platform",
+                "manylinux2010_x86_64",
+                "--platform",
                 "manylinux2014_x86_64",
+                "--platform",
+                "manylinux_2_17_x86_64",
                 "--implementation",
                 "cp",
                 "--abi",
@@ -663,7 +741,17 @@ def test_allowlist_sqlalchemy(self, tmpdir, osutils, pip_runner):
                 "--only-binary=:all:",
                 "--no-deps",
                 "--platform",
+                "any",
+                "--platform",
+                "linux_x86_64",
+                "--platform",
+                "manylinux1_x86_64",
+                "--platform",
+                "manylinux2010_x86_64",
+                "--platform",
                 "manylinux2014_x86_64",
+                "--platform",
+                "manylinux_2_17_x86_64",
                 "--implementation",
                 "cp",
                 "--abi",
@@ -798,7 +886,17 @@ def test_build_into_existing_dir_with_preinstalled_packages(self, tmpdir, osutil
                 "--only-binary=:all:",
                 "--no-deps",
                 "--platform",
+                "any",
+                "--platform",
+                "linux_x86_64",
+                "--platform",
+                "manylinux1_x86_64",
+                "--platform",
+                "manylinux2010_x86_64",
+                "--platform",
                 "manylinux2014_x86_64",
+                "--platform",
+                "manylinux_2_17_x86_64",
                 "--implementation",
                 "cp",
                 "--abi",

tests/unit/workflows/python_pip/test_packager.py

@@ -249,13 +249,35 @@ def test_download_wheels(self, pip_factory):
         # for getting lambda compatible wheels.
         pip, runner = pip_factory()
         packages = ["foo", "bar", "baz"]
-        runner.download_manylinux_wheels(packages, "directory", "abi")
+        runner.download_manylinux_wheels(
+            packages,
+            "directory",
+            "abi",
+            [
+                "any",
+                "linux_x86_64",
+                "manylinux1_x86_64",
+                "manylinux2010_x86_64",
+                "manylinux2014_x86_64",
+                "manylinux_2_17_x86_64",
+            ],
+        )
         expected_prefix = [
             "download",
             "--only-binary=:all:",
             "--no-deps",
             "--platform",
+            "any",
+            "--platform",
+            "linux_x86_64",
+            "--platform",
+            "manylinux1_x86_64",
+            "--platform",
+            "manylinux2010_x86_64",
+            "--platform",
             "manylinux2014_x86_64",
+            "--platform",
+            "manylinux_2_17_x86_64",
             "--implementation",
             "cp",
             "--abi",
@@ -270,7 +292,7 @@ def test_download_wheels(self, pip_factory):
 
     def test_download_wheels_no_wheels(self, pip_factory):
         pip, runner = pip_factory()
-        runner.download_manylinux_wheels([], "directory", "abi")
+        runner.download_manylinux_wheels([], "directory", "abi", [])
         assert len(pip.calls) == 0
 
     def test_does_find_local_directory(self, pip_factory):