Support _retain_test_patterns key for prebuilt wheels as sources in recipes

integration_tests/Makefile

@@ -13,6 +13,16 @@ test-recipe: check
 	fi; \
 	$(UV_RUN_PREFIX) pyodide build-recipes --recipe-dir=recipes --install --force-rebuild "$${RECIPE_TARGET}"
 
+# 	colorama test checks for combined behaviour of unvendor-tests and retain-test-patterns, and does it to a wheel.
+	@if [ -f dist/colorama-0.4.6-py2.py3-none-any.whl ]; then \
+		echo "... Verifying colorama test retention"; \
+		TEST_FILES=$$(python -m zipfile -l dist/colorama-0.4.6-py2.py3-none-any.whl | grep 'colorama/tests/' | grep '\.py' | awk '{print $$1}'); \
+		echo "$$TEST_FILES" | grep -q 'colorama/tests/winterm_test.py' || (echo "ERROR: colorama/tests/winterm_test.py not retained" && exit 1); \
+		TEST_COUNT=$$(echo "$$TEST_FILES" | wc -l | tr -d ' '); \
+		[ "$$TEST_COUNT" -eq 1 ] || (echo "ERROR: Expected 1 test file in wheel, found $$TEST_COUNT: $$TEST_FILES" && exit 1); \
+		echo "... colorama test retention verified"; \
+	fi
+
 	@echo "... Passed"
 
 .PHONY: test-src

integration_tests/recipes/README.md

@@ -2,16 +2,17 @@ This directory contains a few curated recipes to test the build process of pyodi
 
 ### List of recipes and their purpose
 
-- numpy: The most popular and widely used library in the scientific computing community.
-- pywavelets: a library with C and Cython extensions that uses the NumPy-C API.
-- orjson: Tests rust extension modules.
-- zlib: Tests static libraries.
-- geos: Tests shared libraries.
-- shapely: Tests a python package that depends on a shared library.
-- pydoc_data: Unvendored cpython module
-- boost-histogram: Tests scikit-build-core and cmake build system.
+- `numpy`: The most popular and widely used library in the scientific computing community.
+- `pywavelets`: a library with C and Cython extensions that uses the NumPy-C API.
+- `orjson`: Tests rust extension modules.
+- `zlib`: Tests static libraries.
+- `geos`: Tests shared libraries.
+- `shapely`: Tests a python package that depends on a shared library.
+- `pydoc_data`: Unvendored cpython module
+- `boost-histogram``: Tests scikit-build-core and cmake build system.
+- `colorama`: Uses prebuilt wheel as source to check test file unvendoring capabilities.
 
 ### For maintainers
 
 - Do not put too many recipes in this directory. It is meant to be a small collection of recipes that are representative of the build process.
-- The recipes in this directory is originally copied from `pyodide/pyodide`. It does not need to be updated frequently unless there is a change in the build process.
+- The recipes in this directory are originally copied from `pyodide/pyodide`. They do not need to be updated frequently unless there is a change in the build process.

integration_tests/recipes/colorama/meta.yaml

@@ -0,0 +1,17 @@
+package:
+  name: colorama
+  version: 0.4.6
+  top-level:
+    - colorama
+source:
+  url: https://files.pythonhosted.org/packages/py2.py3/c/colorama/colorama-0.4.6-py2.py3-none-any.whl
+  sha256: 4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+build:
+  # Should retain only winterm_test.py file in the colorama/tests/ directory
+  _retain-test-patterns:
+    - "*winterm_test*.py"
+about:
+  home: ""
+  PyPI: https://pypi.org/project/colorama
+  summary: Cross-platform colored terminal text.
+  license: ""

pyodide_build/recipe/spec.py

@@ -217,6 +217,7 @@ def _check_wheel_host_requirements(self) -> Self:
                 "cross_build_files",
                 "exports",
                 "unvendor_tests",
+                "retain_test_patterns",
                 "package_type",
             }
             for key in self.build.model_fields_set:

pyodide_build/recipe/unvendor.py

@@ -89,20 +89,28 @@ def unvendor_tests(
         root_rel = Path(root).relative_to(install_prefix)
         if root_rel.name == "__pycache__" or root_rel.name.endswith(".egg_info"):
             continue
-        if root_rel.name in {"test", "tests"}:
-            # This is a test folder
+
+        is_test_dir = root_rel.name in {"test", "tests"}
+
+        if is_test_dir and not retain_test_patterns:
             (test_install_prefix / root_rel).parent.mkdir(exist_ok=True, parents=True)
             shutil.move(install_prefix / root_rel, test_install_prefix / root_rel)
             n_moved += 1
             continue
+
         out_files.append(root)
         for fpath in files:
-            if (
+            is_test_file = (
                 fnmatch.fnmatchcase(fpath, "test_*.py")
                 or fnmatch.fnmatchcase(fpath, "*_test.py")
                 or fpath == "conftest.py"
-            ):
-                if any(fnmatch.fnmatchcase(fpath, pat) for pat in retain_test_patterns):
+            )
+
+            if is_test_dir or is_test_file:
+                file_rel_path = str(root_rel / fpath)
+                if any(
+                    fnmatch.fnmatch(file_rel_path, pat) for pat in retain_test_patterns
+                ):
                     continue
                 (test_install_prefix / root_rel).mkdir(exist_ok=True, parents=True)
                 shutil.move(

pyodide_build/tests/recipe/test_spec.py

@@ -73,3 +73,16 @@ def test_is_rust_package_2(reqs):
         **reqs,
     )
     assert not pkg.is_rust_package()
+
+
+def test_wheel_source_with_retain_test_patterns():
+    pkg = MetaConfig(
+        package={"name": "test-pkg", "version": "1.0.0"},
+        source={"url": "test.whl", "sha256": "abc123"},
+        build={
+            "unvendor-tests": True,
+            "_retain-test-patterns": ["*conftest.py", "*test_keep.py"],
+        },
+    )
+    assert pkg.build.unvendor_tests is True
+    assert pkg.build.retain_test_patterns == ["*conftest.py", "*test_keep.py"]