Port verify_distribution.py into pythonbuild (#1058)

We move the custom `unittest` based tests from this file into a new
`pythonbuild/disttests` package, deleting `src/verify_distribution.py`
in the process.

We teach the new `pythonbuild.testdist` code to run our custom unittests
given an extracted Python distribution. They now run by default.

Rust code for executing the Python in the distribution has been removed.

CI tests have been updated to invoke `test-distribution.py` when we
support running the interpreter.

Behavior of `test-distribution.py` / `pythonbuild.disttests` has been
changed so stdlib tests are no longer run by default. This is necessary
to preserve backwards compatibility to keep CI passing.

Functionality is still a bit klunky IMO. I'll clean up the CLI UX a bit
in future commits.

Author: indygreg

.github/workflows/linux.yml

@@ -318,18 +318,19 @@ jobs:
         run: |
           chmod +x build/pythonbuild
 
+          build/pythonbuild validate-distribution dist/*.tar.zst
+          
           if [ "${MATRIX_RUN}" == "true" ]; then
             if [ "${MATRIX_LIBC}" == "musl" ]; then
               sudo apt install musl-dev
-
+          
               # GitHub's setup-python action sets `LD_LIBRARY_PATH` which overrides `RPATH`
               # as used in the musl builds.
               unset LD_LIBRARY_PATH
             fi
-            EXTRA_ARGS="--run"
+          
+            ./test-distribution.py dist/*.tar.zst
           fi
-
-          build/pythonbuild validate-distribution ${EXTRA_ARGS} dist/*.tar.zst
         env:
           MATRIX_RUN: ${{ matrix.run }}
           MATRIX_LIBC: ${{ matrix.libc }}

.github/workflows/macos.yml

@@ -178,10 +178,10 @@ jobs:
         run: |
           chmod +x build/pythonbuild
 
+          build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst
+          
           if [ "${MATRIX_RUN}" == "true" ]; then
-            EXTRA_ARGS="--run"
+            ./test-distribution.py dist/*.tar.zst
           fi
-
-          build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks ${EXTRA_ARGS} dist/*.tar.zst
         env:
           MATRIX_RUN: ${{ matrix.run }}

.github/workflows/windows.yml

@@ -185,4 +185,5 @@ jobs:
         if: ${{ ! matrix.dry-run }}
         run: |
           $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
-          .\pythonbuild.exe validate-distribution --run $Dists
+          .\pythonbuild.exe validate-distribution $Dists
+          uv run test-distribution.py $Dists

Cargo.lock

@@ -9,7 +9,6 @@ anyhow = "1.0.100"
 apple-sdk = "0.6.0"
 bytes = "1.11.0"
 clap = "4.5.52"
-duct = "1.1.1"
 flate2 = "1.1.5"
 futures = "0.3.30"
 goblin = "0.10.3"

Cargo.toml

@@ -41,7 +41,7 @@ This repository contains ``test-distribution.py`` script that can be
 used to run the Python test harness from a distribution archive.
 
 Here, we track the various known failures when running
-``test-distribution.py /path/to/distribution.tar.zst -u all``.
+``test-distribution.py --stdlib -- /path/to/distribution.tar.zst -u all``.
 
 ``test__locale``
 ----------------

docs/status.rst

@@ -10,6 +10,7 @@
 import tempfile
 import unittest
 from pathlib import Path
+from typing import Optional
 
 TERMINFO_DIRS = [
     "/etc/terminfo",
@@ -123,12 +124,12 @@ def test_hashlib(self):
         "_testcapi not available on statically-linked distributions",
     )
     def test_testcapi(self):
-        import _testcapi
+        import _testcapi  # type: ignore
 
         self.assertIsNotNone(_testcapi)
 
         if sys.version_info[0:2] >= (3, 13):
-            import _testlimitedcapi
+            import _testlimitedcapi  # type: ignore
 
             self.assertIsNotNone(_testlimitedcapi)
 
@@ -235,7 +236,7 @@ def test_gil_disabled(self):
         "zstd is only available in 3.14+",
     )
     def test_zstd_multithreaded(self):
-        from compression import zstd
+        from compression import zstd  # type: ignore
 
         max_threads = zstd.CompressionParameter.nb_workers.bounds()[1]
         assert max_threads > 0, (
@@ -294,7 +295,7 @@ def assertLibc(value):
     )
     @unittest.skipIf(os.name == "nt", "no symlinks or argv[0] on Windows")
     def test_getpath(self):
-        def assertPythonWorks(path: Path, argv0: str = None):
+        def assertPythonWorks(path: Path, argv0: Optional[str] = None):
             output = subprocess.check_output(
                 [argv0 or path, "-c", "print(42)"], executable=path, text=True
             )