Overhaul stdlib test running

This commit overhauls our ability to run the stdlib test harness.

Author: indygreg

.github/workflows/linux.yml

@@ -329,12 +329,20 @@ jobs:
               unset LD_LIBRARY_PATH
             fi
           
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
           fi
         env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
           MATRIX_RUN: ${{ matrix.run }}
           MATRIX_LIBC: ${{ matrix.libc }}
 
+      - name: Upload Junit Test Results
+        if: ${{ matrix.run == 'true' }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
+
   build-1:
     needs:
       - generate-matrix

.github/workflows/macos.yml

@@ -181,7 +181,15 @@ jobs:
           build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst
           
           if [ "${MATRIX_RUN}" == "true" ]; then
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
           fi
         env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
           MATRIX_RUN: ${{ matrix.run }}
+
+      - name: Upload Junit Test Results
+        if: ${{ matrix.run == 'true' }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"

.github/workflows/windows.yml

@@ -186,4 +186,12 @@ jobs:
         run: |
           $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
           .\pythonbuild.exe validate-distribution $Dists
-          uv run --no-dev test-distribution.py $Dists
+          uv run --no-dev test-distribution.py --junit-xml $JUNIT_FILENAME --stdlib $Dists
+        env:
+          JUNIT_FILENAME: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"
+
+      - name: Upload Junit Test Results
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: "junit-cpython-${{ matrix.python }}-${{ matrix.target_triple}}-${{ matrix.build_options }}.xml"

Cargo.lock

@@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz"
 PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
-cat Setup.local
-mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
+# Put critical config files in logs to aid debugging.
+for f in Setup.local Makefile.extra stdlib-test-annotations.json; do
+  echo "BEGIN $f"
+  cat $f
+  echo "END $f"
+done
 
-cat Makefile.extra
+mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
 
 pushd "Python-${PYTHON_VERSION}"
 
@@ -1386,14 +1390,18 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/"
 cp -av Python/frozen.c "${ROOT}/out/python/build/Python/"
 cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/"
 
-# Copy the test hardness runner for convenience.
+# Copy the test harness runner for convenience.
 # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py"
 else
     cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/"
 fi
 
+# Copy standard library test annotations so it is in the artifact and the
+# distribution self-describes expected test wonkiness.
+cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json"
+
 # Don't hard-code the build-time prefix into the pkg-config files. See
 # the description of `pcfiledir` in `man pkg-config`.
 find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \

cpython-unix/build-cpython.sh

@@ -24,6 +24,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
 )
 from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
@@ -50,6 +51,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -732,6 +734,13 @@ def build_cpython(
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     ems = extension_modules_config(EXTENSION_MODULES)
 
     setup = derive_setup_local(
@@ -804,6 +813,14 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="Makefile.extra")
 
+        # Install the derived test annotations.
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+            test_annotations.json_dump(fh)
+            fh.flush()
+
+            build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
@@ -1070,7 +1087,7 @@ def main():
             write_dockerfiles(SUPPORT, BUILD)
         elif action == "makefiles":
             targets = get_targets(TARGETS_CONFIG)
-            write_triples_makefiles(targets, BUILD, SUPPORT)
+            write_triples_makefiles(targets, ROOT, BUILD, SUPPORT)
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 

cpython-unix/build.py

@@ -22,6 +22,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_config_c,
+    stdlib_test_annotations,
 )
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.utils import (
@@ -39,6 +40,7 @@
 BUILD = ROOT / "build"
 DIST = ROOT / "dist"
 SUPPORT = ROOT / "cpython-windows"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LOG_PREFIX = [None]
 LOG_FH = [None]
@@ -128,7 +130,6 @@
     "_zstd": ["zstd"],
 }
 
-
 # Tests to run during PGO profiling.
 #
 # This set was copied from test.libregrtest.pgo in the CPython source
@@ -1407,6 +1408,13 @@ def build_cpython(
     else:
         raise Exception("unhandled architecture: %s" % arch)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     tempdir_opts = (
         {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {}
     )
@@ -1763,6 +1771,12 @@ def build_cpython(
                 out_dir / "python" / "build" / "run_tests.py",
             )
 
+        # Install a JSON file annotating tests.
+        with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open(
+            "w", encoding="utf-8"
+        ) as fh:
+            test_annotations.json_dump(fh)
+
         licenses_dir = out_dir / "python" / "licenses"
         licenses_dir.mkdir()
         for f in sorted(os.listdir(ROOT)):

cpython-windows/build.py

@@ -10,6 +10,7 @@ dependencies = [
     "docker>=7.1.0",
     "jinja2>=3.1.5",
     "jsonschema>=4.23.0",
+    "junitparser>=4.0.2",
     "pyyaml>=6.0.2",
     "six>=1.17.0",
     "tomli>=2.2.1",