verify: run stdlib tests

Author: indygreg

Cargo.lock

@@ -30,6 +30,7 @@ rustls = { version = "0.23" , default-features = false, features = ["aws_lc_rs"]
 semver = "1.0.27"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.145"
+serde_yaml = "0.9.33"
 sha2 = "0.10.9"
 tar = "0.4.44"
 tempfile = "3.23.0"

Cargo.toml

@@ -24,6 +24,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
 )
 from pythonbuild.docker import (
     build_docker_image,
@@ -56,6 +57,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -764,6 +766,13 @@ def build_cpython(
     setup_local_content = setup["setup_local"]
     extra_make_content = setup["make_data"]
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     with build_environment(client, image) as build_env:
         if settings.get("needs_toolchain"):
             build_env.install_toolchain(

cpython-unix/build.py

@@ -6,6 +6,8 @@
 import re
 import tarfile
 
+from dataclasses import dataclass
+
 import jsonschema
 import yaml
 
@@ -137,6 +139,37 @@
     },
 }
 
+STDLIB_TEST_ANNOTATIONS_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "expected-failures": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "reason": {"type": "string"},
+                    "targets": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    },
+                    "ignore-targets": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    },
+                    "minimum-python-version": {"type": "string"},
+                    "maximum-python-version": {"type": "string"},
+                    "build-option": {"type": "string"},
+                    "no-build-option": {"type": "string"},
+                    "exclude": {"type": "boolean"},
+                    "dont-verify": {"type": "boolean"},
+                },
+                "additionalProperties": False,
+                "required": ["name", "reason"],
+            },
+        },
+    },
+}
 
 # Packages that define tests.
 STDLIB_TEST_PACKAGES = {
@@ -750,3 +783,92 @@ def extension_modules_config(yaml_path: pathlib.Path):
     jsonschema.validate(data, EXTENSION_MODULES_SCHEMA)
 
     return data
+
+
+@dataclass
+class TestAnnotation:
+    # Name/pattern of test.
+    name: str
+    # Describes why the annotation exists.
+    reason: str
+    # Whether the test is expected to fail.
+    expect_test_failure: bool
+    # Whether test failure is intermittent. Should only be true if
+    # expect_test_failure also true.
+    intermittent_test_failure: bool
+    # Whether to exclude loading the test module when running tests.
+    exclude_testing: bool
+
+
+def stdlib_test_annotations(
+    yaml_path: pathlib.Path,
+    python_version: str,
+    target_triple: str,
+    build_options: set[str],
+) -> list[TestAnnotation]:
+    """Processes the test-annotations.yml file for a given build configuration."""
+    with yaml_path.open("r", encoding="utf-8") as fh:
+        data = yaml.load(fh, Loader=yaml.SafeLoader)
+
+    jsonschema.validate(data, STDLIB_TEST_ANNOTATIONS_SCHEMA)
+
+    annotations = []
+
+    log(f"processing {len(data['expected-failures'])} stdlib test annotations")
+
+    for test in data["expected-failures"]:
+        name = test["name"]
+
+        if targets := test.get("targets"):
+            matches_target = any(re.match(p, target_triple) for p in targets)
+        else:
+            matches_target = True
+
+        for m in test.get("ignore-targets", []):
+            if re.match(m, target_triple):
+                matches_target = False
+
+        if not matches_target:
+            log(f"ignoring rule (target doesn't match): {name}")
+            continue
+
+        python_minimum_version = test.get("minimum-python-version", "1.0")
+        python_maximum_version = test.get("maximum-python-version", "100.0")
+
+        if not meets_python_minimum_version(python_version, python_minimum_version):
+            log(
+                f"ignoring rule ({python_version} < {python_minimum_version} (min)): {name}"
+            )
+            continue
+
+        if not meets_python_maximum_version(python_version, python_maximum_version):
+            log(
+                f"ignoring rule ({python_version} > {python_maximum_version} (max)): {name}"
+            )
+            continue
+
+        if option := test.get("build-option"):
+            if option not in build_options:
+                log(f"ignoring rule (build option {option} not present): {name}")
+                continue
+
+        if option := test.get("no-build-option"):
+            if option in build_options:
+                log(f"ignoring rule (build option {option} is present): {name}")
+                continue
+
+        # Filtering complete. This rule applies to the current build.
+
+        log(f"relevant test annotation: {name}: {test['reason']}")
+
+        annotations.append(
+            TestAnnotation(
+                name=name,
+                reason=test["reason"],
+                expect_test_failure=True,
+                intermittent_test_failure=test.get("dont-verify", False),
+                exclude_testing=test.get("exclude", False),
+            )
+        )
+
+    return annotations

pythonbuild/cpython.py

@@ -855,6 +855,7 @@ const SHARED_LIBRARY_EXTENSIONS: &[&str] = &[
 ];
 
 const PYTHON_VERIFICATIONS: &str = include_str!("verify_distribution.py");
+const TEST_ANNOTATIONS: &str = include_str!("../stdlib-test-annotations.yml");
 
 fn allowed_dylibs_for_triple(triple: &str) -> Vec<MachOAllowedDylib> {
     match triple {
@@ -2171,6 +2172,12 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result<Vec<String>> {
     let test_file = temp_dir.path().join("verify.py");
     std::fs::write(&test_file, PYTHON_VERIFICATIONS.as_bytes())?;
 
+    // Normalize annotations to JSON so the Python stdlib can read it.
+    let annotations_json: serde_json::Value = serde_yaml::from_str(TEST_ANNOTATIONS)?;
+    let annotations_pretty = serde_json::to_vec_pretty(&annotations_json)?;
+    let test_annotations_file = temp_dir.path().join("test-annotations.json");
+    std::fs::write(&test_annotations_file, annotations_pretty)?;
+
     eprintln!("  running interpreter tests (output should follow)");
     let output = duct::cmd(&python_exe, [test_file.display().to_string()])
         .stdout_to_stderr()