verify: run stdlib tests

Author: indygreg

Cargo.lock

@@ -30,6 +30,7 @@ rustls = { version = "0.23" , default-features = false, features = ["aws_lc_rs"]
 semver = "1.0.27"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.145"
+serde_yaml = "0.9.33"
 sha2 = "0.10.9"
 tar = "0.4.44"
 tempfile = "3.23.0"

Cargo.toml

@@ -855,6 +855,7 @@ const SHARED_LIBRARY_EXTENSIONS: &[&str] = &[
 ];
 
 const PYTHON_VERIFICATIONS: &str = include_str!("verify_distribution.py");
+const TEST_ANNOTATIONS: &str = include_str!("../stdlib-test-annotations.yml");
 
 fn allowed_dylibs_for_triple(triple: &str) -> Vec<MachOAllowedDylib> {
     match triple {
@@ -2171,6 +2172,12 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result<Vec<String>> {
     let test_file = temp_dir.path().join("verify.py");
     std::fs::write(&test_file, PYTHON_VERIFICATIONS.as_bytes())?;
 
+    // Normalize annotations to JSON so the Python stdlib can read it.
+    let annotations_json: serde_json::Value = serde_yaml::from_str(TEST_ANNOTATIONS)?;
+    let annotations_pretty = serde_json::to_vec_pretty(&annotations_json)?;
+    let test_annotations_file = temp_dir.path().join("test-annotations.json");
+    std::fs::write(&test_annotations_file, annotations_pretty)?;
+
     eprintln!("  running interpreter tests (output should follow)");
     let output = duct::cmd(&python_exe, [test_file.display().to_string()])
         .stdout_to_stderr()

src/validation.rs

@@ -3,7 +3,9 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import importlib.machinery
+import json
 import os
+import re
 import struct
 import subprocess
 import sys
@@ -318,5 +320,179 @@ def assertPythonWorks(path: Path, argv0: str = None):
             assertPythonWorks(sys.executable, argv0="/dev/null")
 
 
+def run_us() -> int:
+    t = unittest.TestProgram(exit=False)
+    if not t.result.wasSuccessful():
+        return 1
+    elif t.result.testsRun == 0:
+        return 1
+    else:
+        return 0
+
+
+def load_test_annotations():
+    here = Path(__file__).parent
+    p = here / "test-annotations.json"
+    with p.open("rb") as fh:
+        return json.load(fh)
+
+
+def python_version_compare(a: str, b: str) -> int:
+    a_parts = a.split(".")
+    b_parts = b.split(".")
+
+    a_ver = (int(a_parts[0]), int(a_parts[1]))
+    b_ver = (int(b_parts[0]), int(b_parts[1]))
+
+    if a_ver < b_ver:
+        return -1
+    elif a_ver == b_ver:
+        return 0
+    else:
+        return 1
+
+
+def run_stdlib() -> int:
+    target_triple = os.environ["TARGET_TRIPLE"]
+    build_options = os.environ["BUILD_OPTIONS"]
+    build_options_set = set(build_options.split("+"))
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+    # musl debug builds fail to run the test harness for some reason. Skip them.
+    if target_triple.endswith("-unknown-linux-musl") and "debug" in build_options_set:
+        print("stdlib tests being skipped for musl debug build because it doesn't work")
+        return 0
+
+    annotations = load_test_annotations()
+
+    expect_failures = []
+    excludes = []
+    dont_verify = set()
+
+    for test in annotations["expected-failures"]:
+        name = test["name"]
+
+        if targets := test.get("targets"):
+            matches_target = any(re.match(p, target_triple) for p in targets)
+        else:
+            matches_target = True
+
+        for m in test.get("ignore-targets", []):
+            if re.match(m, target_triple):
+                matches_target = False
+
+        if not matches_target:
+            print(f"ignoring rule (target doesn't match): {name}")
+            continue
+
+        python_minimum_version = test.get("minimum-python-version", "1.0")
+        python_maximum_version = test.get("maximum-python-version", "100.0")
+
+        if python_version_compare(python_version, python_minimum_version) < 0:
+            print(
+                f"ignoring rule ({python_version} < {python_minimum_version}): {name}"
+            )
+            continue
+
+        if python_version_compare(python_version, python_maximum_version) > 0:
+            print(
+                f"ignoring rule ({python_version} > {python_maximum_version}): {name}"
+            )
+            continue
+
+        if option := test.get("build-option"):
+            if option not in build_options_set:
+                print(f"ignoring rule (build option {option} not present): {name}")
+                continue
+
+        if option := test.get("no-build-option"):
+            if option in build_options_set:
+                print(f"ignoring rule (build option {option} is present): {name}")
+                continue
+
+        print(f"expected test failure {name}: {test['reason']}")
+        expect_failures.append(name)
+
+        if test.get("exclude"):
+            excludes.append(name)
+
+        if test.get("dont-verify"):
+            dont_verify.add(name)
+
+    sys.stdout.flush()
+
+    # Spawn the test harness as its own process. Otherwise it gets confused.
+    base_args = [
+        sys.executable,
+        "-u",
+        "-W",
+        "default",
+        "-bb",
+        "-E",
+        "-m",
+        "test",
+    ]
+
+    args = list(base_args)
+
+    args.extend(
+        [
+            "-w",  # Re-run failed tests in verbose mode to aid debugging.
+            "-j",
+            "0",  # Run tests in parallel using all available CPUs.
+        ]
+    )
+
+    for t in excludes:
+        args.extend(["--exclude", t])
+    for i in expect_failures:
+        args.extend(["--ignore", i])
+
+    code = subprocess.run(args).returncode
+    if code != 0:
+        print("main test harness failed")
+        sys.stdout.flush()
+
+    # Run ignored / expected failures tests and verify they actually fail.
+    for i, t in enumerate(expect_failures):
+        args = list(base_args)
+
+        args.append("-v")
+
+        # Always provide the test_* name as a positional argument to limit
+        # which tests are loaded. Otherwise we load all test files and incur
+        # substantial overhead.
+        parts = t.split(".")
+        if parts[0] == "test":
+            args.extend(["-m", t, parts[1]])
+        else:
+            args.append(t)
+
+        print(f"[{i + 1}/{len(expect_failures)}] verifying {t} fails... ", end="")
+        res = subprocess.run(args, capture_output=True)
+
+        unexpected = False
+        if res.returncode != 0:
+            status = "yes"
+        elif t in dont_verify:
+            status = "no (ignored)"
+        else:
+            status = "no (unexpected pass)"
+            unexpected = True
+
+        print(status)
+
+        if unexpected:
+            code = 1
+            print(res.stdout.decode("utf-8", errors="ignore"))
+
+        sys.stdout.flush()
+
+    return code
+
+
 if __name__ == "__main__":
-    unittest.main()
+    res = [run_us(), run_stdlib()]
+    for code in res:
+        if code:
+            sys.exit(1)