|
| 1 | +//! checker/runtime divergence harness. |
| 2 | +//! |
| 3 | +//! every `.by` code block in the basedpython mdtests that the checker accepts |
| 4 | +//! (no `# error:` assertions) must also transpile and *execute* cleanly: the |
| 5 | +//! mdtest framework verifies ty's diagnostics, this test verifies the runtime |
| 6 | +//! half of the contract. divergences of the form "checks clean but crashes at |
| 7 | +//! runtime" (enum constants becoming members, transform composition leaks, |
| 8 | +//! unsound lowerings) are exactly the bug class this catches. |
| 9 | +//! |
| 10 | +//! blocks carrying expected diagnostics are skipped — their runtime behaviour |
| 11 | +//! is intentionally unspecified. |
| 12 | +
|
| 13 | +use std::fs; |
| 14 | +use std::io::Write; |
| 15 | +use std::path::{Path, PathBuf}; |
| 16 | +use std::process::{Command, Stdio}; |
| 17 | +use std::sync::Mutex; |
| 18 | +use std::sync::atomic::{AtomicUsize, Ordering}; |
| 19 | + |
| 20 | +fn mdtest_dir() -> PathBuf { |
| 21 | + Path::new(env!("CARGO_MANIFEST_DIR")).join("../ty_python_semantic/resources/mdtest") |
| 22 | +} |
| 23 | + |
| 24 | +/// A CPython 3.13 interpreter, provisioned through uv so the harness runs the |
| 25 | +/// same interpreter everywhere instead of riding on whatever `python3` the host |
| 26 | +/// happens to ship. The transpiler emits modern syntax — PEP 695 generics, PEP |
| 27 | +/// 696 type-parameter defaults (`class C[T = int]`), PEP 646 unpacking — whose |
| 28 | +/// runtime floor is 3.13; CI runners range from 3.10 upward, so a checker-clean |
| 29 | +/// block can fail to even parse on an older interpreter. Returns `None` (the |
| 30 | +/// test then skips) when uv or the interpreter can't be obtained. |
| 31 | +#[cfg(not(windows))] |
| 32 | +fn python() -> Option<String> { |
| 33 | + if let Ok(p) = std::env::var("PYTHON") { |
| 34 | + return Some(p); |
| 35 | + } |
| 36 | + let find = || { |
| 37 | + let out = Command::new("uv") |
| 38 | + .args(["python", "find", "3.13"]) |
| 39 | + .output() |
| 40 | + .ok()?; |
| 41 | + out.status |
| 42 | + .success() |
| 43 | + .then(|| String::from_utf8_lossy(&out.stdout).trim().to_owned()) |
| 44 | + }; |
| 45 | + if let Some(path) = find() { |
| 46 | + return Some(path); |
| 47 | + } |
| 48 | + // not discoverable yet — let uv download a managed build, then locate it |
| 49 | + Command::new("uv") |
| 50 | + .args(["python", "install", "3.13"]) |
| 51 | + .output() |
| 52 | + .ok()?; |
| 53 | + find() |
| 54 | +} |
| 55 | + |
| 56 | +/// On windows the harness is skipped unless `PYTHON` is set explicitly: it drives |
| 57 | +/// a python subprocess whose interpreter discovery and stdout encoding differ |
| 58 | +/// from unix, and the checker/runtime contract it validates is platform |
| 59 | +/// independent, so unix coverage suffices. |
| 60 | +#[cfg(windows)] |
| 61 | +fn python() -> Option<String> { |
| 62 | + std::env::var("PYTHON").ok() |
| 63 | +} |
| 64 | + |
| 65 | +/// `major.minor` of the interpreter the blocks will execute on, so the |
| 66 | +/// transpile targets what it actually supports. |
| 67 | +fn python_version(python: &str) -> Option<String> { |
| 68 | + let output = Command::new(python) |
| 69 | + .arg("-c") |
| 70 | + .arg("import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')") |
| 71 | + .output() |
| 72 | + .ok()?; |
| 73 | + output |
| 74 | + .status |
| 75 | + .success() |
| 76 | + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned()) |
| 77 | +} |
| 78 | + |
| 79 | +/// Extract the `by` fenced code blocks of a markdown file, in order, with a flag |
| 80 | +/// for blocks living in a multi-file section (one that declares companion |
| 81 | +/// modules via a `` `name.py`: `` marker) — those import section-local modules |
| 82 | +/// and cannot run standalone. |
| 83 | +fn by_blocks(markdown: &str) -> Vec<(String, bool)> { |
| 84 | + let mut blocks: Vec<(String, usize)> = Vec::new(); |
| 85 | + let mut multi_file_sections: Vec<usize> = Vec::new(); |
| 86 | + let mut section = 0usize; |
| 87 | + let mut current: Option<String> = None; |
| 88 | + for line in markdown.lines() { |
| 89 | + if current.is_none() && line.starts_with('#') { |
| 90 | + section += 1; |
| 91 | + } |
| 92 | + // a companion-module marker: `` `pylib.py`: `` ahead of its fence |
| 93 | + if current.is_none() |
| 94 | + && line.trim().starts_with('`') |
| 95 | + && (line.trim().ends_with(".py`:") |
| 96 | + || line.trim().ends_with(".by`:") |
| 97 | + || line.trim().ends_with(".byi`:")) |
| 98 | + { |
| 99 | + multi_file_sections.push(section); |
| 100 | + } |
| 101 | + match &mut current { |
| 102 | + None if line.trim() == "```by" => current = Some(String::new()), |
| 103 | + None => {} |
| 104 | + Some(block) => { |
| 105 | + if line.trim() == "```" { |
| 106 | + blocks.push((current.take().expect("block in progress"), section)); |
| 107 | + } else { |
| 108 | + block.push_str(line); |
| 109 | + block.push('\n'); |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + } |
| 114 | + blocks |
| 115 | + .into_iter() |
| 116 | + .map(|(b, s)| (b, multi_file_sections.contains(&s))) |
| 117 | + .collect() |
| 118 | +} |
| 119 | + |
| 120 | +fn transpile(source: &str, min_version: &str) -> Result<String, String> { |
| 121 | + let mut child = Command::new(env!("CARGO_BIN_EXE_by")) |
| 122 | + .args(["transpile", "--min-version", min_version]) |
| 123 | + .stdin(Stdio::piped()) |
| 124 | + .stdout(Stdio::piped()) |
| 125 | + .stderr(Stdio::piped()) |
| 126 | + .spawn() |
| 127 | + .expect("failed to spawn by"); |
| 128 | + child |
| 129 | + .stdin |
| 130 | + .as_mut() |
| 131 | + .unwrap() |
| 132 | + .write_all(source.as_bytes()) |
| 133 | + .unwrap(); |
| 134 | + let output = child.wait_with_output().unwrap(); |
| 135 | + if output.status.success() { |
| 136 | + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) |
| 137 | + } else { |
| 138 | + Err(String::from_utf8_lossy(&output.stderr).into_owned()) |
| 139 | + } |
| 140 | +} |
| 141 | + |
| 142 | +/// Stub `reveal_type` (an mdtest debugging device with no runtime binding) |
| 143 | +/// after the `__future__` import, which must stay first. |
| 144 | +fn with_reveal_stub(transpiled: &str) -> String { |
| 145 | + const STUB: &str = "def reveal_type(x, *a, **k):\n return x\n"; |
| 146 | + match transpiled.strip_prefix("from __future__ import annotations\n") { |
| 147 | + Some(rest) => format!("from __future__ import annotations\n{STUB}{rest}"), |
| 148 | + None => format!("{STUB}{transpiled}"), |
| 149 | + } |
| 150 | +} |
| 151 | + |
| 152 | +#[test] |
| 153 | +#[expect( |
| 154 | + clippy::print_stderr, |
| 155 | + reason = "skip diagnostic when python is unavailable" |
| 156 | +)] |
| 157 | +fn clean_mdtest_blocks_run() { |
| 158 | + let Some(python) = python() else { |
| 159 | + eprintln!("skipping: no python 3.13 interpreter available (uv on unix, or set PYTHON)"); |
| 160 | + return; |
| 161 | + }; |
| 162 | + let Some(version) = python_version(&python) else { |
| 163 | + eprintln!("skipping: `{python}` not runnable"); |
| 164 | + return; |
| 165 | + }; |
| 166 | + |
| 167 | + // third-party runtime deps are environment-dependent; skip blocks that |
| 168 | + // need one the interpreter doesn't have |
| 169 | + let has_typing_extensions = Command::new(&python) |
| 170 | + .args(["-c", "import typing_extensions"]) |
| 171 | + .output() |
| 172 | + .is_ok_and(|o| o.status.success()); |
| 173 | + |
| 174 | + let dir = mdtest_dir(); |
| 175 | + let mut files: Vec<PathBuf> = fs::read_dir(&dir) |
| 176 | + .expect("mdtest dir") |
| 177 | + .filter_map(|e| e.ok().map(|e| e.path())) |
| 178 | + .filter(|p| { |
| 179 | + p.extension() |
| 180 | + .is_some_and(|ext| ext.eq_ignore_ascii_case("md")) |
| 181 | + && p.file_name() |
| 182 | + .and_then(|n| n.to_str()) |
| 183 | + .is_some_and(|n| n.starts_with("basedpython_")) |
| 184 | + }) |
| 185 | + .collect(); |
| 186 | + files.sort(); |
| 187 | + assert!(!files.is_empty(), "no basedpython mdtests found in {dir:?}"); |
| 188 | + |
| 189 | + // gather every runnable block up front — skipping ones with expected |
| 190 | + // diagnostics (only checker-clean blocks carry the contract) or that import |
| 191 | + // section-local companion modules this harness doesn't materialize — so the |
| 192 | + // work can be spread across a pool of workers rather than run serially. each |
| 193 | + // block is an independent `by transpile` + python subprocess pair, so the |
| 194 | + // harness is dominated by process spawn latency and parallelises cleanly. |
| 195 | + let mut items: Vec<(String, usize, String)> = Vec::new(); |
| 196 | + for file in &files { |
| 197 | + let name = file.file_name().unwrap().to_string_lossy().into_owned(); |
| 198 | + let markdown = fs::read_to_string(file).expect("read mdtest"); |
| 199 | + for (i, (block, multi_file)) in by_blocks(&markdown).into_iter().enumerate() { |
| 200 | + if block.contains("# error:") || multi_file { |
| 201 | + continue; |
| 202 | + } |
| 203 | + items.push((name.clone(), i, block)); |
| 204 | + } |
| 205 | + } |
| 206 | + let total = items.len(); |
| 207 | + assert!(total > 0, "no checker-clean by blocks found"); |
| 208 | + |
| 209 | + let tmp = tempfile::tempdir().expect("tempdir"); |
| 210 | + let failures: Mutex<Vec<String>> = Mutex::new(Vec::new()); |
| 211 | + let next = AtomicUsize::new(0); |
| 212 | + // each block alternates a cpu-bound `by transpile` with a wait on the python |
| 213 | + // subprocess, and `by` startup also reads typeshed from disk — so a little |
| 214 | + // oversubscription past the core count overlaps that i/o and keeps the cores |
| 215 | + // busy, while the cap bounds how many processes are ever live at once |
| 216 | + let workers = std::thread::available_parallelism() |
| 217 | + .map_or(4, |n| n.get() * 2) |
| 218 | + .clamp(1, 16) |
| 219 | + .min(total); |
| 220 | + |
| 221 | + // work-stealing pool: each worker claims the next index and processes it |
| 222 | + // until the list is drained. the closures borrow the shared state (the |
| 223 | + // work list, the cursor, the failure sink) so `failures` stays owned for |
| 224 | + // the drain below. temp files are keyed by `name_i`, unique per block, so |
| 225 | + // concurrent writes never collide. |
| 226 | + std::thread::scope(|scope| { |
| 227 | + for _ in 0..workers { |
| 228 | + scope.spawn(|| { |
| 229 | + while let Some((name, i, block)) = |
| 230 | + items.get(next.fetch_add(1, Ordering::Relaxed)) |
| 231 | + { |
| 232 | + let transpiled = match transpile(block, &version) { |
| 233 | + Ok(t) => t, |
| 234 | + Err(e) => { |
| 235 | + failures.lock().unwrap().push(format!( |
| 236 | + "{name} block {i}: transpile failed:\n{e}\n--- block ---\n{block}" |
| 237 | + )); |
| 238 | + continue; |
| 239 | + } |
| 240 | + }; |
| 241 | + if !has_typing_extensions && transpiled.contains("typing_extensions") { |
| 242 | + continue; |
| 243 | + } |
| 244 | + let py = tmp.path().join(format!( |
| 245 | + "{}_{i}.py", |
| 246 | + name.trim_end_matches(".md").replace('-', "_") |
| 247 | + )); |
| 248 | + fs::write(&py, with_reveal_stub(&transpiled)).unwrap(); |
| 249 | + let run = Command::new(&python) |
| 250 | + .arg(&py) |
| 251 | + .output() |
| 252 | + .expect("failed to run python"); |
| 253 | + if !run.status.success() { |
| 254 | + failures.lock().unwrap().push(format!( |
| 255 | + "{name} block {i}: checker-clean block crashed at runtime:\n{}\n--- block ---\n{block}", |
| 256 | + String::from_utf8_lossy(&run.stderr) |
| 257 | + )); |
| 258 | + } |
| 259 | + } |
| 260 | + }); |
| 261 | + } |
| 262 | + }); |
| 263 | + |
| 264 | + let failures = failures.into_inner().expect("failures mutex poisoned"); |
| 265 | + assert!( |
| 266 | + failures.is_empty(), |
| 267 | + "{} of {} checker-clean blocks diverge at runtime:\n\n{}", |
| 268 | + failures.len(), |
| 269 | + total, |
| 270 | + failures.join("\n\n") |
| 271 | + ); |
| 272 | +} |
0 commit comments