From 8ecc485759ea5a38876dc516b645cb36c49594f0 Mon Sep 17 00:00:00 2001 From: Thomas Pellissier-Tanon Date: Mon, 23 Mar 2026 10:51:17 +0100 Subject: [PATCH 1/3] Introspection: add a CLI to generate stubs Useful for setuptools-rust Use it also for the integration tests --- newsfragments/5904.added.md | 1 + noxfile.py | 76 +++++++++++++--------- pyo3-introspection/Cargo.toml | 3 - pyo3-introspection/src/main.rs | 24 +++++++ pyo3-introspection/tests/test.rs | 106 ------------------------------- 5 files changed, 72 insertions(+), 138 deletions(-) create mode 100644 newsfragments/5904.added.md create mode 100644 pyo3-introspection/src/main.rs delete mode 100644 pyo3-introspection/tests/test.rs diff --git a/newsfragments/5904.added.md b/newsfragments/5904.added.md new file mode 100644 index 00000000000..cd5eeb1f123 --- /dev/null +++ b/newsfragments/5904.added.md @@ -0,0 +1 @@ +`pyo3-introspection`: add a small CLI to generate stubs \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index b7130b30d4c..1e5b7fd52ea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1427,35 +1427,53 @@ def update_ui_tests(session: nox.Session): @nox.session(name="test-introspection") def test_introspection(session: nox.Session): - session.install("maturin") - session.install("ruff") - options = [] - target = os.environ.get("CARGO_BUILD_TARGET") - if target is not None: - options += ("--target", target) - profile = os.environ.get("CARGO_BUILD_PROFILE") - if profile == "release": - options.append("--release") - session.run_always( - "maturin", - "develop", - "-m", - "./pytests/Cargo.toml", - "--features", - "experimental-async,experimental-inspect", - *options, - ) - lib_file = session.run( - "python", - "-c", - "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", - silent=True, - ).strip() - _run_cargo_test( - session, - package="pyo3-introspection", - env={"PYO3_PYTEST_LIB_PATH": lib_file}, - ) + with tempfile.TemporaryDirectory() as stub_dir: + session.install("maturin") + session.install("ruff") + options = [] + target = os.environ.get("CARGO_BUILD_TARGET") + if target is not None: + options += ("--target", target) + profile = os.environ.get("CARGO_BUILD_PROFILE") + if profile == "release": + options.append("--release") + _run( + session, + "maturin", + "develop", + "-m", + "./pytests/Cargo.toml", + "--features", + "experimental-async,experimental-inspect", + *options, + ) + lib_file = session.run( + "python", + "-c", + "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", + silent=True, + ).strip() + _run_cargo(session, "run", "-p", "pyo3-introspection", "--", lib_file, stub_dir) + _run(session, "ruff", "format", stub_dir) + _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs")) + + +def _ensure_directory_equals(expected_dir: Path, actual_dir: Path): + # Assert all expected files are in actual and are equals + for expected_file_path in expected_dir.rglob("*"): + file_path = expected_file_path.relative_to(expected_dir) + actual_file_path = actual_dir / file_path + assert actual_file_path.exists(), f"File {file_path} does not exist" + assert expected_file_path.read_text() == actual_file_path.read_text(), ( + f"Content is different in {file_path}" + ) + # Assert all actual files are expected + for actual_file_path in actual_dir.rglob("*"): + file_path = actual_file_path.relative_to(actual_dir) + expected_file_path = expected_dir / file_path + assert expected_file_path.exists(), ( + f"File {file_path} exist even if not expected" + ) @lru_cache() diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml index 0958555c20b..c023c69248a 100644 --- a/pyo3-introspection/Cargo.toml +++ b/pyo3-introspection/Cargo.toml @@ -14,8 +14,5 @@ goblin = ">=0.9, <0.11" serde = { version = "1", features = ["derive"] } serde_json = "1" -[dev-dependencies] -tempfile = "3.12.0" - [lints] workspace = true diff --git a/pyo3-introspection/src/main.rs b/pyo3-introspection/src/main.rs new file mode 100644 index 00000000000..6a5c6506e6c --- /dev/null +++ b/pyo3-introspection/src/main.rs @@ -0,0 +1,24 @@ +//! Small CLI entry point to introspect a Python cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +use anyhow::{anyhow, Context, Result}; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; +use std::path::Path; +use std::{env, fs}; + +fn main() -> Result<()> { + let [_, binary_path, output_path] = env::args().collect::>().try_into().map_err(|_| anyhow!("pyo3-introspection takes two arguments, the path of the binary to introspect and the path of the directory to write the stub to"))?; + let module = introspect_cdylib(&binary_path, "pyo3_pytests") + .with_context(|| format!("Failed to introspect module {binary_path}"))?; + let actual_stubs = module_stub_files(&module); + for (path, module) in actual_stubs { + let file_path = Path::new(&output_path).join(path); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent).with_context(|| { + format!("Failed to create output directory {}", file_path.display()) + })?; + } + fs::write(&file_path, module) + .with_context(|| format!("Failed to write module {}", file_path.display()))?; + } + Ok(()) +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs deleted file mode 100644 index cf01329e9b1..00000000000 --- a/pyo3-introspection/tests/test.rs +++ /dev/null @@ -1,106 +0,0 @@ -use anyhow::{ensure, Result}; -use pyo3_introspection::{introspect_cdylib, module_stub_files}; -use std::collections::HashMap; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, fs}; -use tempfile::NamedTempFile; - -#[test] -fn pytests_stubs() -> Result<()> { - // We run the introspection - let binary = env::var_os("PYO3_PYTEST_LIB_PATH") - .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); - let module = introspect_cdylib(binary, "pyo3_pytests")?; - let actual_stubs = module_stub_files(&module); - - // We read the expected stubs - let expected_subs_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("pytests") - .join("stubs"); - let mut expected_subs = HashMap::new(); - add_dir_files( - &expected_subs_dir, - &expected_subs_dir.canonicalize()?, - &mut expected_subs, - )?; - - // We ensure we do not have extra generated files - for file_name in actual_stubs.keys() { - assert!( - expected_subs.contains_key(file_name), - "The generated file {} is not in the expected stubs directory pytests/stubs", - file_name.display() - ); - } - - // We ensure the expected files are generated properly - for (file_name, expected_file_content) in &expected_subs { - let actual_file_content = actual_stubs.get(file_name).unwrap_or_else(|| { - panic!( - "The expected stub file {} has not been generated", - file_name.display() - ) - }); - - let actual_file_content = format_with_ruff(actual_file_content)?; - - // We normalize line jumps for compatibility with Windows - assert_eq!( - expected_file_content.replace('\r', ""), - actual_file_content.replace('\r', ""), - "The content of file {} is different", - file_name.display() - ) - } - - Ok(()) -} - -fn add_dir_files( - dir_path: &Path, - base_dir_path: &Path, - output: &mut HashMap, -) -> Result<()> { - for entry in fs::read_dir(dir_path)? { - let entry = entry?; - if entry.file_type()?.is_dir() { - add_dir_files(&entry.path(), base_dir_path, output)?; - } else { - output.insert( - entry - .path() - .canonicalize()? - .strip_prefix(base_dir_path)? - .into(), - fs::read_to_string(entry.path())?, - ); - } - } - Ok(()) -} - -fn format_with_ruff(code: &str) -> Result { - let temp_file = NamedTempFile::with_suffix(".pyi")?; - // Write to file - { - let mut file = temp_file.as_file(); - file.write_all(code.as_bytes())?; - file.flush()?; - file.seek(SeekFrom::Start(0))?; - } - ensure!( - Command::new("ruff") - .arg("format") - .arg(temp_file.path()) - .status()? - .success(), - "Failed to run ruff" - ); - let mut content = String::new(); - temp_file.as_file().read_to_string(&mut content)?; - Ok(content) -} From b2828f3c6bce9c6c7d5d24c50397fcc423cbc8f9 Mon Sep 17 00:00:00 2001 From: Thomas Pellissier-Tanon Date: Wed, 25 Mar 2026 18:17:48 +0100 Subject: [PATCH 2/3] Makes sure to inject the module name --- noxfile.py | 11 ++++++++++- pyo3-introspection/src/main.rs | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 1e5b7fd52ea..dcea53ff427 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1453,7 +1453,16 @@ def test_introspection(session: nox.Session): "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", silent=True, ).strip() - _run_cargo(session, "run", "-p", "pyo3-introspection", "--", lib_file, stub_dir) + _run_cargo( + session, + "run", + "-p", + "pyo3-introspection", + "--", + lib_file, + "pyo3_pytests", + stub_dir, + ) _run(session, "ruff", "format", stub_dir) _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs")) diff --git a/pyo3-introspection/src/main.rs b/pyo3-introspection/src/main.rs index 6a5c6506e6c..f0c64031da2 100644 --- a/pyo3-introspection/src/main.rs +++ b/pyo3-introspection/src/main.rs @@ -6,8 +6,8 @@ use std::path::Path; use std::{env, fs}; fn main() -> Result<()> { - let [_, binary_path, output_path] = env::args().collect::>().try_into().map_err(|_| anyhow!("pyo3-introspection takes two arguments, the path of the binary to introspect and the path of the directory to write the stub to"))?; - let module = introspect_cdylib(&binary_path, "pyo3_pytests") + let [_, binary_path, module_name, output_path] = env::args().collect::>().try_into().map_err(|_| anyhow!("pyo3-introspection takes three arguments, the path of the binary to introspect, the name of the python module to introspect and and the path of the directory to write the stub to"))?; + let module = introspect_cdylib(&binary_path, &module_name) .with_context(|| format!("Failed to introspect module {binary_path}"))?; let actual_stubs = module_stub_files(&module); for (path, module) in actual_stubs { From c697543c27c1f177c7d7145ec9b7a1b04578a641 Mon Sep 17 00:00:00 2001 From: Thomas Pellissier-Tanon Date: Wed, 29 Apr 2026 15:59:46 +0200 Subject: [PATCH 3/3] Add documentation --- guide/src/type-stub.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/guide/src/type-stub.md b/guide/src/type-stub.md index 0ac11a54e5f..3069c88434d 100644 --- a/guide/src/type-stub.md +++ b/guide/src/type-stub.md @@ -11,7 +11,7 @@ It works using: 1. PyO3 macros (`#[pyclass]`) that generate constant JSON strings that are then included in the built binaries by rustc if the `experimental-inspect` feature is enabled. 2. The `pyo3-introspection` crate that can parse the generated binaries, extract the JSON strings and build stub files from it. -3. \[Not done yet\] Build tools like `maturin` exposing `pyo3-introspection` features in their CLI API. +3. Build tools like `maturin` exposing options in their CLI API to generate the stubs file. For example, the following Rust code @@ -72,6 +72,11 @@ def list_of_int_identity(arg: "list[int]") -> "list[int]": ... The only piece of new syntax is that the `#[pyo3(signature = ...)]` attribute can contain type annotations like `#[pyo3(signature = (arg: "list[int]") -> "list[int]")]` (note the `""` around type annotations). This is useful when PyO3 is not able to derive proper type annotations by itself. +To generate stubs file with `maturin` you can use `maturin generate-stubs --output stubs` that will build the project then generate the stubs in the `stubs` directory. +You can also directly integrate the stubs in the built wheels by doing `maturin build --generate-stubs` (works also with `maturin develop`). + +PyO3 also provides the smaller `pyo3-introspection` binary that allows to generate stubs from an existing built extension using something like `pyo3-introspection my_module_binary.so my_module_name output` to introspect the `my_module_binary.so` dynamic library for the `my_module_name` Python module and generate its stubs in the `output` directory. + ## Constraints and limitations - The `experimental-inspect` feature is required to generate the introspection fragments.